Skip to main content

rustc_target/callconv/
mod.rs

1use std::{fmt, iter};
2
3use arrayvec::ArrayVec;
4use rustc_abi::{
5    AddressSpace, Align, BackendRepr, CanonAbi, ExternAbi, FieldsShape, HasDataLayout, Primitive,
6    Reg, RegKind, Scalar, Size, TyAbiInterface, TyAndLayout, Variants,
7};
8use rustc_macros::StableHash;
9
10pub use crate::spec::AbiMap;
11use crate::spec::{Arch, HasTargetSpec, HasX86AbiOpt, RustcAbi};
12
13mod aarch64;
14mod amdgpu;
15mod arm;
16mod avr;
17mod bpf;
18mod csky;
19mod hexagon;
20mod loongarch;
21mod m68k;
22mod mips;
23mod mips64;
24mod msp430;
25mod nvptx64;
26mod powerpc;
27mod powerpc64;
28mod riscv;
29mod s390x;
30mod sparc;
31mod sparc64;
32mod wasm;
33mod x86;
34mod x86_64;
35mod x86_win32;
36mod x86_win64;
37mod xtensa;
38
39/// Different modes in which indirect arguments can be passed.
40#[derive(#[automatically_derived]impl ::core::marker::Copy for IndirectMode { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for IndirectMode {
}
#[automatically_derived]impl ::core::clone::Clone for IndirectMode {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    IndirectMode {
}
#[automatically_derived]impl ::core::cmp::PartialEq for IndirectMode {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
            ::core::intrinsics::discriminant_value(other)
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for IndirectMode { }Eq, #[automatically_derived]impl ::core::hash::Hash for IndirectMode {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&::core::intrinsics::discriminant_value(self),
            state)
    }
}Hash, #[automatically_derived]impl ::core::fmt::Debug for IndirectMode {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                IndirectMode::Pointer => "Pointer",
                IndirectMode::OnStack => "OnStack",
                IndirectMode::AmdgpuKernelArg => "AmdgpuKernelArg",
            })
    }
}Debug, const _: () =
    {
        impl ::rustc_data_structures::stable_hash::StableHash for IndirectMode
            {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    IndirectMode::Pointer => 0u8.stable_hash(__hcx, __hasher),
                    IndirectMode::OnStack => 1u8.stable_hash(__hcx, __hasher),
                    IndirectMode::AmdgpuKernelArg =>
                        2u8.stable_hash(__hcx, __hasher),
                }
                match *self {
                    IndirectMode::Pointer => {}
                    IndirectMode::OnStack => {}
                    IndirectMode::AmdgpuKernelArg => {}
                }
            }
        }
    };StableHash)]
41pub enum IndirectMode {
42    /// Passed as a normal pointer, nothing special.
43    Pointer,
44    /// The value should be passed at a fixed stack offset in accordance to
45    /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument
46    /// attribute. The `byval` argument will use a byte array with the same size as the Rust type
47    /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout),
48    /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's
49    /// alignment (if `None`). This means that the alignment will not always
50    /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info.
51    OnStack,
52    /// `AmdgpuKernelArg` behaves similar to `OnStack` except that the pointer does not necessarily
53    /// point to the stack, no extra copy is made, and the passed argument should not be modified.
54    /// This corresponds to the `byref` LLVM argument attribute.
55    AmdgpuKernelArg,
56}
57
58#[derive(#[automatically_derived]impl ::core::clone::Clone for PassMode {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        match self {
            Self::Ignore => PassMode::Ignore,
            Self::Direct(__self_0) =>
                PassMode::Direct(::core::clone::Clone::clone(__self_0)),
            Self::Pair(__self_0, __self_1) =>
                PassMode::Pair(::core::clone::Clone::clone(__self_0),
                    ::core::clone::Clone::clone(__self_1)),
            Self::Cast { pad_i32_count: __self_0, cast: __self_1 } =>
                PassMode::Cast {
                    pad_i32_count: ::core::clone::Clone::clone(__self_0),
                    cast: ::core::clone::Clone::clone(__self_1),
                },
            Self::Indirect {
                attrs: __self_0, address_space: __self_1, mode: __self_2 } =>
                PassMode::Indirect {
                    attrs: ::core::clone::Clone::clone(__self_0),
                    address_space: ::core::clone::Clone::clone(__self_1),
                    mode: ::core::clone::Clone::clone(__self_2),
                },
            Self::IndirectUnsized { attrs: __self_0, meta_attrs: __self_1 } =>
                PassMode::IndirectUnsized {
                    attrs: ::core::clone::Clone::clone(__self_0),
                    meta_attrs: ::core::clone::Clone::clone(__self_1),
                },
        }
    }
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for PassMode
    {
}
#[automatically_derived]impl ::core::cmp::PartialEq for PassMode {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
                ::core::intrinsics::discriminant_value(other) &&
            match (self, other) {
                (Self::Direct(__self_0), Self::Direct(__arg1_0)) =>
                    __self_0 == __arg1_0,
                (Self::Pair(__self_0, __self_1),
                    Self::Pair(__arg1_0, __arg1_1)) =>
                    __self_0 == __arg1_0 && __self_1 == __arg1_1,
                (Self::Cast { pad_i32_count: __self_0, cast: __self_1 },
                    Self::Cast { pad_i32_count: __arg1_0, cast: __arg1_1 }) =>
                    __self_0 == __arg1_0 && __self_1 == __arg1_1,
                (Self::Indirect {
                    attrs: __self_0, address_space: __self_1, mode: __self_2 },
                    Self::Indirect {
                    attrs: __arg1_0, address_space: __arg1_1, mode: __arg1_2 })
                    =>
                    __self_0 == __arg1_0 && __self_1 == __arg1_1 &&
                        __self_2 == __arg1_2,
                (Self::IndirectUnsized { attrs: __self_0, meta_attrs: __self_1
                    }, Self::IndirectUnsized {
                    attrs: __arg1_0, meta_attrs: __arg1_1 }) =>
                    __self_0 == __arg1_0 && __self_1 == __arg1_1,
                _ => true,
            }
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for PassMode {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<ArgAttributes>;
        let _: ::core::cmp::AssertParamIsEq<u8>;
        let _: ::core::cmp::AssertParamIsEq<Box<CastTarget>>;
        let _: ::core::cmp::AssertParamIsEq<Option<AddressSpace>>;
        let _: ::core::cmp::AssertParamIsEq<IndirectMode>;
    }
}Eq, #[automatically_derived]impl ::core::hash::Hash for PassMode {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&::core::intrinsics::discriminant_value(self),
            state);
        match self {
            Self::Direct(__self_0) =>
                ::core::hash::Hash::hash(__self_0, state),
            Self::Pair(__self_0, __self_1) => {
                ::core::hash::Hash::hash(__self_0, state);
                ::core::hash::Hash::hash(__self_1, state)
            }
            Self::Cast { pad_i32_count: __self_0, cast: __self_1 } => {
                ::core::hash::Hash::hash(__self_0, state);
                ::core::hash::Hash::hash(__self_1, state)
            }
            Self::Indirect {
                attrs: __self_0, address_space: __self_1, mode: __self_2 } =>
                {
                ::core::hash::Hash::hash(__self_0, state);
                ::core::hash::Hash::hash(__self_1, state);
                ::core::hash::Hash::hash(__self_2, state)
            }
            Self::IndirectUnsized { attrs: __self_0, meta_attrs: __self_1 } =>
                {
                ::core::hash::Hash::hash(__self_0, state);
                ::core::hash::Hash::hash(__self_1, state)
            }
            _ => {}
        }
    }
}Hash, #[automatically_derived]impl ::core::fmt::Debug for PassMode {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        match self {
            Self::Ignore => ::core::fmt::Formatter::write_str(f, "Ignore"),
            Self::Direct(__self_0) =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "Direct", &__self_0),
            Self::Pair(__self_0, __self_1) =>
                ::core::fmt::Formatter::debug_struct_field2_finish(f, "Pair",
                    __self_0, &__self_1),
            Self::Cast { pad_i32_count: __self_0, cast: __self_1 } =>
                ::core::fmt::Formatter::debug_struct_field2_finish(f,
                    "Cast pad_i32_count cast", __self_0, &__self_1),
            Self::Indirect {
                attrs: __self_0, address_space: __self_1, mode: __self_2 } =>
                ::core::fmt::Formatter::debug_struct_field3_finish(f,
                    "Indirect attrs address_space mode", __self_0, __self_1,
                    &__self_2),
            Self::IndirectUnsized { attrs: __self_0, meta_attrs: __self_1 } =>
                ::core::fmt::Formatter::debug_struct_field2_finish(f,
                    "IndirectUnsized attrs meta_attrs", __self_0, &__self_1),
        }
    }
}Debug, const _: () =
    {
        impl ::rustc_data_structures::stable_hash::StableHash for PassMode {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    PassMode::Ignore => 0u8.stable_hash(__hcx, __hasher),
                    PassMode::Direct(ref __binding_0) =>
                        1u8.stable_hash(__hcx, __hasher),
                    PassMode::Pair(ref __binding_0, ref __binding_1) =>
                        2u8.stable_hash(__hcx, __hasher),
                    PassMode::Cast {
                        pad_i32_count: ref __binding_0, cast: ref __binding_1 } =>
                        3u8.stable_hash(__hcx, __hasher),
                    PassMode::Indirect {
                        attrs: ref __binding_0,
                        address_space: ref __binding_1,
                        mode: ref __binding_2 } => 4u8.stable_hash(__hcx, __hasher),
                    PassMode::IndirectUnsized {
                        attrs: ref __binding_0, meta_attrs: ref __binding_1 } =>
                        5u8.stable_hash(__hcx, __hasher),
                }
                match *self {
                    PassMode::Ignore => {}
                    PassMode::Direct(ref __binding_0) => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                    }
                    PassMode::Pair(ref __binding_0, ref __binding_1) => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                    }
                    PassMode::Cast {
                        pad_i32_count: ref __binding_0, cast: ref __binding_1 } => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                    }
                    PassMode::Indirect {
                        attrs: ref __binding_0,
                        address_space: ref __binding_1,
                        mode: ref __binding_2 } => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                        { __binding_2.stable_hash(__hcx, __hasher); }
                    }
                    PassMode::IndirectUnsized {
                        attrs: ref __binding_0, meta_attrs: ref __binding_1 } => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                    }
                }
            }
        }
    };StableHash)]
59pub enum PassMode {
60    /// Ignore the argument.
61    ///
62    /// The argument is a ZST.
63    Ignore,
64    /// Pass the argument directly.
65    ///
66    /// The argument has a layout abi of `Scalar` or `Vector`.
67    /// Unfortunately due to past mistakes, in rare cases on wasm, it can also be `Aggregate`.
68    /// This is bad since it leaks LLVM implementation details into the ABI.
69    /// (Also see <https://github.com/rust-lang/rust/issues/115666>.)
70    Direct(ArgAttributes),
71    /// Pass a pair's elements directly in two arguments.
72    ///
73    /// The argument has a layout abi of `ScalarPair`.
74    Pair(ArgAttributes, ArgAttributes),
75    /// Pass the argument after casting it. See the `CastTarget` docs for details.
76    ///
77    /// `pad_i32` indicates how many `Reg::i32()` dummy arguments are emitted before the real
78    /// argument.
79    Cast { pad_i32_count: u8, cast: Box<CastTarget> },
80    /// Pass the argument indirectly via a hidden pointer.
81    ///
82    /// `address_space` specifies if the pointer is in a special address space or the default one.
83    ///
84    /// `mode` can be a special way to pass an argument indirectly.
85    Indirect { attrs: ArgAttributes, address_space: Option<AddressSpace>, mode: IndirectMode },
86
87    /// Like `Indirect`, but for unsized parameters.
88    ///
89    /// The `meta_attrs` value, is for the metadata (vtable or length) of an unsized argument.
90    /// (This is the only mode that supports unsized arguments.) `on_stack` is always false.
91    IndirectUnsized { attrs: ArgAttributes, meta_attrs: ArgAttributes },
92}
93
94impl PassMode {
95    /// Checks if these two `PassMode` are equal enough to be considered "the same for all
96    /// function call ABIs". However, the `Layout` can also impact ABI decisions,
97    /// so that needs to be compared as well!
98    pub fn eq_abi(&self, other: &Self) -> bool {
99        match (self, other) {
100            (PassMode::Ignore, PassMode::Ignore) => true,
101            (PassMode::Direct(a1), PassMode::Direct(a2)) => a1.eq_abi(a2),
102            (PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => a1.eq_abi(a2) && b1.eq_abi(b2),
103            (
104                PassMode::Cast { cast: c1, pad_i32_count: pad1 },
105                PassMode::Cast { cast: c2, pad_i32_count: pad2 },
106            ) => c1.eq_abi(c2) && pad1 == pad2,
107            (
108                PassMode::Indirect { attrs: a1, address_space: as1, mode: m1 },
109                PassMode::Indirect { attrs: a2, address_space: as2, mode: m2 },
110            ) => a1.eq_abi(a2) && as1 == as2 && m1 == m2,
111            (
112                PassMode::IndirectUnsized { attrs: a1, meta_attrs: e1 },
113                PassMode::IndirectUnsized { attrs: a2, meta_attrs: e2 },
114            ) => a1.eq_abi(a2) && e1.eq_abi(e2),
115            _ => false,
116        }
117    }
118}
119
120// Hack to disable non_upper_case_globals only for the bitflags! and not for the rest
121// of this module
122pub use attr_impl::ArgAttribute;
123
124#[allow(non_upper_case_globals)]
125#[allow(unused)]
126mod attr_impl {
127    use rustc_macros::StableHash;
128
129    // The subset of llvm::Attribute needed for arguments, packed into a bitfield.
130    #[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for ArgAttribute {
}
#[automatically_derived]impl ::core::clone::Clone for ArgAttribute {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<u16>;
        *self
    }
}Clone, #[automatically_derived]impl ::core::marker::Copy for ArgAttribute { }Copy, #[automatically_derived]impl ::core::default::Default for ArgAttribute {
    #[inline]
    fn default() -> Self { ArgAttribute(::core::default::Default::default()) }
}Default, #[automatically_derived]impl ::core::hash::Hash for ArgAttribute {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&self.0, state)
    }
}Hash, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    ArgAttribute {
}
#[automatically_derived]impl ::core::cmp::PartialEq for ArgAttribute {
    #[inline]
    fn eq(&self, other: &Self) -> bool { self.0 == other.0 }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for ArgAttribute {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<u16>;
    }
}Eq, const _: () =
    {
        impl ::rustc_data_structures::stable_hash::StableHash for ArgAttribute
            {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    ArgAttribute(ref __binding_0) => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                    }
                }
            }
        }
    };StableHash)]
131    pub struct ArgAttribute(u16);
132    impl ArgAttribute {
    #[allow(deprecated, non_upper_case_globals,)]
    pub const CapturesNone: Self = Self::from_bits_retain(0b111);
    #[allow(deprecated, non_upper_case_globals,)]
    pub const CapturesAddress: Self = Self::from_bits_retain(0b110);
    #[allow(deprecated, non_upper_case_globals,)]
    pub const CapturesReadOnly: Self = Self::from_bits_retain(0b100);
    #[allow(deprecated, non_upper_case_globals,)]
    pub const NoAlias: Self = Self::from_bits_retain(1 << 3);
    #[allow(deprecated, non_upper_case_globals,)]
    pub const NonNull: Self = Self::from_bits_retain(1 << 4);
    #[allow(deprecated, non_upper_case_globals,)]
    pub const ReadOnly: Self = Self::from_bits_retain(1 << 5);
    #[allow(deprecated, non_upper_case_globals,)]
    pub const InReg: Self = Self::from_bits_retain(1 << 6);
    #[allow(deprecated, non_upper_case_globals,)]
    pub const NoUndef: Self = Self::from_bits_retain(1 << 7);
    #[allow(deprecated, non_upper_case_globals,)]
    pub const Writable: Self = Self::from_bits_retain(1 << 8);
    #[doc =
    r" It is UB for this pointer or any pointer derived from it to be used for"]
    #[doc =
    r" deallocation (except for zero-sized deallocation) while the function is"]
    #[doc =
    r" executing. Only valid on arguments (including return values that are passed"]
    #[doc = r" indirectly as arguments)."]
    #[allow(deprecated, non_upper_case_globals,)]
    pub const NoFree: Self = Self::from_bits_retain(1 << 9);
}
impl ::bitflags::Flags for ArgAttribute {
    const FLAGS: &'static [::bitflags::Flag<ArgAttribute>] =
        &[{

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("CapturesNone",
                            ArgAttribute::CapturesNone)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("CapturesAddress",
                            ArgAttribute::CapturesAddress)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("CapturesReadOnly",
                            ArgAttribute::CapturesReadOnly)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("NoAlias", ArgAttribute::NoAlias)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("NonNull", ArgAttribute::NonNull)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("ReadOnly", ArgAttribute::ReadOnly)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("InReg", ArgAttribute::InReg)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("NoUndef", ArgAttribute::NoUndef)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("Writable", ArgAttribute::Writable)
                    },
                    {

                        #[allow(deprecated, non_upper_case_globals,)]
                        ::bitflags::Flag::new("NoFree", ArgAttribute::NoFree)
                    }];
    type Bits = u16;
    fn bits(&self) -> u16 { ArgAttribute::bits(self) }
    fn from_bits_retain(bits: u16) -> ArgAttribute {
        ArgAttribute::from_bits_retain(bits)
    }
}
#[allow(dead_code, deprecated, unused_doc_comments, unused_attributes,
unused_mut, unused_imports, non_upper_case_globals, clippy ::
assign_op_pattern, clippy :: iter_without_into_iter,)]
const _: () =
    {
        #[allow(dead_code, deprecated, unused_attributes)]
        impl ArgAttribute {
            /// Get a flags value with all bits unset.
            #[inline]
            pub const fn empty() -> Self {
                Self(<u16 as ::bitflags::Bits>::EMPTY)
            }
            /// Get a flags value with all known bits set.
            #[inline]
            pub const fn all() -> Self {
                let mut truncated = <u16 as ::bitflags::Bits>::EMPTY;
                let mut i = 0;
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                {
                    {
                        let flag =
                            <ArgAttribute as
                                            ::bitflags::Flags>::FLAGS[i].value().bits();
                        truncated = truncated | flag;
                        i += 1;
                    }
                };
                let _ = i;
                Self(truncated)
            }
            /// Get the underlying bits value.
            ///
            /// The returned value is exactly the bits set in this flags value.
            #[inline]
            pub const fn bits(&self) -> u16 { self.0 }
            /// Convert from a bits value.
            ///
            /// This method will return `None` if any unknown bits are set.
            #[inline]
            pub const fn from_bits(bits: u16)
                -> ::bitflags::__private::core::option::Option<Self> {
                let truncated = Self::from_bits_truncate(bits).0;
                if truncated == bits {
                    ::bitflags::__private::core::option::Option::Some(Self(bits))
                } else { ::bitflags::__private::core::option::Option::None }
            }
            /// Convert from a bits value, unsetting any unknown bits.
            #[inline]
            pub const fn from_bits_truncate(bits: u16) -> Self {
                Self(bits & Self::all().0)
            }
            /// Convert from a bits value exactly.
            #[inline]
            pub const fn from_bits_retain(bits: u16) -> Self { Self(bits) }
            /// Get a flags value with the bits of a flag with the given name set.
            ///
            /// This method will return `None` if `name` is empty or doesn't
            /// correspond to any named flag.
            #[inline]
            pub fn from_name(name: &str)
                -> ::bitflags::__private::core::option::Option<Self> {
                {
                    if name == "CapturesNone" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::CapturesNone.bits()));
                    }
                };
                ;
                {
                    if name == "CapturesAddress" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::CapturesAddress.bits()));
                    }
                };
                ;
                {
                    if name == "CapturesReadOnly" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::CapturesReadOnly.bits()));
                    }
                };
                ;
                {
                    if name == "NoAlias" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::NoAlias.bits()));
                    }
                };
                ;
                {
                    if name == "NonNull" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::NonNull.bits()));
                    }
                };
                ;
                {
                    if name == "ReadOnly" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::ReadOnly.bits()));
                    }
                };
                ;
                {
                    if name == "InReg" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::InReg.bits()));
                    }
                };
                ;
                {
                    if name == "NoUndef" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::NoUndef.bits()));
                    }
                };
                ;
                {
                    if name == "Writable" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::Writable.bits()));
                    }
                };
                ;
                {
                    if name == "NoFree" {
                        return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::NoFree.bits()));
                    }
                };
                ;
                let _ = name;
                ::bitflags::__private::core::option::Option::None
            }
            /// Whether all bits in this flags value are unset.
            #[inline]
            pub const fn is_empty(&self) -> bool {
                self.0 == <u16 as ::bitflags::Bits>::EMPTY
            }
            /// Whether all known bits in this flags value are set.
            #[inline]
            pub const fn is_all(&self) -> bool {
                Self::all().0 | self.0 == self.0
            }
            /// Whether any set bits in a source flags value are also set in a target flags value.
            #[inline]
            pub const fn intersects(&self, other: Self) -> bool {
                self.0 & other.0 != <u16 as ::bitflags::Bits>::EMPTY
            }
            /// Whether all set bits in a source flags value are also set in a target flags value.
            #[inline]
            pub const fn contains(&self, other: Self) -> bool {
                self.0 & other.0 == other.0
            }
            /// The bitwise or (`|`) of the bits in two flags values.
            #[inline]
            pub fn insert(&mut self, other: Self) {
                *self = Self(self.0).union(other);
            }
            /// The intersection of a source flags value with the complement of a target flags
            /// value (`&!`).
            ///
            /// This method is not equivalent to `self & !other` when `other` has unknown bits set.
            /// `remove` won't truncate `other`, but the `!` operator will.
            #[inline]
            pub fn remove(&mut self, other: Self) {
                *self = Self(self.0).difference(other);
            }
            /// The bitwise exclusive-or (`^`) of the bits in two flags values.
            #[inline]
            pub fn toggle(&mut self, other: Self) {
                *self = Self(self.0).symmetric_difference(other);
            }
            /// Call `insert` when `value` is `true` or `remove` when `value` is `false`.
            #[inline]
            pub fn set(&mut self, other: Self, value: bool) {
                if value { self.insert(other); } else { self.remove(other); }
            }
            /// The bitwise and (`&`) of the bits in two flags values.
            #[inline]
            #[must_use]
            pub const fn intersection(self, other: Self) -> Self {
                Self(self.0 & other.0)
            }
            /// The bitwise or (`|`) of the bits in two flags values.
            #[inline]
            #[must_use]
            pub const fn union(self, other: Self) -> Self {
                Self(self.0 | other.0)
            }
            /// The intersection of a source flags value with the complement of a target flags
            /// value (`&!`).
            ///
            /// This method is not equivalent to `self & !other` when `other` has unknown bits set.
            /// `difference` won't truncate `other`, but the `!` operator will.
            #[inline]
            #[must_use]
            pub const fn difference(self, other: Self) -> Self {
                Self(self.0 & !other.0)
            }
            /// The bitwise exclusive-or (`^`) of the bits in two flags values.
            #[inline]
            #[must_use]
            pub const fn symmetric_difference(self, other: Self) -> Self {
                Self(self.0 ^ other.0)
            }
            /// The bitwise negation (`!`) of the bits in a flags value, truncating the result.
            #[inline]
            #[must_use]
            pub const fn complement(self) -> Self {
                Self::from_bits_truncate(!self.0)
            }
        }
        impl ::bitflags::__private::core::fmt::Binary for ArgAttribute {
            fn fmt(&self, f: &mut ::bitflags::__private::core::fmt::Formatter)
                -> ::bitflags::__private::core::fmt::Result {
                let inner = self.0;
                ::bitflags::__private::core::fmt::Binary::fmt(&inner, f)
            }
        }
        impl ::bitflags::__private::core::fmt::Octal for ArgAttribute {
            fn fmt(&self, f: &mut ::bitflags::__private::core::fmt::Formatter)
                -> ::bitflags::__private::core::fmt::Result {
                let inner = self.0;
                ::bitflags::__private::core::fmt::Octal::fmt(&inner, f)
            }
        }
        impl ::bitflags::__private::core::fmt::LowerHex for ArgAttribute {
            fn fmt(&self, f: &mut ::bitflags::__private::core::fmt::Formatter)
                -> ::bitflags::__private::core::fmt::Result {
                let inner = self.0;
                ::bitflags::__private::core::fmt::LowerHex::fmt(&inner, f)
            }
        }
        impl ::bitflags::__private::core::fmt::UpperHex for ArgAttribute {
            fn fmt(&self, f: &mut ::bitflags::__private::core::fmt::Formatter)
                -> ::bitflags::__private::core::fmt::Result {
                let inner = self.0;
                ::bitflags::__private::core::fmt::UpperHex::fmt(&inner, f)
            }
        }
        impl ::bitflags::__private::core::ops::BitOr for ArgAttribute {
            type Output = Self;
            /// The bitwise or (`|`) of the bits in two flags values.
            #[inline]
            fn bitor(self, other: ArgAttribute) -> Self { self.union(other) }
        }
        impl ::bitflags::__private::core::ops::BitOrAssign for ArgAttribute {
            /// The bitwise or (`|`) of the bits in two flags values.
            #[inline]
            fn bitor_assign(&mut self, other: Self) { self.insert(other); }
        }
        impl ::bitflags::__private::core::ops::BitXor for ArgAttribute {
            type Output = Self;
            /// The bitwise exclusive-or (`^`) of the bits in two flags values.
            #[inline]
            fn bitxor(self, other: Self) -> Self {
                self.symmetric_difference(other)
            }
        }
        impl ::bitflags::__private::core::ops::BitXorAssign for ArgAttribute {
            /// The bitwise exclusive-or (`^`) of the bits in two flags values.
            #[inline]
            fn bitxor_assign(&mut self, other: Self) { self.toggle(other); }
        }
        impl ::bitflags::__private::core::ops::BitAnd for ArgAttribute {
            type Output = Self;
            /// The bitwise and (`&`) of the bits in two flags values.
            #[inline]
            fn bitand(self, other: Self) -> Self { self.intersection(other) }
        }
        impl ::bitflags::__private::core::ops::BitAndAssign for ArgAttribute {
            /// The bitwise and (`&`) of the bits in two flags values.
            #[inline]
            fn bitand_assign(&mut self, other: Self) {
                *self =
                    Self::from_bits_retain(self.bits()).intersection(other);
            }
        }
        impl ::bitflags::__private::core::ops::Sub for ArgAttribute {
            type Output = Self;
            /// The intersection of a source flags value with the complement of a target flags value (`&!`).
            ///
            /// This method is not equivalent to `self & !other` when `other` has unknown bits set.
            /// `difference` won't truncate `other`, but the `!` operator will.
            #[inline]
            fn sub(self, other: Self) -> Self { self.difference(other) }
        }
        impl ::bitflags::__private::core::ops::SubAssign for ArgAttribute {
            /// The intersection of a source flags value with the complement of a target flags value (`&!`).
            ///
            /// This method is not equivalent to `self & !other` when `other` has unknown bits set.
            /// `difference` won't truncate `other`, but the `!` operator will.
            #[inline]
            fn sub_assign(&mut self, other: Self) { self.remove(other); }
        }
        impl ::bitflags::__private::core::ops::Not for ArgAttribute {
            type Output = Self;
            /// The bitwise negation (`!`) of the bits in a flags value, truncating the result.
            #[inline]
            fn not(self) -> Self { self.complement() }
        }
        impl ::bitflags::__private::core::iter::Extend<ArgAttribute> for
            ArgAttribute {
            /// The bitwise or (`|`) of the bits in each flags value.
            fn extend<T: ::bitflags::__private::core::iter::IntoIterator<Item
                = Self>>(&mut self, iterator: T) {
                for item in iterator { self.insert(item) }
            }
        }
        impl ::bitflags::__private::core::iter::FromIterator<ArgAttribute> for
            ArgAttribute {
            /// The bitwise or (`|`) of the bits in each flags value.
            fn from_iter<T: ::bitflags::__private::core::iter::IntoIterator<Item
                = Self>>(iterator: T) -> Self {
                use ::bitflags::__private::core::iter::Extend;
                let mut result = Self::empty();
                result.extend(iterator);
                result
            }
        }
        impl ArgAttribute {
            /// Yield a set of contained flags values.
            ///
            /// Each yielded flags value will correspond to a defined named flag. Any unknown bits
            /// will be yielded together as a final flags value.
            #[inline]
            pub const fn iter(&self) -> ::bitflags::iter::Iter<ArgAttribute> {
                ::bitflags::iter::Iter::__private_const_new(<ArgAttribute as
                        ::bitflags::Flags>::FLAGS,
                    ArgAttribute::from_bits_retain(self.bits()),
                    ArgAttribute::from_bits_retain(self.bits()))
            }
            /// Yield a set of contained named flags values.
            ///
            /// This method is like [`iter`](#method.iter), except only yields bits in contained named flags.
            /// Any unknown bits, or bits not corresponding to a contained flag will not be yielded.
            #[inline]
            pub const fn iter_names(&self)
                -> ::bitflags::iter::IterNames<ArgAttribute> {
                ::bitflags::iter::IterNames::__private_const_new(<ArgAttribute
                        as ::bitflags::Flags>::FLAGS,
                    ArgAttribute::from_bits_retain(self.bits()),
                    ArgAttribute::from_bits_retain(self.bits()))
            }
        }
        impl ::bitflags::__private::core::iter::IntoIterator for ArgAttribute
            {
            type Item = ArgAttribute;
            type IntoIter = ::bitflags::iter::Iter<ArgAttribute>;
            fn into_iter(self) -> Self::IntoIter { self.iter() }
        }
    };bitflags::bitflags! {
133        impl ArgAttribute: u16 {
134            const CapturesNone     = 0b111;
135            const CapturesAddress  = 0b110;
136            const CapturesReadOnly = 0b100;
137            const NoAlias  = 1 << 3;
138            const NonNull  = 1 << 4;
139            const ReadOnly = 1 << 5;
140            const InReg    = 1 << 6;
141            const NoUndef  = 1 << 7;
142            const Writable = 1 << 8;
143            /// It is UB for this pointer or any pointer derived from it to be used for
144            /// deallocation (except for zero-sized deallocation) while the function is
145            /// executing. Only valid on arguments (including return values that are passed
146            /// indirectly as arguments).
147            const NoFree   = 1 << 9;
148        }
149    }
150    impl ::std::fmt::Debug for ArgAttribute {
    fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
        ::bitflags::parser::to_writer(self, f)
    }
}rustc_data_structures::external_bitflags_debug! { ArgAttribute }
151}
152
153/// Sometimes an ABI requires small integers to be extended to a full or partial register. This enum
154/// defines if this extension should be zero-extension or sign-extension when necessary. When it is
155/// not necessary to extend the argument, this enum is ignored.
156#[derive(#[automatically_derived]impl ::core::marker::Copy for ArgExtension { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for ArgExtension {
}
#[automatically_derived]impl ::core::clone::Clone for ArgExtension {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    ArgExtension {
}
#[automatically_derived]impl ::core::cmp::PartialEq for ArgExtension {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
            ::core::intrinsics::discriminant_value(other)
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for ArgExtension { }Eq, #[automatically_derived]impl ::core::hash::Hash for ArgExtension {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&::core::intrinsics::discriminant_value(self),
            state)
    }
}Hash, #[automatically_derived]impl ::core::fmt::Debug for ArgExtension {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                ArgExtension::None => "None",
                ArgExtension::Zext => "Zext",
                ArgExtension::Sext => "Sext",
            })
    }
}Debug, const _: () =
    {
        impl ::rustc_data_structures::stable_hash::StableHash for ArgExtension
            {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    ArgExtension::None => 0u8.stable_hash(__hcx, __hasher),
                    ArgExtension::Zext => 1u8.stable_hash(__hcx, __hasher),
                    ArgExtension::Sext => 2u8.stable_hash(__hcx, __hasher),
                }
                match *self {
                    ArgExtension::None => {}
                    ArgExtension::Zext => {}
                    ArgExtension::Sext => {}
                }
            }
        }
    };StableHash)]
157pub enum ArgExtension {
158    None,
159    Zext,
160    Sext,
161}
162
163/// A compact representation of LLVM attributes (at least those relevant for this module)
164/// that can be manipulated without interacting with LLVM's Attribute machinery.
165#[derive(#[automatically_derived]impl ::core::marker::Copy for ArgAttributes { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for ArgAttributes {
}
#[automatically_derived]impl ::core::clone::Clone for ArgAttributes {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<ArgAttribute>;
        let _: ::core::clone::AssertParamIsClone<ArgExtension>;
        let _: ::core::clone::AssertParamIsClone<Size>;
        let _: ::core::clone::AssertParamIsClone<Option<Align>>;
        *self
    }
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    ArgAttributes {
}
#[automatically_derived]impl ::core::cmp::PartialEq for ArgAttributes {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        self.regular == other.regular && self.arg_ext == other.arg_ext &&
                self.pointee_size == other.pointee_size &&
            self.pointee_align == other.pointee_align
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for ArgAttributes {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<ArgAttribute>;
        let _: ::core::cmp::AssertParamIsEq<ArgExtension>;
        let _: ::core::cmp::AssertParamIsEq<Size>;
        let _: ::core::cmp::AssertParamIsEq<Option<Align>>;
    }
}Eq, #[automatically_derived]impl ::core::hash::Hash for ArgAttributes {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&self.regular, state);
        ::core::hash::Hash::hash(&self.arg_ext, state);
        ::core::hash::Hash::hash(&self.pointee_size, state);
        ::core::hash::Hash::hash(&self.pointee_align, state)
    }
}Hash, #[automatically_derived]impl ::core::fmt::Debug for ArgAttributes {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field4_finish(f,
            "ArgAttributes regular arg_ext pointee_size pointee_align",
            &self.regular, &self.arg_ext, &self.pointee_size,
            &&self.pointee_align)
    }
}Debug, const _: () =
    {
        impl ::rustc_data_structures::stable_hash::StableHash for
            ArgAttributes {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    ArgAttributes {
                        regular: ref __binding_0,
                        arg_ext: ref __binding_1,
                        pointee_size: ref __binding_2,
                        pointee_align: ref __binding_3 } => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                        { __binding_2.stable_hash(__hcx, __hasher); }
                        { __binding_3.stable_hash(__hcx, __hasher); }
                    }
                }
            }
        }
    };StableHash)]
166pub struct ArgAttributes {
167    pub regular: ArgAttribute,
168    pub arg_ext: ArgExtension,
169    /// If the pointer is not null, the minimum dereferenceable size of the pointee, at the time of
170    /// function entry (for arguments) or function return (for return values).
171    pub pointee_size: Size,
172    /// The minimum alignment of the pointee, if any.
173    pub pointee_align: Option<Align>,
174}
175
176impl ArgAttributes {
177    pub fn new() -> Self {
178        ArgAttributes {
179            regular: ArgAttribute::default(),
180            arg_ext: ArgExtension::None,
181            pointee_size: Size::ZERO,
182            pointee_align: None,
183        }
184    }
185
186    pub fn ext(&mut self, ext: ArgExtension) -> &mut Self {
187        if !(self.arg_ext == ArgExtension::None || self.arg_ext == ext) {
    {
        ::core::panicking::panic_fmt(format_args!("cannot set {0:?} when {1:?} is already set",
                ext, self.arg_ext));
    }
};assert!(
188            self.arg_ext == ArgExtension::None || self.arg_ext == ext,
189            "cannot set {:?} when {:?} is already set",
190            ext,
191            self.arg_ext
192        );
193        self.arg_ext = ext;
194        self
195    }
196
197    pub fn set(&mut self, attr: ArgAttribute) -> &mut Self {
198        self.regular |= attr;
199        self
200    }
201
202    pub fn contains(&self, attr: ArgAttribute) -> bool {
203        self.regular.contains(attr)
204    }
205
206    /// Checks if these two `ArgAttributes` are equal enough to be considered "the same for all
207    /// function call ABIs".
208    pub fn eq_abi(&self, other: &Self) -> bool {
209        // There's only one regular attribute that matters for the call ABI: InReg.
210        // Everything else is things like noalias, dereferenceable, nonnull, ...
211        // (This also applies to pointee_size, pointee_align.)
212        if self.regular.contains(ArgAttribute::InReg) != other.regular.contains(ArgAttribute::InReg)
213        {
214            return false;
215        }
216        // We also compare the sign extension mode -- this could let the callee make assumptions
217        // about bits that conceptually were not even passed.
218        if self.arg_ext != other.arg_ext {
219            return false;
220        }
221        true
222    }
223}
224
225impl From<ArgAttribute> for ArgAttributes {
226    fn from(value: ArgAttribute) -> Self {
227        Self {
228            regular: value,
229            arg_ext: ArgExtension::None,
230            pointee_size: Size::ZERO,
231            pointee_align: None,
232        }
233    }
234}
235
236/// An argument passed entirely registers with the
237/// same kind (e.g., HFA / HVA on PPC64 and AArch64).
238#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for Uniform {
}
#[automatically_derived]impl ::core::clone::Clone for Uniform {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<Reg>;
        let _: ::core::clone::AssertParamIsClone<Size>;
        let _: ::core::clone::AssertParamIsClone<bool>;
        *self
    }
}Clone, #[automatically_derived]impl ::core::marker::Copy for Uniform { }Copy, #[automatically_derived]impl ::core::marker::StructuralPartialEq for Uniform {
}
#[automatically_derived]impl ::core::cmp::PartialEq for Uniform {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        self.is_consecutive == other.is_consecutive && self.unit == other.unit
            && self.total == other.total
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for Uniform {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<Reg>;
        let _: ::core::cmp::AssertParamIsEq<Size>;
        let _: ::core::cmp::AssertParamIsEq<bool>;
    }
}Eq, #[automatically_derived]impl ::core::hash::Hash for Uniform {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&self.unit, state);
        ::core::hash::Hash::hash(&self.total, state);
        ::core::hash::Hash::hash(&self.is_consecutive, state)
    }
}Hash, #[automatically_derived]impl ::core::fmt::Debug for Uniform {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field3_finish(f,
            "Uniform unit total is_consecutive", &self.unit, &self.total,
            &&self.is_consecutive)
    }
}Debug, const _: () =
    {
        impl ::rustc_data_structures::stable_hash::StableHash for Uniform {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    Uniform {
                        unit: ref __binding_0,
                        total: ref __binding_1,
                        is_consecutive: ref __binding_2 } => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                        { __binding_2.stable_hash(__hcx, __hasher); }
                    }
                }
            }
        }
    };StableHash)]
239pub struct Uniform {
240    pub unit: Reg,
241
242    /// The total size of the argument, which can be:
243    /// * equal to `unit.size` (one scalar/vector),
244    /// * a multiple of `unit.size` (an array of scalar/vectors),
245    /// * if `unit.kind` is `Integer`, the last element can be shorter, i.e., `{ i64, i64, i32 }`
246    ///   for 64-bit integers with a total size of 20 bytes. When the argument is actually passed,
247    ///   this size will be rounded up to the nearest multiple of `unit.size`.
248    pub total: Size,
249
250    /// Indicate that the argument is consecutive, in the sense that either all values need to be
251    /// passed in register, or all on the stack. If they are passed on the stack, there should be
252    /// no additional padding between elements.
253    pub is_consecutive: bool,
254}
255
256impl From<Reg> for Uniform {
257    fn from(unit: Reg) -> Uniform {
258        Uniform { unit, total: unit.size, is_consecutive: false }
259    }
260}
261
262impl Uniform {
263    /// Alignment used for the scratch space when this type is part of a `PassMode::Cast`.
264    pub fn scratch_align<C: HasDataLayout>(&self, cx: &C) -> Align {
265        self.unit.scratch_align(cx)
266    }
267
268    /// Pass using one or more values of the given type, without requiring them to be consecutive.
269    /// That is, some values may be passed in register and some on the stack.
270    pub fn new(unit: Reg, total: Size) -> Self {
271        Uniform { unit, total, is_consecutive: false }
272    }
273
274    /// Pass using one or more consecutive values of the given type. Either all values will be
275    /// passed in registers, or all on the stack.
276    pub fn consecutive(unit: Reg, total: Size) -> Self {
277        Uniform { unit, total, is_consecutive: true }
278    }
279}
280
281/// Describes the type used for `PassMode::Cast`.
282///
283/// Passing arguments in this mode works as follows: the registers in the `prefix` (the ones that
284/// are `Some`) get laid out one after the other (using `repr(C)` layout rules). Then the
285/// `rest.unit` register type gets repeated often enough to cover `rest.size`. This describes the
286/// actual type used for the call; the Rust type of the argument is then transmuted to this ABI type
287/// (and all data in the padding between the registers is dropped).
288#[derive(#[automatically_derived]impl ::core::clone::Clone for CastTarget {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        CastTarget {
            prefix: ::core::clone::Clone::clone(&self.prefix),
            rest_offset: ::core::clone::Clone::clone(&self.rest_offset),
            rest: ::core::clone::Clone::clone(&self.rest),
            attrs: ::core::clone::Clone::clone(&self.attrs),
            pass_via_x87_floating_point_stack: ::core::clone::Clone::clone(&self.pass_via_x87_floating_point_stack),
        }
    }
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    CastTarget {
}
#[automatically_derived]impl ::core::cmp::PartialEq for CastTarget {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        self.pass_via_x87_floating_point_stack ==
                            other.pass_via_x87_floating_point_stack &&
                        self.prefix == other.prefix &&
                    self.rest_offset == other.rest_offset &&
                self.rest == other.rest && self.attrs == other.attrs
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for CastTarget {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<ArrayVec<Reg, 8>>;
        let _: ::core::cmp::AssertParamIsEq<Option<Size>>;
        let _: ::core::cmp::AssertParamIsEq<Uniform>;
        let _: ::core::cmp::AssertParamIsEq<ArgAttributes>;
        let _: ::core::cmp::AssertParamIsEq<bool>;
    }
}Eq, #[automatically_derived]impl ::core::hash::Hash for CastTarget {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&self.prefix, state);
        ::core::hash::Hash::hash(&self.rest_offset, state);
        ::core::hash::Hash::hash(&self.rest, state);
        ::core::hash::Hash::hash(&self.attrs, state);
        ::core::hash::Hash::hash(&self.pass_via_x87_floating_point_stack,
            state)
    }
}Hash, #[automatically_derived]impl ::core::fmt::Debug for CastTarget {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field5_finish(f,
            "CastTarget prefix rest_offset rest attrs pass_via_x87_floating_point_stack",
            &self.prefix, &self.rest_offset, &self.rest, &self.attrs,
            &&self.pass_via_x87_floating_point_stack)
    }
}Debug, const _: () =
    {
        impl ::rustc_data_structures::stable_hash::StableHash for CastTarget {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    CastTarget {
                        prefix: ref __binding_0,
                        rest_offset: ref __binding_1,
                        rest: ref __binding_2,
                        attrs: ref __binding_3,
                        pass_via_x87_floating_point_stack: ref __binding_4 } => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                        { __binding_2.stable_hash(__hcx, __hasher); }
                        { __binding_3.stable_hash(__hcx, __hasher); }
                        { __binding_4.stable_hash(__hcx, __hasher); }
                    }
                }
            }
        }
    };StableHash)]
289pub struct CastTarget {
290    // Note that this is fixed to 8 elements for now as ABIs currently don't
291    // need anything further beyond that, and when this code was originally
292    // refactored to use `ArrayVec` it was already using 8, so that stuck
293    // around.
294    pub prefix: ArrayVec<Reg, 8>,
295    /// The offset of `rest` from the start of the value. Currently only implemented for a `Reg`
296    /// pair created by the `offset_pair` method.
297    pub rest_offset: Option<Size>,
298    pub rest: Uniform,
299    pub attrs: ArgAttributes,
300    /// If `true`, indicates that this type will be passed on the x87 floating point stack on
301    /// 32-bit x86, changing the backend type to `x87_f80`. This is needed as LLVM needs to generate
302    /// extra code to ensure that signalling NaNs are passed losslessly on the x87 floating point
303    /// stack. Only valid on return types on 32-bit x86 with a cast target of either `Reg::f32()` or
304    /// `Reg::f64()`.
305    pub pass_via_x87_floating_point_stack: bool,
306}
307
308impl From<Reg> for CastTarget {
309    fn from(unit: Reg) -> CastTarget {
310        CastTarget::from(Uniform::from(unit))
311    }
312}
313
314impl From<Uniform> for CastTarget {
315    fn from(uniform: Uniform) -> CastTarget {
316        Self::prefixed(Default::default(), uniform)
317    }
318}
319
320impl CastTarget {
321    pub fn prefixed(prefix: ArrayVec<Reg, 8>, rest: Uniform) -> Self {
322        Self {
323            prefix,
324            rest_offset: None,
325            rest,
326            attrs: ArgAttributes::new(),
327            pass_via_x87_floating_point_stack: false,
328        }
329    }
330
331    pub fn offset_pair(a: Reg, offset_from_start: Size, b: Reg) -> Self {
332        let mut prefix = ArrayVec::new();
333        prefix.push(a);
334        Self {
335            prefix,
336            rest_offset: Some(offset_from_start),
337            rest: b.into(),
338            attrs: ArgAttributes::new(),
339            pass_via_x87_floating_point_stack: false,
340        }
341    }
342
343    pub fn with_attrs(mut self, attrs: ArgAttributes) -> Self {
344        self.attrs = attrs;
345        self
346    }
347
348    pub fn pair(a: Reg, b: Reg) -> CastTarget {
349        let mut prefix = ArrayVec::new();
350        prefix.push(a);
351        Self::prefixed(prefix, Uniform::from(b))
352    }
353
354    /// When you only access the range containing valid data, you can use this unaligned size;
355    /// otherwise, use the safer `size` method.
356    pub fn unaligned_size<C: HasDataLayout>(&self, _cx: &C) -> Size {
357        // Prefix arguments are passed in specific designated registers
358        let prefix_size = if let Some(offset_from_start) = self.rest_offset {
359            offset_from_start
360        } else {
361            self.prefix.iter().map(|reg| reg.size).fold(Size::ZERO, |acc, size| acc + size)
362        };
363        // Remaining arguments are passed in chunks of the unit size
364        let rest_size =
365            self.rest.unit.size * self.rest.total.bytes().div_ceil(self.rest.unit.size.bytes());
366
367        prefix_size + rest_size
368    }
369
370    pub fn size<C: HasDataLayout>(&self, cx: &C) -> Size {
371        self.unaligned_size(cx).align_to(self.align(cx))
372    }
373
374    pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align {
375        self.prefix.iter().map(|reg| reg.scratch_align(cx)).fold(
376            cx.data_layout().aggregate_align.max(self.rest.scratch_align(cx)),
377            |acc, align| acc.max(align),
378        )
379    }
380
381    /// Checks if these two `CastTarget` are equal enough to be considered "the same for all
382    /// function call ABIs".
383    pub fn eq_abi(&self, other: &Self) -> bool {
384        let CastTarget {
385            prefix: prefix_l,
386            rest_offset: rest_offset_l,
387            rest: rest_l,
388            attrs: attrs_l,
389            pass_via_x87_floating_point_stack: x87_l,
390        } = self;
391        let CastTarget {
392            prefix: prefix_r,
393            rest_offset: rest_offset_r,
394            rest: rest_r,
395            attrs: attrs_r,
396            pass_via_x87_floating_point_stack: x87_r,
397        } = other;
398        prefix_l == prefix_r
399            && rest_offset_l == rest_offset_r
400            && rest_l == rest_r
401            && attrs_l.eq_abi(attrs_r)
402            && x87_l == x87_r
403    }
404}
405
406/// Information about how to pass an argument to,
407/// or return a value from, a function, under some ABI.
408#[derive(#[automatically_derived]impl<'a, Ty: ::core::clone::Clone>
    ::core::clone::Clone for ArgAbi<'a, Ty> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        ArgAbi {
            layout: ::core::clone::Clone::clone(&self.layout),
            mode: ::core::clone::Clone::clone(&self.mode),
        }
    }
}Clone, #[automatically_derived]impl<'a, Ty: ::core::cmp::PartialEq>
    ::core::marker::StructuralPartialEq for ArgAbi<'a, Ty> {
}
#[automatically_derived]impl<'a, Ty: ::core::cmp::PartialEq>
    ::core::cmp::PartialEq for ArgAbi<'a, Ty> {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        self.layout == other.layout && self.mode == other.mode
    }
}PartialEq, #[automatically_derived]impl<'a, Ty: ::core::cmp::Eq> ::core::cmp::Eq for
    ArgAbi<'a, Ty> {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<TyAndLayout<'a, Ty>>;
        let _: ::core::cmp::AssertParamIsEq<PassMode>;
    }
}Eq, #[automatically_derived]impl<'a, Ty: ::core::hash::Hash> ::core::hash::Hash
    for ArgAbi<'a, Ty> {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&self.layout, state);
        ::core::hash::Hash::hash(&self.mode, state)
    }
}Hash, const _: () =
    {
        impl<'a, Ty> ::rustc_data_structures::stable_hash::StableHash for
            ArgAbi<'a, Ty> where
            Ty: ::rustc_data_structures::stable_hash::StableHash {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    ArgAbi { layout: ref __binding_0, mode: ref __binding_1 } =>
                        {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                    }
                }
            }
        }
    };StableHash)]
409pub struct ArgAbi<'a, Ty> {
410    pub layout: TyAndLayout<'a, Ty>,
411    pub mode: PassMode,
412}
413
414// Needs to be a custom impl because of the bounds on the `TyAndLayout` debug impl.
415impl<'a, Ty: fmt::Display> fmt::Debug for ArgAbi<'a, Ty> {
416    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
417        let ArgAbi { layout, mode } = self;
418        f.debug_struct("ArgAbi").field("layout", layout).field("mode", mode).finish()
419    }
420}
421
422impl<'a, Ty> ArgAbi<'a, Ty> {
423    /// This defines the "default ABI" for that type, that is then later adjusted in `fn_abi_adjust_for_abi`.
424    pub fn new(
425        layout: TyAndLayout<'a, Ty>,
426        scalar_attrs: impl Fn(Scalar, Size) -> ArgAttributes,
427    ) -> Self {
428        let mode = match layout.backend_repr {
429            _ if layout.is_zst() => PassMode::Ignore,
430            BackendRepr::Scalar(scalar) => PassMode::Direct(scalar_attrs(scalar, Size::ZERO)),
431            BackendRepr::ScalarPair { a, b, b_offset } => {
432                PassMode::Pair(scalar_attrs(a, Size::ZERO), scalar_attrs(b, b_offset))
433            }
434            BackendRepr::SimdVector { .. } => PassMode::Direct(ArgAttributes::new()),
435            BackendRepr::Memory { .. } => Self::indirect_pass_mode(&layout),
436            BackendRepr::SimdScalableVector { .. } => PassMode::Direct(ArgAttributes::new()),
437        };
438        ArgAbi { layout, mode }
439    }
440
441    fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode {
442        let mut attrs = ArgAttributes::new();
443
444        // For non-immediate arguments the callee gets its own copy of
445        // the value on the stack, so there are no aliases. The function
446        // can capture the address of the argument, but not the provenance.
447        attrs
448            .set(ArgAttribute::NoAlias)
449            .set(ArgAttribute::CapturesAddress)
450            .set(ArgAttribute::NonNull)
451            .set(ArgAttribute::NoUndef)
452            .set(ArgAttribute::NoFree);
453        attrs.pointee_size = layout.size;
454        attrs.pointee_align = Some(layout.align);
455
456        if layout.is_unsized() {
457            PassMode::IndirectUnsized { attrs, meta_attrs: ArgAttributes::new() }
458        } else {
459            PassMode::Indirect { attrs, address_space: None, mode: IndirectMode::Pointer }
460        }
461    }
462
463    /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
464    /// This is valid for both sized and unsized arguments.
465    #[track_caller]
466    pub fn make_indirect(&mut self) {
467        match self.mode {
468            PassMode::Direct(_) | PassMode::Pair(_, _) => {
469                self.mode = Self::indirect_pass_mode(&self.layout);
470            }
471            PassMode::Indirect { attrs: _, address_space: _, mode: IndirectMode::Pointer }
472            | PassMode::IndirectUnsized { attrs: _, meta_attrs: _ } => {
473                // already indirect
474            }
475            _ => {
    ::core::panicking::panic_fmt(format_args!("Tried to make {0:?} indirect",
            self.mode));
}panic!("Tried to make {:?} indirect", self.mode),
476        }
477    }
478
479    /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
480    /// This is valid for both sized and unsized arguments.
481    #[track_caller]
482    pub fn make_indirect_addrspace(&mut self, addrspace: AddressSpace) {
483        self.make_indirect();
484        match self.mode {
485            PassMode::Indirect { ref mut address_space, .. } => {
486                *address_space = Some(addrspace);
487            }
488            _ => ::core::panicking::unreachable_default_msg()unreachable!(),
489        }
490    }
491
492    /// Same as `make_indirect`, but for arguments that are ignored. Only needed for ABIs that pass
493    /// ZSTs indirectly.
494    #[track_caller]
495    pub fn make_indirect_from_ignore(&mut self) {
496        match self.mode {
497            PassMode::Ignore => {
498                self.mode = Self::indirect_pass_mode(&self.layout);
499            }
500            PassMode::Indirect { attrs: _, address_space: _, mode: IndirectMode::Pointer }
501            | PassMode::IndirectUnsized { attrs: _, meta_attrs: _ } => {
502                // already indirect
503            }
504            _ => {
    ::core::panicking::panic_fmt(format_args!("Tried to make {0:?} indirect (expected `PassMode::Ignore`)",
            self.mode));
}panic!("Tried to make {:?} indirect (expected `PassMode::Ignore`)", self.mode),
505        }
506    }
507
508    /// Pass this argument indirectly, by placing it at a fixed stack offset.
509    /// This corresponds to the `byval` LLVM argument attribute.
510    /// This is only valid for sized arguments.
511    ///
512    /// `byval_align` specifies the alignment of the `byval` stack slot, which does not need to
513    /// correspond to the type's alignment. This will be `Some` if the target's ABI specifies that
514    /// stack slots used for arguments passed by-value have specific alignment requirements which
515    /// differ from the alignment used in other situations.
516    ///
517    /// If `None`, the type's alignment is used.
518    ///
519    /// If the resulting alignment differs from the type's alignment,
520    /// the argument will be copied to an alloca with sufficient alignment,
521    /// either in the caller (if the type's alignment is lower than the byval alignment)
522    /// or in the callee (if the type's alignment is higher than the byval alignment),
523    /// to ensure that Rust code never sees an underaligned pointer.
524    pub fn pass_by_stack_offset(&mut self, byval_align: Option<Align>) {
525        if !!self.layout.is_unsized() {
    {
        ::core::panicking::panic_fmt(format_args!("used byval ABI for unsized layout"));
    }
};assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout");
526        self.make_indirect();
527        match self.mode {
528            PassMode::Indirect { ref mut attrs, address_space: _, ref mut mode } => {
529                *mode = IndirectMode::OnStack;
530
531                // Some platforms, like 32-bit x86, change the alignment of the type when passing
532                // `byval`. Account for that.
533                if let Some(byval_align) = byval_align {
534                    // On all targets with byval align this is currently true, so let's assert it.
535                    if true {
    if !(byval_align >= Align::from_bytes(4).unwrap()) {
        ::core::panicking::panic("assertion failed: byval_align >= Align::from_bytes(4).unwrap()")
    };
};debug_assert!(byval_align >= Align::from_bytes(4).unwrap());
536                    attrs.pointee_align = Some(byval_align);
537                }
538            }
539            _ => ::core::panicking::unreachable_default_msg()unreachable!(),
540        }
541    }
542
543    /// Pass this argument indirectly.
544    /// This corresponds to the `byref` LLVM argument attribute.
545    ///
546    /// `address_space` specifies the address space of the passed pointer.
547    pub fn pass_amdgpu_kernel_arg(&mut self, addrspace: Option<AddressSpace>) {
548        if !!self.layout.is_unsized() {
    {
        ::core::panicking::panic_fmt(format_args!("used amdgpu kernel arg ABI for unsized layout"));
    }
};assert!(!self.layout.is_unsized(), "used amdgpu kernel arg ABI for unsized layout");
549        self.make_indirect();
550        match self.mode {
551            PassMode::Indirect { attrs: _, ref mut address_space, ref mut mode } => {
552                *mode = IndirectMode::AmdgpuKernelArg;
553                *address_space = addrspace;
554            }
555            _ => ::core::panicking::unreachable_default_msg()unreachable!(),
556        }
557    }
558
559    pub fn extend_integer_width_to(&mut self, bits: u64) {
560        // Only integers have signedness
561        if let BackendRepr::Scalar(scalar) = self.layout.backend_repr
562            && let Primitive::Int(i, signed) = scalar.primitive()
563            // bools are i1 but their primitive is i8. Unlike other types,
564            // bools can be extended to their primitive size.
565            && (i.size().bits() < bits || self.layout.backend_repr.is_bool())
566            && let PassMode::Direct(ref mut attrs) = self.mode
567        {
568            if signed {
569                attrs.ext(ArgExtension::Sext)
570            } else {
571                attrs.ext(ArgExtension::Zext)
572            };
573        }
574    }
575
576    pub fn cast_to<T: Into<CastTarget>>(&mut self, target: T) {
577        self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32_count: 0 };
578    }
579
580    pub fn cast_to_with_attrs<T: Into<CastTarget>>(&mut self, target: T, attrs: ArgAttributes) {
581        self.mode =
582            PassMode::Cast { cast: Box::new(target.into().with_attrs(attrs)), pad_i32_count: 0 };
583    }
584
585    /// Cast to `target`, forwarding `NoUndef` only when the layout provably has no uninit
586    /// bytes *and* the cast exactly covers the layout (`target.size(cx) == self.layout.size`).
587    /// A wider cast (e.g. `Uniform::new` rounding a 3-byte aggregate up to an `i32`) covers
588    /// undef padding bytes that must not be marked `noundef`; a narrower cast does not occur,
589    /// since a `PassMode::Cast` target always covers the whole value.
590    pub fn cast_to_maybe_noundef<T, C>(&mut self, target: T, cx: &C)
591    where
592        T: Into<CastTarget>,
593        Ty: TyAbiInterface<'a, C> + Copy,
594        C: HasDataLayout,
595    {
596        let target = target.into();
597        let attr = if layout_is_noundef(self.layout, cx) && target.size(cx) == self.layout.size {
598            ArgAttribute::NoUndef
599        } else {
600            ArgAttribute::default()
601        };
602        self.cast_to_with_attrs(target, attr.into());
603    }
604
605    pub fn cast_to_and_pad_i32<T: Into<CastTarget>>(&mut self, target: T, pad_i32_count: u8) {
606        self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32_count };
607    }
608
609    pub fn is_indirect(&self) -> bool {
610        #[allow(non_exhaustive_omitted_patterns)] match self.mode {
    PassMode::Indirect { .. } | PassMode::IndirectUnsized { .. } => true,
    _ => false,
}matches!(self.mode, PassMode::Indirect { .. } | PassMode::IndirectUnsized { .. })
611    }
612
613    pub fn is_sized_indirect(&self) -> bool {
614        #[allow(non_exhaustive_omitted_patterns)] match self.mode {
    PassMode::Indirect { attrs: _, address_space: _, mode: _ } => true,
    _ => false,
}matches!(self.mode, PassMode::Indirect { attrs: _, address_space: _, mode: _ })
615    }
616
617    pub fn is_unsized_indirect(&self) -> bool {
618        #[allow(non_exhaustive_omitted_patterns)] match self.mode {
    PassMode::IndirectUnsized { attrs: _, meta_attrs: _ } => true,
    _ => false,
}matches!(self.mode, PassMode::IndirectUnsized { attrs: _, meta_attrs: _ })
619    }
620
621    pub fn is_ignore(&self) -> bool {
622        #[allow(non_exhaustive_omitted_patterns)] match self.mode {
    PassMode::Ignore => true,
    _ => false,
}matches!(self.mode, PassMode::Ignore)
623    }
624
625    /// Checks if these two `ArgAbi` are equal enough to be considered "the same for all
626    /// function call ABIs".
627    pub fn eq_abi(&self, other: &Self) -> bool
628    where
629        Ty: PartialEq,
630    {
631        // Ideally we'd just compare the `mode`, but that is not enough -- for some modes LLVM will
632        // look at the type.
633        self.layout.eq_abi(&other.layout) && self.mode.eq_abi(&other.mode)
634    }
635}
636
637/// Metadata describing how the arguments to a native function
638/// should be passed in order to respect the native ABI.
639///
640/// The signature represented by this type may not match the MIR function signature.
641/// Certain attributes, like `#[track_caller]` can introduce additional arguments, which are present in [`FnAbi`], but not in `FnSig`.
642/// The std::offload module also adds an addition dyn_ptr argument to the GpuKernel ABI.
643/// While this difference is rarely relevant, it should still be kept in mind.
644///
645/// I will do my best to describe this structure, but these
646/// comments are reverse-engineered and may be inaccurate. -NDM
647#[derive(#[automatically_derived]impl<'a, Ty: ::core::clone::Clone>
    ::core::clone::Clone for FnAbi<'a, Ty> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        FnAbi {
            args: ::core::clone::Clone::clone(&self.args),
            ret: ::core::clone::Clone::clone(&self.ret),
            c_variadic: ::core::clone::Clone::clone(&self.c_variadic),
            fixed_count: ::core::clone::Clone::clone(&self.fixed_count),
            conv: ::core::clone::Clone::clone(&self.conv),
            can_unwind: ::core::clone::Clone::clone(&self.can_unwind),
        }
    }
}Clone, #[automatically_derived]impl<'a, Ty: ::core::cmp::PartialEq>
    ::core::marker::StructuralPartialEq for FnAbi<'a, Ty> {
}
#[automatically_derived]impl<'a, Ty: ::core::cmp::PartialEq>
    ::core::cmp::PartialEq for FnAbi<'a, Ty> {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        self.c_variadic == other.c_variadic &&
                            self.fixed_count == other.fixed_count &&
                        self.can_unwind == other.can_unwind &&
                    self.args == other.args && self.ret == other.ret &&
            self.conv == other.conv
    }
}PartialEq, #[automatically_derived]impl<'a, Ty: ::core::cmp::Eq> ::core::cmp::Eq for
    FnAbi<'a, Ty> {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<Box<[ArgAbi<'a, Ty>]>>;
        let _: ::core::cmp::AssertParamIsEq<ArgAbi<'a, Ty>>;
        let _: ::core::cmp::AssertParamIsEq<bool>;
        let _: ::core::cmp::AssertParamIsEq<u32>;
        let _: ::core::cmp::AssertParamIsEq<CanonAbi>;
    }
}Eq, #[automatically_derived]impl<'a, Ty: ::core::hash::Hash> ::core::hash::Hash
    for FnAbi<'a, Ty> {
    #[inline]
    fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
        ::core::hash::Hash::hash(&self.args, state);
        ::core::hash::Hash::hash(&self.ret, state);
        ::core::hash::Hash::hash(&self.c_variadic, state);
        ::core::hash::Hash::hash(&self.fixed_count, state);
        ::core::hash::Hash::hash(&self.conv, state);
        ::core::hash::Hash::hash(&self.can_unwind, state)
    }
}Hash, const _: () =
    {
        impl<'a, Ty> ::rustc_data_structures::stable_hash::StableHash for
            FnAbi<'a, Ty> where
            Ty: ::rustc_data_structures::stable_hash::StableHash {
            #[inline]
            fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
                __hcx: &mut __Hcx,
                __hasher:
                    &mut ::rustc_data_structures::stable_hash::StableHasher) {
                match *self {
                    FnAbi {
                        args: ref __binding_0,
                        ret: ref __binding_1,
                        c_variadic: ref __binding_2,
                        fixed_count: ref __binding_3,
                        conv: ref __binding_4,
                        can_unwind: ref __binding_5 } => {
                        { __binding_0.stable_hash(__hcx, __hasher); }
                        { __binding_1.stable_hash(__hcx, __hasher); }
                        { __binding_2.stable_hash(__hcx, __hasher); }
                        { __binding_3.stable_hash(__hcx, __hasher); }
                        { __binding_4.stable_hash(__hcx, __hasher); }
                        { __binding_5.stable_hash(__hcx, __hasher); }
                    }
                }
            }
        }
    };StableHash)]
648pub struct FnAbi<'a, Ty> {
649    /// The type, layout, and information about how each argument is passed.
650    pub args: Box<[ArgAbi<'a, Ty>]>,
651
652    /// The layout, type, and the way a value is returned from this function.
653    pub ret: ArgAbi<'a, Ty>,
654
655    /// Marks this function as variadic (accepting a variable number of arguments).
656    pub c_variadic: bool,
657
658    /// The count of non-variadic arguments.
659    ///
660    /// Should only be different from args.len() when c_variadic is true.
661    /// This can be used to know whether an argument is variadic or not.
662    pub fixed_count: u32,
663    /// The calling convention of this function.
664    pub conv: CanonAbi,
665    /// Indicates if an unwind may happen across a call to this function.
666    pub can_unwind: bool,
667}
668
669// Needs to be a custom impl because of the bounds on the `TyAndLayout` debug impl.
670impl<'a, Ty: fmt::Display> fmt::Debug for FnAbi<'a, Ty> {
671    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
672        let FnAbi { args, ret, c_variadic, fixed_count, conv, can_unwind } = self;
673        f.debug_struct("FnAbi")
674            .field("args", args)
675            .field("ret", ret)
676            .field("c_variadic", c_variadic)
677            .field("fixed_count", fixed_count)
678            .field("conv", conv)
679            .field("can_unwind", can_unwind)
680            .finish()
681    }
682}
683
684impl<'a, Ty> FnAbi<'a, Ty> {
685    pub fn adjust_for_foreign_abi<C>(&mut self, cx: &C, abi: ExternAbi)
686    where
687        Ty: TyAbiInterface<'a, C> + Copy,
688        C: HasDataLayout + HasTargetSpec + HasX86AbiOpt,
689    {
690        if abi == ExternAbi::X86Interrupt {
691            if let Some(arg) = self.args.first_mut() {
692                arg.pass_by_stack_offset(None);
693            }
694            return;
695        }
696
697        let spec = cx.target_spec();
698        match &spec.arch {
699            Arch::X86 => {
700                let flavor = match abi {
701                    ExternAbi::Fastcall { .. } => x86::Flavor::Fastcall,
702                    ExternAbi::Vectorcall { .. } => x86::Flavor::Vectorcall,
703                    ExternAbi::C { .. } | ExternAbi::Cdecl { .. } | ExternAbi::Stdcall { .. } => {
704                        x86::Flavor::General { regparam: cx.x86_abi_opt().regparm }
705                    }
706                    _ => x86::Flavor::General { regparam: None },
707                };
708                let reg_struct_return = cx.x86_abi_opt().reg_struct_return;
709                let opts = x86::X86Options { flavor, reg_struct_return };
710                if spec.is_like_msvc {
711                    x86_win32::compute_abi_info(cx, self, opts);
712                } else {
713                    x86::compute_abi_info(cx, self, opts);
714                }
715            }
716            Arch::X86_64 => match abi {
717                ExternAbi::SysV64 { .. } => x86_64::compute_abi_info(cx, self),
718                ExternAbi::Win64 { .. } | ExternAbi::Vectorcall { .. } => {
719                    x86_win64::compute_abi_info(cx, self)
720                }
721                _ => {
722                    if cx.target_spec().is_like_windows {
723                        x86_win64::compute_abi_info(cx, self)
724                    } else {
725                        x86_64::compute_abi_info(cx, self)
726                    }
727                }
728            },
729            Arch::AArch64 | Arch::Arm64EC => {
730                let kind = if cx.target_spec().is_like_darwin {
731                    aarch64::AbiKind::DarwinPCS
732                } else if cx.target_spec().is_like_windows {
733                    aarch64::AbiKind::Win64
734                } else {
735                    aarch64::AbiKind::AAPCS
736                };
737                aarch64::compute_abi_info(cx, self, kind)
738            }
739            Arch::AmdGpu => amdgpu::compute_abi_info(cx, self),
740            Arch::Arm => arm::compute_abi_info(cx, self),
741            Arch::Avr => avr::compute_abi_info(cx, self),
742            Arch::LoongArch32 | Arch::LoongArch64 => loongarch::compute_abi_info(cx, self),
743            Arch::M68k => m68k::compute_abi_info(cx, self),
744            Arch::CSky => csky::compute_abi_info(cx, self),
745            Arch::Mips | Arch::Mips32r6 => mips::compute_abi_info(cx, self),
746            Arch::Mips64 | Arch::Mips64r6 => mips64::compute_abi_info(cx, self),
747            Arch::PowerPC => powerpc::compute_abi_info(cx, self),
748            Arch::PowerPC64 => powerpc64::compute_abi_info(cx, self),
749            Arch::S390x => s390x::compute_abi_info(cx, self),
750            Arch::Msp430 => msp430::compute_abi_info(cx, self),
751            Arch::Sparc => sparc::compute_abi_info(cx, self),
752            Arch::Sparc64 => sparc64::compute_abi_info(cx, self),
753            Arch::Nvptx64 => {
754                if abi == ExternAbi::PtxKernel || abi == ExternAbi::GpuKernel {
755                    nvptx64::compute_ptx_kernel_abi_info(cx, self)
756                } else {
757                    nvptx64::compute_abi_info(cx, self)
758                }
759            }
760            Arch::Hexagon => hexagon::compute_abi_info(cx, self),
761            Arch::Xtensa => xtensa::compute_abi_info(cx, self),
762            Arch::RiscV32 | Arch::RiscV64 => riscv::compute_abi_info(cx, self),
763            Arch::Wasm32 | Arch::Wasm64 => wasm::compute_abi_info(cx, self),
764            Arch::Bpf => bpf::compute_abi_info(cx, self),
765            arch @ (Arch::SpirV | Arch::Other(_)) => {
766                {
    ::core::panicking::panic_fmt(format_args!("no lowering implemented for {0}",
            arch));
}panic!("no lowering implemented for {arch}")
767            }
768        }
769    }
770
771    pub fn adjust_for_rust_abi<C>(&mut self, cx: &C)
772    where
773        Ty: TyAbiInterface<'a, C> + Copy,
774        C: HasDataLayout + HasTargetSpec,
775    {
776        let spec = cx.target_spec();
777        match &spec.arch {
778            Arch::X86 => x86::compute_rust_abi_info(cx, self),
779            Arch::RiscV32 | Arch::RiscV64 => riscv::compute_rust_abi_info(cx, self),
780            Arch::LoongArch32 | Arch::LoongArch64 => loongarch::compute_rust_abi_info(cx, self),
781            Arch::AArch64 => aarch64::compute_rust_abi_info(cx, self),
782            Arch::Bpf => bpf::compute_rust_abi_info(self),
783            _ => {}
784        };
785
786        // Decides whether we can pass the given SIMD argument via `PassMode::Direct`.
787        // May only return `true` if the target will always pass those arguments the same way,
788        // no matter what the user does with `-Ctarget-feature`! In other words, whatever
789        // target features are required to pass a SIMD value in registers must be listed in
790        // the `abi_required_features` for the current target and ABI.
791        let can_pass_simd_directly = |arg: &ArgAbi<'_, Ty>| match &spec.arch {
792            // On x86, if we have SSE2 (which we have by default for x86_64), we can always pass up
793            // to 128-bit-sized vectors.
794            Arch::X86 if spec.rustc_abi == Some(RustcAbi::X86Sse2) => arg.layout.size.bits() <= 128,
795            Arch::X86_64 if spec.rustc_abi != Some(RustcAbi::Softfloat) => {
796                // x86-64 non-softfloat targets all require SSE2 so we can use SSE registers.
797                arg.layout.size.bits() <= 128
798            }
799            Arch::AArch64 if spec.rustc_abi != Some(RustcAbi::Softfloat) => {
800                // aarch64 non-softfloat targets all require Neon
801                arg.layout.size.bits() <= 128
802            }
803            _ => false,
804        };
805
806        for (arg_idx, arg) in self
807            .args
808            .iter_mut()
809            .enumerate()
810            .map(|(idx, arg)| (Some(idx), arg))
811            .chain(iter::once((None, &mut self.ret)))
812        {
813            // If the logic above already picked a specific type to cast the argument to, leave that
814            // in place.
815            if #[allow(non_exhaustive_omitted_patterns)] match arg.mode {
    PassMode::Ignore | PassMode::Cast { .. } => true,
    _ => false,
}matches!(arg.mode, PassMode::Ignore | PassMode::Cast { .. }) {
816                continue;
817            }
818
819            // Always extend `bool` in the Rust ABI
820            let extend_bool = |attrs: &mut ArgAttributes, scalar: Scalar| {
821                if scalar.is_bool() {
822                    attrs.ext(ArgExtension::Zext);
823                }
824            };
825
826            if let PassMode::Direct(attrs) = &mut arg.mode
827                && let BackendRepr::Scalar(scalar) = arg.layout.backend_repr
828            {
829                extend_bool(attrs, scalar);
830            } else if let PassMode::Pair(a_attrs, b_attrs) = &mut arg.mode
831                && let BackendRepr::ScalarPair { a, b, b_offset: _ } = arg.layout.backend_repr
832            {
833                extend_bool(a_attrs, a);
834                extend_bool(b_attrs, b);
835            }
836
837            if arg_idx.is_none()
838                && arg.layout.size > Primitive::Pointer(AddressSpace::ZERO).size(cx) * 2
839                && !#[allow(non_exhaustive_omitted_patterns)] match arg.layout.backend_repr {
    BackendRepr::SimdVector { .. } | BackendRepr::SimdScalableVector { .. } =>
        true,
    _ => false,
}matches!(
840                    arg.layout.backend_repr,
841                    BackendRepr::SimdVector { .. } | BackendRepr::SimdScalableVector { .. }
842                )
843            {
844                // Return values larger than 2 registers using a return area
845                // pointer. LLVM and Cranelift disagree about how to return
846                // values that don't fit in the registers designated for return
847                // values. LLVM will force the entire return value to be passed
848                // by return area pointer, while Cranelift will look at each IR level
849                // return value independently and decide to pass it in a
850                // register or not, which would result in the return value
851                // being passed partially in registers and partially through a
852                // return area pointer. For large IR-level values such as `i128`,
853                // cranelift will even split up the value into smaller chunks.
854                //
855                // While Cranelift may need to be fixed as the LLVM behavior is
856                // generally more correct with respect to the surface language,
857                // forcing this behavior in rustc itself makes it easier for
858                // other backends to conform to the Rust ABI and for the C ABI
859                // rustc already handles this behavior anyway.
860                //
861                // In addition LLVM's decision to pass the return value in
862                // registers or using a return area pointer depends on how
863                // exactly the return type is lowered to an LLVM IR type. For
864                // example `Option<u128>` can be lowered as `{ i128, i128 }`
865                // in which case the x86_64 backend would use a return area
866                // pointer, or it could be passed as `{ i32, i128 }` in which
867                // case the x86_64 backend would pass it in registers by taking
868                // advantage of an LLVM ABI extension that allows using 3
869                // registers for the x86_64 sysv call conv rather than the
870                // officially specified 2 registers.
871                //
872                // FIXME: Technically we should look at the amount of available
873                // return registers rather than guessing that there are 2
874                // registers for return values. In practice only a couple of
875                // architectures have less than 2 return registers. None of
876                // which supported by Cranelift.
877                //
878                // NOTE: This adjustment is only necessary for the Rust ABI as
879                // for other ABI's the calling convention implementations in
880                // rustc_target already ensure any return value which doesn't
881                // fit in the available amount of return registers is passed in
882                // the right way for the current target.
883                //
884                // The adjustment is not necessary nor desired for types with a vector
885                // representation; those are handled below.
886                arg.make_indirect();
887                continue;
888            }
889
890            match arg.layout.backend_repr {
891                BackendRepr::Memory { .. } => {
892                    // Compute `Aggregate` ABI.
893
894                    let is_indirect_not_on_stack = #[allow(non_exhaustive_omitted_patterns)] match arg.mode {
    PassMode::Indirect { mode: IndirectMode::Pointer, .. } |
        PassMode::IndirectUnsized { .. } => true,
    _ => false,
}matches!(
895                        arg.mode,
896                        PassMode::Indirect { mode: IndirectMode::Pointer, .. }
897                            | PassMode::IndirectUnsized { .. }
898                    );
899                    if !is_indirect_not_on_stack {
    ::core::panicking::panic("assertion failed: is_indirect_not_on_stack")
};assert!(is_indirect_not_on_stack);
900
901                    let size = arg.layout.size;
902                    if arg.layout.is_sized()
903                        && size <= Primitive::Pointer(AddressSpace::ZERO).size(cx)
904                    {
905                        // We want to pass small aggregates as immediates, but using
906                        // an LLVM aggregate type for this leads to bad optimizations,
907                        // so we pick an appropriately sized integer type instead.
908                        arg.cast_to_maybe_noundef(Reg { kind: RegKind::Integer, size }, cx);
909                    } else if self.conv == CanonAbi::RustTail && arg_idx.is_some() {
910                        if !arg.layout.is_sized() {
    {
        ::core::panicking::panic_fmt(format_args!("extern \"tail\" arguments must be sized"));
    }
};assert!(arg.layout.is_sized(), "extern \"tail\" arguments must be sized");
911                        arg.pass_by_stack_offset(None);
912                    }
913                }
914
915                BackendRepr::SimdVector { .. } => {
916                    // This is a fun case! The gist of what this is doing is
917                    // that we want callers and callees to always agree on the
918                    // ABI of how they pass SIMD arguments. If we were to *not*
919                    // make these arguments indirect then they'd be immediates
920                    // in LLVM, which means that they'd used whatever the
921                    // appropriate ABI is for the callee and the caller. That
922                    // means, for example, if the caller doesn't have AVX
923                    // enabled but the callee does, then passing an AVX argument
924                    // across this boundary would cause corrupt data to show up.
925                    //
926                    // This problem is fixed by passing most SIMD arguments through memory between
927                    // callers and callees which should get them all to agree on ABI regardless of
928                    // target feature sets, except for those that rely on target features that we
929                    // know to be always available. Some more information about this issue can be
930                    // found in #44367 and the comment on `can_pass_simd_directly`.
931                    if spec.simd_types_indirect && !can_pass_simd_directly(arg) {
932                        arg.make_indirect();
933                    }
934                }
935
936                _ => {}
937            }
938        }
939    }
940}
941
942/// Determines whether `layout` contains no uninit bytes (no padding, no unions),
943/// using only the computed layout.
944///
945/// Conservative: returns `false` for anything it cannot prove fully initialized,
946/// including multi-variant enums and SIMD vectors.
947// FIXME: extend to multi-variant enums (per-variant padding analysis needed).
948fn layout_is_noundef<'a, Ty, C>(layout: TyAndLayout<'a, Ty>, cx: &C) -> bool
949where
950    Ty: TyAbiInterface<'a, C> + Copy,
951    C: HasDataLayout,
952{
953    match layout.backend_repr {
954        BackendRepr::Scalar(scalar) => !scalar.is_uninit_valid(),
955        BackendRepr::ScalarPair { a: s1, b: s2, b_offset: _ } => {
956            !s1.is_uninit_valid()
957                && !s2.is_uninit_valid()
958                // Ensure there is no padding.
959                && s1.size(cx) + s2.size(cx) == layout.size
960        }
961        BackendRepr::Memory { .. } => match layout.fields {
962            FieldsShape::Primitive | FieldsShape::Union(_) => false,
963            // Array elements are at stride offsets with no inter-element gaps.
964            FieldsShape::Array { stride: _, count } => {
965                count == 0 || layout_is_noundef(layout.field(cx, 0), cx)
966            }
967            FieldsShape::Arbitrary { .. } => {
968                // With `Variants::Multiple`, `layout.fields` only covers shared
969                // bytes (niche/discriminant); per-variant data is absent, so
970                // full coverage cannot be proven.
971                #[allow(non_exhaustive_omitted_patterns)] match layout.variants {
    Variants::Single { .. } => true,
    _ => false,
}matches!(layout.variants, Variants::Single { .. }) && fields_are_noundef(layout, cx)
972            }
973        },
974        BackendRepr::SimdVector { .. } | BackendRepr::SimdScalableVector { .. } => false,
975    }
976}
977
978/// Returns `true` if the fields of `layout` contiguously cover bytes `0..layout.size`
979/// with no padding gaps and each field is recursively `layout_is_noundef`.
980fn fields_are_noundef<'a, Ty, C>(layout: TyAndLayout<'a, Ty>, cx: &C) -> bool
981where
982    Ty: TyAbiInterface<'a, C> + Copy,
983    C: HasDataLayout,
984{
985    let mut cursor = Size::ZERO;
986    for i in layout.fields.index_by_increasing_offset() {
987        let field = layout.field(cx, i);
988        if field.size == Size::ZERO {
989            continue;
990        }
991        if layout.fields.offset(i) != cursor {
992            return false;
993        }
994        if !layout_is_noundef(field, cx) {
995            return false;
996        }
997        cursor += field.size;
998    }
999    cursor == layout.size
1000}
1001
1002// Some types are used a lot. Make sure they don't unintentionally get bigger.
1003#[cfg(target_pointer_width = "64")]
1004mod size_asserts {
1005    use rustc_data_structures::static_assert_size;
1006
1007    use super::*;
1008    // tidy-alphabetical-start
1009    const _: [(); 56] = [(); ::std::mem::size_of::<ArgAbi<'_, usize>>()];static_assert_size!(ArgAbi<'_, usize>, 56);
1010    const _: [(); 80] = [(); ::std::mem::size_of::<FnAbi<'_, usize>>()];static_assert_size!(FnAbi<'_, usize>, 80);
1011    // tidy-alphabetical-end
1012}