1use std::{fmt, iter};
23use arrayvec::ArrayVec;
4use rustc_abi::{
5AddressSpace, Align, BackendRepr, CanonAbi, ExternAbi, FieldsShape, HasDataLayout, Primitive,
6Reg, RegKind, Scalar, Size, TyAbiInterface, TyAndLayout, Variants,
7};
8use rustc_macros::StableHash;
910pub use crate::spec::AbiMap;
11use crate::spec::{Arch, HasTargetSpec, HasX86AbiOpt, RustcAbi};
1213mod aarch64;
14mod amdgpu;
15mod arm;
16mod avr;
17mod bpf;
18mod csky;
19mod hexagon;
20mod loongarch;
21mod m68k;
22mod mips;
23mod mips64;
24mod msp430;
25mod nvptx64;
26mod powerpc;
27mod powerpc64;
28mod riscv;
29mod s390x;
30mod sparc;
31mod sparc64;
32mod wasm;
33mod x86;
34mod x86_64;
35mod x86_win32;
36mod x86_win64;
37mod xtensa;
3839/// Different modes in which indirect arguments can be passed.
40#[derive(#[automatically_derived]impl ::core::marker::Copy for IndirectMode { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for IndirectMode {
}
#[automatically_derived]impl ::core::clone::Clone for IndirectMode {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
IndirectMode {
}
#[automatically_derived]impl ::core::cmp::PartialEq for IndirectMode {
#[inline]
fn eq(&self, other: &Self) -> bool {
::core::intrinsics::discriminant_value(self) ==
::core::intrinsics::discriminant_value(other)
}
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for IndirectMode { }Eq, #[automatically_derived]impl ::core::hash::Hash for IndirectMode {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&::core::intrinsics::discriminant_value(self),
state)
}
}Hash, #[automatically_derived]impl ::core::fmt::Debug for IndirectMode {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
::core::fmt::Formatter::write_str(f,
match self {
IndirectMode::Pointer => "Pointer",
IndirectMode::OnStack => "OnStack",
IndirectMode::AmdgpuKernelArg => "AmdgpuKernelArg",
})
}
}Debug, const _: () =
{
impl ::rustc_data_structures::stable_hash::StableHash for IndirectMode
{
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
IndirectMode::Pointer => 0u8.stable_hash(__hcx, __hasher),
IndirectMode::OnStack => 1u8.stable_hash(__hcx, __hasher),
IndirectMode::AmdgpuKernelArg =>
2u8.stable_hash(__hcx, __hasher),
}
match *self {
IndirectMode::Pointer => {}
IndirectMode::OnStack => {}
IndirectMode::AmdgpuKernelArg => {}
}
}
}
};StableHash)]
41pub enum IndirectMode {
42/// Passed as a normal pointer, nothing special.
43Pointer,
44/// The value should be passed at a fixed stack offset in accordance to
45 /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument
46 /// attribute. The `byval` argument will use a byte array with the same size as the Rust type
47 /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout),
48 /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's
49 /// alignment (if `None`). This means that the alignment will not always
50 /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info.
51OnStack,
52/// `AmdgpuKernelArg` behaves similar to `OnStack` except that the pointer does not necessarily
53 /// point to the stack, no extra copy is made, and the passed argument should not be modified.
54 /// This corresponds to the `byref` LLVM argument attribute.
55AmdgpuKernelArg,
56}
5758#[derive(#[automatically_derived]impl ::core::clone::Clone for PassMode {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
match self {
Self::Ignore => PassMode::Ignore,
Self::Direct(__self_0) =>
PassMode::Direct(::core::clone::Clone::clone(__self_0)),
Self::Pair(__self_0, __self_1) =>
PassMode::Pair(::core::clone::Clone::clone(__self_0),
::core::clone::Clone::clone(__self_1)),
Self::Cast { pad_i32_count: __self_0, cast: __self_1 } =>
PassMode::Cast {
pad_i32_count: ::core::clone::Clone::clone(__self_0),
cast: ::core::clone::Clone::clone(__self_1),
},
Self::Indirect {
attrs: __self_0, address_space: __self_1, mode: __self_2 } =>
PassMode::Indirect {
attrs: ::core::clone::Clone::clone(__self_0),
address_space: ::core::clone::Clone::clone(__self_1),
mode: ::core::clone::Clone::clone(__self_2),
},
Self::IndirectUnsized { attrs: __self_0, meta_attrs: __self_1 } =>
PassMode::IndirectUnsized {
attrs: ::core::clone::Clone::clone(__self_0),
meta_attrs: ::core::clone::Clone::clone(__self_1),
},
}
}
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for PassMode
{
}
#[automatically_derived]impl ::core::cmp::PartialEq for PassMode {
#[inline]
fn eq(&self, other: &Self) -> bool {
::core::intrinsics::discriminant_value(self) ==
::core::intrinsics::discriminant_value(other) &&
match (self, other) {
(Self::Direct(__self_0), Self::Direct(__arg1_0)) =>
__self_0 == __arg1_0,
(Self::Pair(__self_0, __self_1),
Self::Pair(__arg1_0, __arg1_1)) =>
__self_0 == __arg1_0 && __self_1 == __arg1_1,
(Self::Cast { pad_i32_count: __self_0, cast: __self_1 },
Self::Cast { pad_i32_count: __arg1_0, cast: __arg1_1 }) =>
__self_0 == __arg1_0 && __self_1 == __arg1_1,
(Self::Indirect {
attrs: __self_0, address_space: __self_1, mode: __self_2 },
Self::Indirect {
attrs: __arg1_0, address_space: __arg1_1, mode: __arg1_2 })
=>
__self_0 == __arg1_0 && __self_1 == __arg1_1 &&
__self_2 == __arg1_2,
(Self::IndirectUnsized { attrs: __self_0, meta_attrs: __self_1
}, Self::IndirectUnsized {
attrs: __arg1_0, meta_attrs: __arg1_1 }) =>
__self_0 == __arg1_0 && __self_1 == __arg1_1,
_ => true,
}
}
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for PassMode {
#[inline]
#[doc(hidden)]
#[coverage(off)]
fn assert_fields_are_eq(&self) {
let _: ::core::cmp::AssertParamIsEq<ArgAttributes>;
let _: ::core::cmp::AssertParamIsEq<u8>;
let _: ::core::cmp::AssertParamIsEq<Box<CastTarget>>;
let _: ::core::cmp::AssertParamIsEq<Option<AddressSpace>>;
let _: ::core::cmp::AssertParamIsEq<IndirectMode>;
}
}Eq, #[automatically_derived]impl ::core::hash::Hash for PassMode {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&::core::intrinsics::discriminant_value(self),
state);
match self {
Self::Direct(__self_0) =>
::core::hash::Hash::hash(__self_0, state),
Self::Pair(__self_0, __self_1) => {
::core::hash::Hash::hash(__self_0, state);
::core::hash::Hash::hash(__self_1, state)
}
Self::Cast { pad_i32_count: __self_0, cast: __self_1 } => {
::core::hash::Hash::hash(__self_0, state);
::core::hash::Hash::hash(__self_1, state)
}
Self::Indirect {
attrs: __self_0, address_space: __self_1, mode: __self_2 } =>
{
::core::hash::Hash::hash(__self_0, state);
::core::hash::Hash::hash(__self_1, state);
::core::hash::Hash::hash(__self_2, state)
}
Self::IndirectUnsized { attrs: __self_0, meta_attrs: __self_1 } =>
{
::core::hash::Hash::hash(__self_0, state);
::core::hash::Hash::hash(__self_1, state)
}
_ => {}
}
}
}Hash, #[automatically_derived]impl ::core::fmt::Debug for PassMode {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
match self {
Self::Ignore => ::core::fmt::Formatter::write_str(f, "Ignore"),
Self::Direct(__self_0) =>
::core::fmt::Formatter::debug_struct_field1_finish(f,
"Direct", &__self_0),
Self::Pair(__self_0, __self_1) =>
::core::fmt::Formatter::debug_struct_field2_finish(f, "Pair",
__self_0, &__self_1),
Self::Cast { pad_i32_count: __self_0, cast: __self_1 } =>
::core::fmt::Formatter::debug_struct_field2_finish(f,
"Cast pad_i32_count cast", __self_0, &__self_1),
Self::Indirect {
attrs: __self_0, address_space: __self_1, mode: __self_2 } =>
::core::fmt::Formatter::debug_struct_field3_finish(f,
"Indirect attrs address_space mode", __self_0, __self_1,
&__self_2),
Self::IndirectUnsized { attrs: __self_0, meta_attrs: __self_1 } =>
::core::fmt::Formatter::debug_struct_field2_finish(f,
"IndirectUnsized attrs meta_attrs", __self_0, &__self_1),
}
}
}Debug, const _: () =
{
impl ::rustc_data_structures::stable_hash::StableHash for PassMode {
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
PassMode::Ignore => 0u8.stable_hash(__hcx, __hasher),
PassMode::Direct(ref __binding_0) =>
1u8.stable_hash(__hcx, __hasher),
PassMode::Pair(ref __binding_0, ref __binding_1) =>
2u8.stable_hash(__hcx, __hasher),
PassMode::Cast {
pad_i32_count: ref __binding_0, cast: ref __binding_1 } =>
3u8.stable_hash(__hcx, __hasher),
PassMode::Indirect {
attrs: ref __binding_0,
address_space: ref __binding_1,
mode: ref __binding_2 } => 4u8.stable_hash(__hcx, __hasher),
PassMode::IndirectUnsized {
attrs: ref __binding_0, meta_attrs: ref __binding_1 } =>
5u8.stable_hash(__hcx, __hasher),
}
match *self {
PassMode::Ignore => {}
PassMode::Direct(ref __binding_0) => {
{ __binding_0.stable_hash(__hcx, __hasher); }
}
PassMode::Pair(ref __binding_0, ref __binding_1) => {
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
}
PassMode::Cast {
pad_i32_count: ref __binding_0, cast: ref __binding_1 } => {
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
}
PassMode::Indirect {
attrs: ref __binding_0,
address_space: ref __binding_1,
mode: ref __binding_2 } => {
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
{ __binding_2.stable_hash(__hcx, __hasher); }
}
PassMode::IndirectUnsized {
attrs: ref __binding_0, meta_attrs: ref __binding_1 } => {
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
}
}
}
}
};StableHash)]
59pub enum PassMode {
60/// Ignore the argument.
61 ///
62 /// The argument is a ZST.
63Ignore,
64/// Pass the argument directly.
65 ///
66 /// The argument has a layout abi of `Scalar` or `Vector`.
67 /// Unfortunately due to past mistakes, in rare cases on wasm, it can also be `Aggregate`.
68 /// This is bad since it leaks LLVM implementation details into the ABI.
69 /// (Also see <https://github.com/rust-lang/rust/issues/115666>.)
70Direct(ArgAttributes),
71/// Pass a pair's elements directly in two arguments.
72 ///
73 /// The argument has a layout abi of `ScalarPair`.
74Pair(ArgAttributes, ArgAttributes),
75/// Pass the argument after casting it. See the `CastTarget` docs for details.
76 ///
77 /// `pad_i32` indicates how many `Reg::i32()` dummy arguments are emitted before the real
78 /// argument.
79Cast { pad_i32_count: u8, cast: Box<CastTarget> },
80/// Pass the argument indirectly via a hidden pointer.
81 ///
82 /// `address_space` specifies if the pointer is in a special address space or the default one.
83 ///
84 /// `mode` can be a special way to pass an argument indirectly.
85Indirect { attrs: ArgAttributes, address_space: Option<AddressSpace>, mode: IndirectMode },
8687/// Like `Indirect`, but for unsized parameters.
88 ///
89 /// The `meta_attrs` value, is for the metadata (vtable or length) of an unsized argument.
90 /// (This is the only mode that supports unsized arguments.) `on_stack` is always false.
91IndirectUnsized { attrs: ArgAttributes, meta_attrs: ArgAttributes },
92}
9394impl PassMode {
95/// Checks if these two `PassMode` are equal enough to be considered "the same for all
96 /// function call ABIs". However, the `Layout` can also impact ABI decisions,
97 /// so that needs to be compared as well!
98pub fn eq_abi(&self, other: &Self) -> bool {
99match (self, other) {
100 (PassMode::Ignore, PassMode::Ignore) => true,
101 (PassMode::Direct(a1), PassMode::Direct(a2)) => a1.eq_abi(a2),
102 (PassMode::Pair(a1, b1), PassMode::Pair(a2, b2)) => a1.eq_abi(a2) && b1.eq_abi(b2),
103 (
104 PassMode::Cast { cast: c1, pad_i32_count: pad1 },
105 PassMode::Cast { cast: c2, pad_i32_count: pad2 },
106 ) => c1.eq_abi(c2) && pad1 == pad2,
107 (
108 PassMode::Indirect { attrs: a1, address_space: as1, mode: m1 },
109 PassMode::Indirect { attrs: a2, address_space: as2, mode: m2 },
110 ) => a1.eq_abi(a2) && as1 == as2 && m1 == m2,
111 (
112 PassMode::IndirectUnsized { attrs: a1, meta_attrs: e1 },
113 PassMode::IndirectUnsized { attrs: a2, meta_attrs: e2 },
114 ) => a1.eq_abi(a2) && e1.eq_abi(e2),
115_ => false,
116 }
117 }
118}
119120// Hack to disable non_upper_case_globals only for the bitflags! and not for the rest
121// of this module
122pub use attr_impl::ArgAttribute;
123124#[allow(non_upper_case_globals)]
125#[allow(unused)]
126mod attr_impl {
127use rustc_macros::StableHash;
128129// The subset of llvm::Attribute needed for arguments, packed into a bitfield.
130#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for ArgAttribute {
}
#[automatically_derived]impl ::core::clone::Clone for ArgAttribute {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
let _: ::core::clone::AssertParamIsClone<u16>;
*self
}
}Clone, #[automatically_derived]impl ::core::marker::Copy for ArgAttribute { }Copy, #[automatically_derived]impl ::core::default::Default for ArgAttribute {
#[inline]
fn default() -> Self { ArgAttribute(::core::default::Default::default()) }
}Default, #[automatically_derived]impl ::core::hash::Hash for ArgAttribute {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&self.0, state)
}
}Hash, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
ArgAttribute {
}
#[automatically_derived]impl ::core::cmp::PartialEq for ArgAttribute {
#[inline]
fn eq(&self, other: &Self) -> bool { self.0 == other.0 }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for ArgAttribute {
#[inline]
#[doc(hidden)]
#[coverage(off)]
fn assert_fields_are_eq(&self) {
let _: ::core::cmp::AssertParamIsEq<u16>;
}
}Eq, const _: () =
{
impl ::rustc_data_structures::stable_hash::StableHash for ArgAttribute
{
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
ArgAttribute(ref __binding_0) => {
{ __binding_0.stable_hash(__hcx, __hasher); }
}
}
}
}
};StableHash)]
131pub struct ArgAttribute(u16);
132impl ArgAttribute {
#[allow(deprecated, non_upper_case_globals,)]
pub const CapturesNone: Self = Self::from_bits_retain(0b111);
#[allow(deprecated, non_upper_case_globals,)]
pub const CapturesAddress: Self = Self::from_bits_retain(0b110);
#[allow(deprecated, non_upper_case_globals,)]
pub const CapturesReadOnly: Self = Self::from_bits_retain(0b100);
#[allow(deprecated, non_upper_case_globals,)]
pub const NoAlias: Self = Self::from_bits_retain(1 << 3);
#[allow(deprecated, non_upper_case_globals,)]
pub const NonNull: Self = Self::from_bits_retain(1 << 4);
#[allow(deprecated, non_upper_case_globals,)]
pub const ReadOnly: Self = Self::from_bits_retain(1 << 5);
#[allow(deprecated, non_upper_case_globals,)]
pub const InReg: Self = Self::from_bits_retain(1 << 6);
#[allow(deprecated, non_upper_case_globals,)]
pub const NoUndef: Self = Self::from_bits_retain(1 << 7);
#[allow(deprecated, non_upper_case_globals,)]
pub const Writable: Self = Self::from_bits_retain(1 << 8);
#[doc =
r" It is UB for this pointer or any pointer derived from it to be used for"]
#[doc =
r" deallocation (except for zero-sized deallocation) while the function is"]
#[doc =
r" executing. Only valid on arguments (including return values that are passed"]
#[doc = r" indirectly as arguments)."]
#[allow(deprecated, non_upper_case_globals,)]
pub const NoFree: Self = Self::from_bits_retain(1 << 9);
}
impl ::bitflags::Flags for ArgAttribute {
const FLAGS: &'static [::bitflags::Flag<ArgAttribute>] =
&[{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("CapturesNone",
ArgAttribute::CapturesNone)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("CapturesAddress",
ArgAttribute::CapturesAddress)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("CapturesReadOnly",
ArgAttribute::CapturesReadOnly)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("NoAlias", ArgAttribute::NoAlias)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("NonNull", ArgAttribute::NonNull)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("ReadOnly", ArgAttribute::ReadOnly)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("InReg", ArgAttribute::InReg)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("NoUndef", ArgAttribute::NoUndef)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("Writable", ArgAttribute::Writable)
},
{
#[allow(deprecated, non_upper_case_globals,)]
::bitflags::Flag::new("NoFree", ArgAttribute::NoFree)
}];
type Bits = u16;
fn bits(&self) -> u16 { ArgAttribute::bits(self) }
fn from_bits_retain(bits: u16) -> ArgAttribute {
ArgAttribute::from_bits_retain(bits)
}
}
#[allow(dead_code, deprecated, unused_doc_comments, unused_attributes,
unused_mut, unused_imports, non_upper_case_globals, clippy ::
assign_op_pattern, clippy :: iter_without_into_iter,)]
const _: () =
{
#[allow(dead_code, deprecated, unused_attributes)]
impl ArgAttribute {
/// Get a flags value with all bits unset.
#[inline]
pub const fn empty() -> Self {
Self(<u16 as ::bitflags::Bits>::EMPTY)
}
/// Get a flags value with all known bits set.
#[inline]
pub const fn all() -> Self {
let mut truncated = <u16 as ::bitflags::Bits>::EMPTY;
let mut i = 0;
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
{
{
let flag =
<ArgAttribute as
::bitflags::Flags>::FLAGS[i].value().bits();
truncated = truncated | flag;
i += 1;
}
};
let _ = i;
Self(truncated)
}
/// Get the underlying bits value.
///
/// The returned value is exactly the bits set in this flags value.
#[inline]
pub const fn bits(&self) -> u16 { self.0 }
/// Convert from a bits value.
///
/// This method will return `None` if any unknown bits are set.
#[inline]
pub const fn from_bits(bits: u16)
-> ::bitflags::__private::core::option::Option<Self> {
let truncated = Self::from_bits_truncate(bits).0;
if truncated == bits {
::bitflags::__private::core::option::Option::Some(Self(bits))
} else { ::bitflags::__private::core::option::Option::None }
}
/// Convert from a bits value, unsetting any unknown bits.
#[inline]
pub const fn from_bits_truncate(bits: u16) -> Self {
Self(bits & Self::all().0)
}
/// Convert from a bits value exactly.
#[inline]
pub const fn from_bits_retain(bits: u16) -> Self { Self(bits) }
/// Get a flags value with the bits of a flag with the given name set.
///
/// This method will return `None` if `name` is empty or doesn't
/// correspond to any named flag.
#[inline]
pub fn from_name(name: &str)
-> ::bitflags::__private::core::option::Option<Self> {
{
if name == "CapturesNone" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::CapturesNone.bits()));
}
};
;
{
if name == "CapturesAddress" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::CapturesAddress.bits()));
}
};
;
{
if name == "CapturesReadOnly" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::CapturesReadOnly.bits()));
}
};
;
{
if name == "NoAlias" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::NoAlias.bits()));
}
};
;
{
if name == "NonNull" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::NonNull.bits()));
}
};
;
{
if name == "ReadOnly" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::ReadOnly.bits()));
}
};
;
{
if name == "InReg" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::InReg.bits()));
}
};
;
{
if name == "NoUndef" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::NoUndef.bits()));
}
};
;
{
if name == "Writable" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::Writable.bits()));
}
};
;
{
if name == "NoFree" {
return ::bitflags::__private::core::option::Option::Some(Self(ArgAttribute::NoFree.bits()));
}
};
;
let _ = name;
::bitflags::__private::core::option::Option::None
}
/// Whether all bits in this flags value are unset.
#[inline]
pub const fn is_empty(&self) -> bool {
self.0 == <u16 as ::bitflags::Bits>::EMPTY
}
/// Whether all known bits in this flags value are set.
#[inline]
pub const fn is_all(&self) -> bool {
Self::all().0 | self.0 == self.0
}
/// Whether any set bits in a source flags value are also set in a target flags value.
#[inline]
pub const fn intersects(&self, other: Self) -> bool {
self.0 & other.0 != <u16 as ::bitflags::Bits>::EMPTY
}
/// Whether all set bits in a source flags value are also set in a target flags value.
#[inline]
pub const fn contains(&self, other: Self) -> bool {
self.0 & other.0 == other.0
}
/// The bitwise or (`|`) of the bits in two flags values.
#[inline]
pub fn insert(&mut self, other: Self) {
*self = Self(self.0).union(other);
}
/// The intersection of a source flags value with the complement of a target flags
/// value (`&!`).
///
/// This method is not equivalent to `self & !other` when `other` has unknown bits set.
/// `remove` won't truncate `other`, but the `!` operator will.
#[inline]
pub fn remove(&mut self, other: Self) {
*self = Self(self.0).difference(other);
}
/// The bitwise exclusive-or (`^`) of the bits in two flags values.
#[inline]
pub fn toggle(&mut self, other: Self) {
*self = Self(self.0).symmetric_difference(other);
}
/// Call `insert` when `value` is `true` or `remove` when `value` is `false`.
#[inline]
pub fn set(&mut self, other: Self, value: bool) {
if value { self.insert(other); } else { self.remove(other); }
}
/// The bitwise and (`&`) of the bits in two flags values.
#[inline]
#[must_use]
pub const fn intersection(self, other: Self) -> Self {
Self(self.0 & other.0)
}
/// The bitwise or (`|`) of the bits in two flags values.
#[inline]
#[must_use]
pub const fn union(self, other: Self) -> Self {
Self(self.0 | other.0)
}
/// The intersection of a source flags value with the complement of a target flags
/// value (`&!`).
///
/// This method is not equivalent to `self & !other` when `other` has unknown bits set.
/// `difference` won't truncate `other`, but the `!` operator will.
#[inline]
#[must_use]
pub const fn difference(self, other: Self) -> Self {
Self(self.0 & !other.0)
}
/// The bitwise exclusive-or (`^`) of the bits in two flags values.
#[inline]
#[must_use]
pub const fn symmetric_difference(self, other: Self) -> Self {
Self(self.0 ^ other.0)
}
/// The bitwise negation (`!`) of the bits in a flags value, truncating the result.
#[inline]
#[must_use]
pub const fn complement(self) -> Self {
Self::from_bits_truncate(!self.0)
}
}
impl ::bitflags::__private::core::fmt::Binary for ArgAttribute {
fn fmt(&self, f: &mut ::bitflags::__private::core::fmt::Formatter)
-> ::bitflags::__private::core::fmt::Result {
let inner = self.0;
::bitflags::__private::core::fmt::Binary::fmt(&inner, f)
}
}
impl ::bitflags::__private::core::fmt::Octal for ArgAttribute {
fn fmt(&self, f: &mut ::bitflags::__private::core::fmt::Formatter)
-> ::bitflags::__private::core::fmt::Result {
let inner = self.0;
::bitflags::__private::core::fmt::Octal::fmt(&inner, f)
}
}
impl ::bitflags::__private::core::fmt::LowerHex for ArgAttribute {
fn fmt(&self, f: &mut ::bitflags::__private::core::fmt::Formatter)
-> ::bitflags::__private::core::fmt::Result {
let inner = self.0;
::bitflags::__private::core::fmt::LowerHex::fmt(&inner, f)
}
}
impl ::bitflags::__private::core::fmt::UpperHex for ArgAttribute {
fn fmt(&self, f: &mut ::bitflags::__private::core::fmt::Formatter)
-> ::bitflags::__private::core::fmt::Result {
let inner = self.0;
::bitflags::__private::core::fmt::UpperHex::fmt(&inner, f)
}
}
impl ::bitflags::__private::core::ops::BitOr for ArgAttribute {
type Output = Self;
/// The bitwise or (`|`) of the bits in two flags values.
#[inline]
fn bitor(self, other: ArgAttribute) -> Self { self.union(other) }
}
impl ::bitflags::__private::core::ops::BitOrAssign for ArgAttribute {
/// The bitwise or (`|`) of the bits in two flags values.
#[inline]
fn bitor_assign(&mut self, other: Self) { self.insert(other); }
}
impl ::bitflags::__private::core::ops::BitXor for ArgAttribute {
type Output = Self;
/// The bitwise exclusive-or (`^`) of the bits in two flags values.
#[inline]
fn bitxor(self, other: Self) -> Self {
self.symmetric_difference(other)
}
}
impl ::bitflags::__private::core::ops::BitXorAssign for ArgAttribute {
/// The bitwise exclusive-or (`^`) of the bits in two flags values.
#[inline]
fn bitxor_assign(&mut self, other: Self) { self.toggle(other); }
}
impl ::bitflags::__private::core::ops::BitAnd for ArgAttribute {
type Output = Self;
/// The bitwise and (`&`) of the bits in two flags values.
#[inline]
fn bitand(self, other: Self) -> Self { self.intersection(other) }
}
impl ::bitflags::__private::core::ops::BitAndAssign for ArgAttribute {
/// The bitwise and (`&`) of the bits in two flags values.
#[inline]
fn bitand_assign(&mut self, other: Self) {
*self =
Self::from_bits_retain(self.bits()).intersection(other);
}
}
impl ::bitflags::__private::core::ops::Sub for ArgAttribute {
type Output = Self;
/// The intersection of a source flags value with the complement of a target flags value (`&!`).
///
/// This method is not equivalent to `self & !other` when `other` has unknown bits set.
/// `difference` won't truncate `other`, but the `!` operator will.
#[inline]
fn sub(self, other: Self) -> Self { self.difference(other) }
}
impl ::bitflags::__private::core::ops::SubAssign for ArgAttribute {
/// The intersection of a source flags value with the complement of a target flags value (`&!`).
///
/// This method is not equivalent to `self & !other` when `other` has unknown bits set.
/// `difference` won't truncate `other`, but the `!` operator will.
#[inline]
fn sub_assign(&mut self, other: Self) { self.remove(other); }
}
impl ::bitflags::__private::core::ops::Not for ArgAttribute {
type Output = Self;
/// The bitwise negation (`!`) of the bits in a flags value, truncating the result.
#[inline]
fn not(self) -> Self { self.complement() }
}
impl ::bitflags::__private::core::iter::Extend<ArgAttribute> for
ArgAttribute {
/// The bitwise or (`|`) of the bits in each flags value.
fn extend<T: ::bitflags::__private::core::iter::IntoIterator<Item
= Self>>(&mut self, iterator: T) {
for item in iterator { self.insert(item) }
}
}
impl ::bitflags::__private::core::iter::FromIterator<ArgAttribute> for
ArgAttribute {
/// The bitwise or (`|`) of the bits in each flags value.
fn from_iter<T: ::bitflags::__private::core::iter::IntoIterator<Item
= Self>>(iterator: T) -> Self {
use ::bitflags::__private::core::iter::Extend;
let mut result = Self::empty();
result.extend(iterator);
result
}
}
impl ArgAttribute {
/// Yield a set of contained flags values.
///
/// Each yielded flags value will correspond to a defined named flag. Any unknown bits
/// will be yielded together as a final flags value.
#[inline]
pub const fn iter(&self) -> ::bitflags::iter::Iter<ArgAttribute> {
::bitflags::iter::Iter::__private_const_new(<ArgAttribute as
::bitflags::Flags>::FLAGS,
ArgAttribute::from_bits_retain(self.bits()),
ArgAttribute::from_bits_retain(self.bits()))
}
/// Yield a set of contained named flags values.
///
/// This method is like [`iter`](#method.iter), except only yields bits in contained named flags.
/// Any unknown bits, or bits not corresponding to a contained flag will not be yielded.
#[inline]
pub const fn iter_names(&self)
-> ::bitflags::iter::IterNames<ArgAttribute> {
::bitflags::iter::IterNames::__private_const_new(<ArgAttribute
as ::bitflags::Flags>::FLAGS,
ArgAttribute::from_bits_retain(self.bits()),
ArgAttribute::from_bits_retain(self.bits()))
}
}
impl ::bitflags::__private::core::iter::IntoIterator for ArgAttribute
{
type Item = ArgAttribute;
type IntoIter = ::bitflags::iter::Iter<ArgAttribute>;
fn into_iter(self) -> Self::IntoIter { self.iter() }
}
};bitflags::bitflags! {
133impl ArgAttribute: u16 {
134const CapturesNone = 0b111;
135const CapturesAddress = 0b110;
136const CapturesReadOnly = 0b100;
137const NoAlias = 1 << 3;
138const NonNull = 1 << 4;
139const ReadOnly = 1 << 5;
140const InReg = 1 << 6;
141const NoUndef = 1 << 7;
142const Writable = 1 << 8;
143/// It is UB for this pointer or any pointer derived from it to be used for
144 /// deallocation (except for zero-sized deallocation) while the function is
145 /// executing. Only valid on arguments (including return values that are passed
146 /// indirectly as arguments).
147const NoFree = 1 << 9;
148 }
149 }150impl ::std::fmt::Debug for ArgAttribute {
fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result {
::bitflags::parser::to_writer(self, f)
}
}rustc_data_structures::external_bitflags_debug! { ArgAttribute }151}
152153/// Sometimes an ABI requires small integers to be extended to a full or partial register. This enum
154/// defines if this extension should be zero-extension or sign-extension when necessary. When it is
155/// not necessary to extend the argument, this enum is ignored.
156#[derive(#[automatically_derived]impl ::core::marker::Copy for ArgExtension { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for ArgExtension {
}
#[automatically_derived]impl ::core::clone::Clone for ArgExtension {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
ArgExtension {
}
#[automatically_derived]impl ::core::cmp::PartialEq for ArgExtension {
#[inline]
fn eq(&self, other: &Self) -> bool {
::core::intrinsics::discriminant_value(self) ==
::core::intrinsics::discriminant_value(other)
}
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for ArgExtension { }Eq, #[automatically_derived]impl ::core::hash::Hash for ArgExtension {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&::core::intrinsics::discriminant_value(self),
state)
}
}Hash, #[automatically_derived]impl ::core::fmt::Debug for ArgExtension {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
::core::fmt::Formatter::write_str(f,
match self {
ArgExtension::None => "None",
ArgExtension::Zext => "Zext",
ArgExtension::Sext => "Sext",
})
}
}Debug, const _: () =
{
impl ::rustc_data_structures::stable_hash::StableHash for ArgExtension
{
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
ArgExtension::None => 0u8.stable_hash(__hcx, __hasher),
ArgExtension::Zext => 1u8.stable_hash(__hcx, __hasher),
ArgExtension::Sext => 2u8.stable_hash(__hcx, __hasher),
}
match *self {
ArgExtension::None => {}
ArgExtension::Zext => {}
ArgExtension::Sext => {}
}
}
}
};StableHash)]
157pub enum ArgExtension {
158None,
159 Zext,
160 Sext,
161}
162163/// A compact representation of LLVM attributes (at least those relevant for this module)
164/// that can be manipulated without interacting with LLVM's Attribute machinery.
165#[derive(#[automatically_derived]impl ::core::marker::Copy for ArgAttributes { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for ArgAttributes {
}
#[automatically_derived]impl ::core::clone::Clone for ArgAttributes {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
let _: ::core::clone::AssertParamIsClone<ArgAttribute>;
let _: ::core::clone::AssertParamIsClone<ArgExtension>;
let _: ::core::clone::AssertParamIsClone<Size>;
let _: ::core::clone::AssertParamIsClone<Option<Align>>;
*self
}
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
ArgAttributes {
}
#[automatically_derived]impl ::core::cmp::PartialEq for ArgAttributes {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.regular == other.regular && self.arg_ext == other.arg_ext &&
self.pointee_size == other.pointee_size &&
self.pointee_align == other.pointee_align
}
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for ArgAttributes {
#[inline]
#[doc(hidden)]
#[coverage(off)]
fn assert_fields_are_eq(&self) {
let _: ::core::cmp::AssertParamIsEq<ArgAttribute>;
let _: ::core::cmp::AssertParamIsEq<ArgExtension>;
let _: ::core::cmp::AssertParamIsEq<Size>;
let _: ::core::cmp::AssertParamIsEq<Option<Align>>;
}
}Eq, #[automatically_derived]impl ::core::hash::Hash for ArgAttributes {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&self.regular, state);
::core::hash::Hash::hash(&self.arg_ext, state);
::core::hash::Hash::hash(&self.pointee_size, state);
::core::hash::Hash::hash(&self.pointee_align, state)
}
}Hash, #[automatically_derived]impl ::core::fmt::Debug for ArgAttributes {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
::core::fmt::Formatter::debug_struct_field4_finish(f,
"ArgAttributes regular arg_ext pointee_size pointee_align",
&self.regular, &self.arg_ext, &self.pointee_size,
&&self.pointee_align)
}
}Debug, const _: () =
{
impl ::rustc_data_structures::stable_hash::StableHash for
ArgAttributes {
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
ArgAttributes {
regular: ref __binding_0,
arg_ext: ref __binding_1,
pointee_size: ref __binding_2,
pointee_align: ref __binding_3 } => {
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
{ __binding_2.stable_hash(__hcx, __hasher); }
{ __binding_3.stable_hash(__hcx, __hasher); }
}
}
}
}
};StableHash)]
166pub struct ArgAttributes {
167pub regular: ArgAttribute,
168pub arg_ext: ArgExtension,
169/// If the pointer is not null, the minimum dereferenceable size of the pointee, at the time of
170 /// function entry (for arguments) or function return (for return values).
171pub pointee_size: Size,
172/// The minimum alignment of the pointee, if any.
173pub pointee_align: Option<Align>,
174}
175176impl ArgAttributes {
177pub fn new() -> Self {
178ArgAttributes {
179 regular: ArgAttribute::default(),
180 arg_ext: ArgExtension::None,
181 pointee_size: Size::ZERO,
182 pointee_align: None,
183 }
184 }
185186pub fn ext(&mut self, ext: ArgExtension) -> &mut Self {
187if !(self.arg_ext == ArgExtension::None || self.arg_ext == ext) {
{
::core::panicking::panic_fmt(format_args!("cannot set {0:?} when {1:?} is already set",
ext, self.arg_ext));
}
};assert!(
188self.arg_ext == ArgExtension::None || self.arg_ext == ext,
189"cannot set {:?} when {:?} is already set",
190 ext,
191self.arg_ext
192 );
193self.arg_ext = ext;
194self195 }
196197pub fn set(&mut self, attr: ArgAttribute) -> &mut Self {
198self.regular |= attr;
199self200 }
201202pub fn contains(&self, attr: ArgAttribute) -> bool {
203self.regular.contains(attr)
204 }
205206/// Checks if these two `ArgAttributes` are equal enough to be considered "the same for all
207 /// function call ABIs".
208pub fn eq_abi(&self, other: &Self) -> bool {
209// There's only one regular attribute that matters for the call ABI: InReg.
210 // Everything else is things like noalias, dereferenceable, nonnull, ...
211 // (This also applies to pointee_size, pointee_align.)
212if self.regular.contains(ArgAttribute::InReg) != other.regular.contains(ArgAttribute::InReg)
213 {
214return false;
215 }
216// We also compare the sign extension mode -- this could let the callee make assumptions
217 // about bits that conceptually were not even passed.
218if self.arg_ext != other.arg_ext {
219return false;
220 }
221true
222}
223}
224225impl From<ArgAttribute> for ArgAttributes {
226fn from(value: ArgAttribute) -> Self {
227Self {
228 regular: value,
229 arg_ext: ArgExtension::None,
230 pointee_size: Size::ZERO,
231 pointee_align: None,
232 }
233 }
234}
235236/// An argument passed entirely registers with the
237/// same kind (e.g., HFA / HVA on PPC64 and AArch64).
238#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for Uniform {
}
#[automatically_derived]impl ::core::clone::Clone for Uniform {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
let _: ::core::clone::AssertParamIsClone<Reg>;
let _: ::core::clone::AssertParamIsClone<Size>;
let _: ::core::clone::AssertParamIsClone<bool>;
*self
}
}Clone, #[automatically_derived]impl ::core::marker::Copy for Uniform { }Copy, #[automatically_derived]impl ::core::marker::StructuralPartialEq for Uniform {
}
#[automatically_derived]impl ::core::cmp::PartialEq for Uniform {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.is_consecutive == other.is_consecutive && self.unit == other.unit
&& self.total == other.total
}
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for Uniform {
#[inline]
#[doc(hidden)]
#[coverage(off)]
fn assert_fields_are_eq(&self) {
let _: ::core::cmp::AssertParamIsEq<Reg>;
let _: ::core::cmp::AssertParamIsEq<Size>;
let _: ::core::cmp::AssertParamIsEq<bool>;
}
}Eq, #[automatically_derived]impl ::core::hash::Hash for Uniform {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&self.unit, state);
::core::hash::Hash::hash(&self.total, state);
::core::hash::Hash::hash(&self.is_consecutive, state)
}
}Hash, #[automatically_derived]impl ::core::fmt::Debug for Uniform {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
::core::fmt::Formatter::debug_struct_field3_finish(f,
"Uniform unit total is_consecutive", &self.unit, &self.total,
&&self.is_consecutive)
}
}Debug, const _: () =
{
impl ::rustc_data_structures::stable_hash::StableHash for Uniform {
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
Uniform {
unit: ref __binding_0,
total: ref __binding_1,
is_consecutive: ref __binding_2 } => {
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
{ __binding_2.stable_hash(__hcx, __hasher); }
}
}
}
}
};StableHash)]
239pub struct Uniform {
240pub unit: Reg,
241242/// The total size of the argument, which can be:
243 /// * equal to `unit.size` (one scalar/vector),
244 /// * a multiple of `unit.size` (an array of scalar/vectors),
245 /// * if `unit.kind` is `Integer`, the last element can be shorter, i.e., `{ i64, i64, i32 }`
246 /// for 64-bit integers with a total size of 20 bytes. When the argument is actually passed,
247 /// this size will be rounded up to the nearest multiple of `unit.size`.
248pub total: Size,
249250/// Indicate that the argument is consecutive, in the sense that either all values need to be
251 /// passed in register, or all on the stack. If they are passed on the stack, there should be
252 /// no additional padding between elements.
253pub is_consecutive: bool,
254}
255256impl From<Reg> for Uniform {
257fn from(unit: Reg) -> Uniform {
258Uniform { unit, total: unit.size, is_consecutive: false }
259 }
260}
261262impl Uniform {
263/// Alignment used for the scratch space when this type is part of a `PassMode::Cast`.
264pub fn scratch_align<C: HasDataLayout>(&self, cx: &C) -> Align {
265self.unit.scratch_align(cx)
266 }
267268/// Pass using one or more values of the given type, without requiring them to be consecutive.
269 /// That is, some values may be passed in register and some on the stack.
270pub fn new(unit: Reg, total: Size) -> Self {
271Uniform { unit, total, is_consecutive: false }
272 }
273274/// Pass using one or more consecutive values of the given type. Either all values will be
275 /// passed in registers, or all on the stack.
276pub fn consecutive(unit: Reg, total: Size) -> Self {
277Uniform { unit, total, is_consecutive: true }
278 }
279}
280281/// Describes the type used for `PassMode::Cast`.
282///
283/// Passing arguments in this mode works as follows: the registers in the `prefix` (the ones that
284/// are `Some`) get laid out one after the other (using `repr(C)` layout rules). Then the
285/// `rest.unit` register type gets repeated often enough to cover `rest.size`. This describes the
286/// actual type used for the call; the Rust type of the argument is then transmuted to this ABI type
287/// (and all data in the padding between the registers is dropped).
288#[derive(#[automatically_derived]impl ::core::clone::Clone for CastTarget {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
CastTarget {
prefix: ::core::clone::Clone::clone(&self.prefix),
rest_offset: ::core::clone::Clone::clone(&self.rest_offset),
rest: ::core::clone::Clone::clone(&self.rest),
attrs: ::core::clone::Clone::clone(&self.attrs),
pass_via_x87_floating_point_stack: ::core::clone::Clone::clone(&self.pass_via_x87_floating_point_stack),
}
}
}Clone, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
CastTarget {
}
#[automatically_derived]impl ::core::cmp::PartialEq for CastTarget {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.pass_via_x87_floating_point_stack ==
other.pass_via_x87_floating_point_stack &&
self.prefix == other.prefix &&
self.rest_offset == other.rest_offset &&
self.rest == other.rest && self.attrs == other.attrs
}
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for CastTarget {
#[inline]
#[doc(hidden)]
#[coverage(off)]
fn assert_fields_are_eq(&self) {
let _: ::core::cmp::AssertParamIsEq<ArrayVec<Reg, 8>>;
let _: ::core::cmp::AssertParamIsEq<Option<Size>>;
let _: ::core::cmp::AssertParamIsEq<Uniform>;
let _: ::core::cmp::AssertParamIsEq<ArgAttributes>;
let _: ::core::cmp::AssertParamIsEq<bool>;
}
}Eq, #[automatically_derived]impl ::core::hash::Hash for CastTarget {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&self.prefix, state);
::core::hash::Hash::hash(&self.rest_offset, state);
::core::hash::Hash::hash(&self.rest, state);
::core::hash::Hash::hash(&self.attrs, state);
::core::hash::Hash::hash(&self.pass_via_x87_floating_point_stack,
state)
}
}Hash, #[automatically_derived]impl ::core::fmt::Debug for CastTarget {
#[inline]
#[rustc_trivial_field_reads]
fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
::core::fmt::Formatter::debug_struct_field5_finish(f,
"CastTarget prefix rest_offset rest attrs pass_via_x87_floating_point_stack",
&self.prefix, &self.rest_offset, &self.rest, &self.attrs,
&&self.pass_via_x87_floating_point_stack)
}
}Debug, const _: () =
{
impl ::rustc_data_structures::stable_hash::StableHash for CastTarget {
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
CastTarget {
prefix: ref __binding_0,
rest_offset: ref __binding_1,
rest: ref __binding_2,
attrs: ref __binding_3,
pass_via_x87_floating_point_stack: ref __binding_4 } => {
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
{ __binding_2.stable_hash(__hcx, __hasher); }
{ __binding_3.stable_hash(__hcx, __hasher); }
{ __binding_4.stable_hash(__hcx, __hasher); }
}
}
}
}
};StableHash)]
289pub struct CastTarget {
290// Note that this is fixed to 8 elements for now as ABIs currently don't
291 // need anything further beyond that, and when this code was originally
292 // refactored to use `ArrayVec` it was already using 8, so that stuck
293 // around.
294pub prefix: ArrayVec<Reg, 8>,
295/// The offset of `rest` from the start of the value. Currently only implemented for a `Reg`
296 /// pair created by the `offset_pair` method.
297pub rest_offset: Option<Size>,
298pub rest: Uniform,
299pub attrs: ArgAttributes,
300/// If `true`, indicates that this type will be passed on the x87 floating point stack on
301 /// 32-bit x86, changing the backend type to `x87_f80`. This is needed as LLVM needs to generate
302 /// extra code to ensure that signalling NaNs are passed losslessly on the x87 floating point
303 /// stack. Only valid on return types on 32-bit x86 with a cast target of either `Reg::f32()` or
304 /// `Reg::f64()`.
305pub pass_via_x87_floating_point_stack: bool,
306}
307308impl From<Reg> for CastTarget {
309fn from(unit: Reg) -> CastTarget {
310CastTarget::from(Uniform::from(unit))
311 }
312}
313314impl From<Uniform> for CastTarget {
315fn from(uniform: Uniform) -> CastTarget {
316Self::prefixed(Default::default(), uniform)
317 }
318}
319320impl CastTarget {
321pub fn prefixed(prefix: ArrayVec<Reg, 8>, rest: Uniform) -> Self {
322Self {
323prefix,
324 rest_offset: None,
325rest,
326 attrs: ArgAttributes::new(),
327 pass_via_x87_floating_point_stack: false,
328 }
329 }
330331pub fn offset_pair(a: Reg, offset_from_start: Size, b: Reg) -> Self {
332let mut prefix = ArrayVec::new();
333prefix.push(a);
334Self {
335prefix,
336 rest_offset: Some(offset_from_start),
337 rest: b.into(),
338 attrs: ArgAttributes::new(),
339 pass_via_x87_floating_point_stack: false,
340 }
341 }
342343pub fn with_attrs(mut self, attrs: ArgAttributes) -> Self {
344self.attrs = attrs;
345self346 }
347348pub fn pair(a: Reg, b: Reg) -> CastTarget {
349let mut prefix = ArrayVec::new();
350prefix.push(a);
351Self::prefixed(prefix, Uniform::from(b))
352 }
353354/// When you only access the range containing valid data, you can use this unaligned size;
355 /// otherwise, use the safer `size` method.
356pub fn unaligned_size<C: HasDataLayout>(&self, _cx: &C) -> Size {
357// Prefix arguments are passed in specific designated registers
358let prefix_size = if let Some(offset_from_start) = self.rest_offset {
359offset_from_start360 } else {
361self.prefix.iter().map(|reg| reg.size).fold(Size::ZERO, |acc, size| acc + size)
362 };
363// Remaining arguments are passed in chunks of the unit size
364let rest_size =
365self.rest.unit.size * self.rest.total.bytes().div_ceil(self.rest.unit.size.bytes());
366367prefix_size + rest_size368 }
369370pub fn size<C: HasDataLayout>(&self, cx: &C) -> Size {
371self.unaligned_size(cx).align_to(self.align(cx))
372 }
373374pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align {
375self.prefix.iter().map(|reg| reg.scratch_align(cx)).fold(
376cx.data_layout().aggregate_align.max(self.rest.scratch_align(cx)),
377 |acc, align| acc.max(align),
378 )
379 }
380381/// Checks if these two `CastTarget` are equal enough to be considered "the same for all
382 /// function call ABIs".
383pub fn eq_abi(&self, other: &Self) -> bool {
384let CastTarget {
385 prefix: prefix_l,
386 rest_offset: rest_offset_l,
387 rest: rest_l,
388 attrs: attrs_l,
389 pass_via_x87_floating_point_stack: x87_l,
390 } = self;
391let CastTarget {
392 prefix: prefix_r,
393 rest_offset: rest_offset_r,
394 rest: rest_r,
395 attrs: attrs_r,
396 pass_via_x87_floating_point_stack: x87_r,
397 } = other;
398prefix_l == prefix_r399 && rest_offset_l == rest_offset_r400 && rest_l == rest_r401 && attrs_l.eq_abi(attrs_r)
402 && x87_l == x87_r403 }
404}
405406/// Information about how to pass an argument to,
407/// or return a value from, a function, under some ABI.
408#[derive(#[automatically_derived]impl<'a, Ty: ::core::clone::Clone>
::core::clone::Clone for ArgAbi<'a, Ty> {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
ArgAbi {
layout: ::core::clone::Clone::clone(&self.layout),
mode: ::core::clone::Clone::clone(&self.mode),
}
}
}Clone, #[automatically_derived]impl<'a, Ty: ::core::cmp::PartialEq>
::core::marker::StructuralPartialEq for ArgAbi<'a, Ty> {
}
#[automatically_derived]impl<'a, Ty: ::core::cmp::PartialEq>
::core::cmp::PartialEq for ArgAbi<'a, Ty> {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.layout == other.layout && self.mode == other.mode
}
}PartialEq, #[automatically_derived]impl<'a, Ty: ::core::cmp::Eq> ::core::cmp::Eq for
ArgAbi<'a, Ty> {
#[inline]
#[doc(hidden)]
#[coverage(off)]
fn assert_fields_are_eq(&self) {
let _: ::core::cmp::AssertParamIsEq<TyAndLayout<'a, Ty>>;
let _: ::core::cmp::AssertParamIsEq<PassMode>;
}
}Eq, #[automatically_derived]impl<'a, Ty: ::core::hash::Hash> ::core::hash::Hash
for ArgAbi<'a, Ty> {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&self.layout, state);
::core::hash::Hash::hash(&self.mode, state)
}
}Hash, const _: () =
{
impl<'a, Ty> ::rustc_data_structures::stable_hash::StableHash for
ArgAbi<'a, Ty> where
Ty: ::rustc_data_structures::stable_hash::StableHash {
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
ArgAbi { layout: ref __binding_0, mode: ref __binding_1 } =>
{
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
}
}
}
}
};StableHash)]
409pub struct ArgAbi<'a, Ty> {
410pub layout: TyAndLayout<'a, Ty>,
411pub mode: PassMode,
412}
413414// Needs to be a custom impl because of the bounds on the `TyAndLayout` debug impl.
415impl<'a, Ty: fmt::Display> fmt::Debugfor ArgAbi<'a, Ty> {
416fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
417let ArgAbi { layout, mode } = self;
418f.debug_struct("ArgAbi").field("layout", layout).field("mode", mode).finish()
419 }
420}
421422impl<'a, Ty> ArgAbi<'a, Ty> {
423/// This defines the "default ABI" for that type, that is then later adjusted in `fn_abi_adjust_for_abi`.
424pub fn new(
425 layout: TyAndLayout<'a, Ty>,
426 scalar_attrs: impl Fn(Scalar, Size) -> ArgAttributes,
427 ) -> Self {
428let mode = match layout.backend_repr {
429_ if layout.is_zst() => PassMode::Ignore,
430 BackendRepr::Scalar(scalar) => PassMode::Direct(scalar_attrs(scalar, Size::ZERO)),
431 BackendRepr::ScalarPair { a, b, b_offset } => {
432 PassMode::Pair(scalar_attrs(a, Size::ZERO), scalar_attrs(b, b_offset))
433 }
434 BackendRepr::SimdVector { .. } => PassMode::Direct(ArgAttributes::new()),
435 BackendRepr::Memory { .. } => Self::indirect_pass_mode(&layout),
436 BackendRepr::SimdScalableVector { .. } => PassMode::Direct(ArgAttributes::new()),
437 };
438ArgAbi { layout, mode }
439 }
440441fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode {
442let mut attrs = ArgAttributes::new();
443444// For non-immediate arguments the callee gets its own copy of
445 // the value on the stack, so there are no aliases. The function
446 // can capture the address of the argument, but not the provenance.
447attrs448 .set(ArgAttribute::NoAlias)
449 .set(ArgAttribute::CapturesAddress)
450 .set(ArgAttribute::NonNull)
451 .set(ArgAttribute::NoUndef)
452 .set(ArgAttribute::NoFree);
453attrs.pointee_size = layout.size;
454attrs.pointee_align = Some(layout.align);
455456if layout.is_unsized() {
457 PassMode::IndirectUnsized { attrs, meta_attrs: ArgAttributes::new() }
458 } else {
459 PassMode::Indirect { attrs, address_space: None, mode: IndirectMode::Pointer }
460 }
461 }
462463/// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
464 /// This is valid for both sized and unsized arguments.
465#[track_caller]
466pub fn make_indirect(&mut self) {
467match self.mode {
468 PassMode::Direct(_) | PassMode::Pair(_, _) => {
469self.mode = Self::indirect_pass_mode(&self.layout);
470 }
471 PassMode::Indirect { attrs: _, address_space: _, mode: IndirectMode::Pointer }
472 | PassMode::IndirectUnsized { attrs: _, meta_attrs: _ } => {
473// already indirect
474}
475_ => {
::core::panicking::panic_fmt(format_args!("Tried to make {0:?} indirect",
self.mode));
}panic!("Tried to make {:?} indirect", self.mode),
476 }
477 }
478479/// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
480 /// This is valid for both sized and unsized arguments.
481#[track_caller]
482pub fn make_indirect_addrspace(&mut self, addrspace: AddressSpace) {
483self.make_indirect();
484match self.mode {
485 PassMode::Indirect { ref mut address_space, .. } => {
486*address_space = Some(addrspace);
487 }
488_ => ::core::panicking::unreachable_default_msg()unreachable!(),
489 }
490 }
491492/// Same as `make_indirect`, but for arguments that are ignored. Only needed for ABIs that pass
493 /// ZSTs indirectly.
494#[track_caller]
495pub fn make_indirect_from_ignore(&mut self) {
496match self.mode {
497 PassMode::Ignore => {
498self.mode = Self::indirect_pass_mode(&self.layout);
499 }
500 PassMode::Indirect { attrs: _, address_space: _, mode: IndirectMode::Pointer }
501 | PassMode::IndirectUnsized { attrs: _, meta_attrs: _ } => {
502// already indirect
503}
504_ => {
::core::panicking::panic_fmt(format_args!("Tried to make {0:?} indirect (expected `PassMode::Ignore`)",
self.mode));
}panic!("Tried to make {:?} indirect (expected `PassMode::Ignore`)", self.mode),
505 }
506 }
507508/// Pass this argument indirectly, by placing it at a fixed stack offset.
509 /// This corresponds to the `byval` LLVM argument attribute.
510 /// This is only valid for sized arguments.
511 ///
512 /// `byval_align` specifies the alignment of the `byval` stack slot, which does not need to
513 /// correspond to the type's alignment. This will be `Some` if the target's ABI specifies that
514 /// stack slots used for arguments passed by-value have specific alignment requirements which
515 /// differ from the alignment used in other situations.
516 ///
517 /// If `None`, the type's alignment is used.
518 ///
519 /// If the resulting alignment differs from the type's alignment,
520 /// the argument will be copied to an alloca with sufficient alignment,
521 /// either in the caller (if the type's alignment is lower than the byval alignment)
522 /// or in the callee (if the type's alignment is higher than the byval alignment),
523 /// to ensure that Rust code never sees an underaligned pointer.
524pub fn pass_by_stack_offset(&mut self, byval_align: Option<Align>) {
525if !!self.layout.is_unsized() {
{
::core::panicking::panic_fmt(format_args!("used byval ABI for unsized layout"));
}
};assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout");
526self.make_indirect();
527match self.mode {
528 PassMode::Indirect { ref mut attrs, address_space: _, ref mut mode } => {
529*mode = IndirectMode::OnStack;
530531// Some platforms, like 32-bit x86, change the alignment of the type when passing
532 // `byval`. Account for that.
533if let Some(byval_align) = byval_align {
534// On all targets with byval align this is currently true, so let's assert it.
535if true {
if !(byval_align >= Align::from_bytes(4).unwrap()) {
::core::panicking::panic("assertion failed: byval_align >= Align::from_bytes(4).unwrap()")
};
};debug_assert!(byval_align >= Align::from_bytes(4).unwrap());
536attrs.pointee_align = Some(byval_align);
537 }
538 }
539_ => ::core::panicking::unreachable_default_msg()unreachable!(),
540 }
541 }
542543/// Pass this argument indirectly.
544 /// This corresponds to the `byref` LLVM argument attribute.
545 ///
546 /// `address_space` specifies the address space of the passed pointer.
547pub fn pass_amdgpu_kernel_arg(&mut self, addrspace: Option<AddressSpace>) {
548if !!self.layout.is_unsized() {
{
::core::panicking::panic_fmt(format_args!("used amdgpu kernel arg ABI for unsized layout"));
}
};assert!(!self.layout.is_unsized(), "used amdgpu kernel arg ABI for unsized layout");
549self.make_indirect();
550match self.mode {
551 PassMode::Indirect { attrs: _, ref mut address_space, ref mut mode } => {
552*mode = IndirectMode::AmdgpuKernelArg;
553*address_space = addrspace;
554 }
555_ => ::core::panicking::unreachable_default_msg()unreachable!(),
556 }
557 }
558559pub fn extend_integer_width_to(&mut self, bits: u64) {
560// Only integers have signedness
561if let BackendRepr::Scalar(scalar) = self.layout.backend_repr
562 && let Primitive::Int(i, signed) = scalar.primitive()
563// bools are i1 but their primitive is i8. Unlike other types,
564 // bools can be extended to their primitive size.
565&& (i.size().bits() < bits || self.layout.backend_repr.is_bool())
566 && let PassMode::Direct(ref mut attrs) = self.mode
567 {
568if signed {
569attrs.ext(ArgExtension::Sext)
570 } else {
571attrs.ext(ArgExtension::Zext)
572 };
573 }
574 }
575576pub fn cast_to<T: Into<CastTarget>>(&mut self, target: T) {
577self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32_count: 0 };
578 }
579580pub fn cast_to_with_attrs<T: Into<CastTarget>>(&mut self, target: T, attrs: ArgAttributes) {
581self.mode =
582 PassMode::Cast { cast: Box::new(target.into().with_attrs(attrs)), pad_i32_count: 0 };
583 }
584585/// Cast to `target`, forwarding `NoUndef` only when the layout provably has no uninit
586 /// bytes *and* the cast exactly covers the layout (`target.size(cx) == self.layout.size`).
587 /// A wider cast (e.g. `Uniform::new` rounding a 3-byte aggregate up to an `i32`) covers
588 /// undef padding bytes that must not be marked `noundef`; a narrower cast does not occur,
589 /// since a `PassMode::Cast` target always covers the whole value.
590pub fn cast_to_maybe_noundef<T, C>(&mut self, target: T, cx: &C)
591where
592T: Into<CastTarget>,
593 Ty: TyAbiInterface<'a, C> + Copy,
594 C: HasDataLayout,
595 {
596let target = target.into();
597let attr = if layout_is_noundef(self.layout, cx) && target.size(cx) == self.layout.size {
598ArgAttribute::NoUndef599 } else {
600ArgAttribute::default()
601 };
602self.cast_to_with_attrs(target, attr.into());
603 }
604605pub fn cast_to_and_pad_i32<T: Into<CastTarget>>(&mut self, target: T, pad_i32_count: u8) {
606self.mode = PassMode::Cast { cast: Box::new(target.into()), pad_i32_count };
607 }
608609pub fn is_indirect(&self) -> bool {
610#[allow(non_exhaustive_omitted_patterns)] match self.mode {
PassMode::Indirect { .. } | PassMode::IndirectUnsized { .. } => true,
_ => false,
}matches!(self.mode, PassMode::Indirect { .. } | PassMode::IndirectUnsized { .. })611 }
612613pub fn is_sized_indirect(&self) -> bool {
614#[allow(non_exhaustive_omitted_patterns)] match self.mode {
PassMode::Indirect { attrs: _, address_space: _, mode: _ } => true,
_ => false,
}matches!(self.mode, PassMode::Indirect { attrs: _, address_space: _, mode: _ })615 }
616617pub fn is_unsized_indirect(&self) -> bool {
618#[allow(non_exhaustive_omitted_patterns)] match self.mode {
PassMode::IndirectUnsized { attrs: _, meta_attrs: _ } => true,
_ => false,
}matches!(self.mode, PassMode::IndirectUnsized { attrs: _, meta_attrs: _ })619 }
620621pub fn is_ignore(&self) -> bool {
622#[allow(non_exhaustive_omitted_patterns)] match self.mode {
PassMode::Ignore => true,
_ => false,
}matches!(self.mode, PassMode::Ignore)623 }
624625/// Checks if these two `ArgAbi` are equal enough to be considered "the same for all
626 /// function call ABIs".
627pub fn eq_abi(&self, other: &Self) -> bool628where
629Ty: PartialEq,
630 {
631// Ideally we'd just compare the `mode`, but that is not enough -- for some modes LLVM will
632 // look at the type.
633self.layout.eq_abi(&other.layout) && self.mode.eq_abi(&other.mode)
634 }
635}
636637/// Metadata describing how the arguments to a native function
638/// should be passed in order to respect the native ABI.
639///
640/// The signature represented by this type may not match the MIR function signature.
641/// Certain attributes, like `#[track_caller]` can introduce additional arguments, which are present in [`FnAbi`], but not in `FnSig`.
642/// The std::offload module also adds an addition dyn_ptr argument to the GpuKernel ABI.
643/// While this difference is rarely relevant, it should still be kept in mind.
644///
645/// I will do my best to describe this structure, but these
646/// comments are reverse-engineered and may be inaccurate. -NDM
647#[derive(#[automatically_derived]impl<'a, Ty: ::core::clone::Clone>
::core::clone::Clone for FnAbi<'a, Ty> {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
FnAbi {
args: ::core::clone::Clone::clone(&self.args),
ret: ::core::clone::Clone::clone(&self.ret),
c_variadic: ::core::clone::Clone::clone(&self.c_variadic),
fixed_count: ::core::clone::Clone::clone(&self.fixed_count),
conv: ::core::clone::Clone::clone(&self.conv),
can_unwind: ::core::clone::Clone::clone(&self.can_unwind),
}
}
}Clone, #[automatically_derived]impl<'a, Ty: ::core::cmp::PartialEq>
::core::marker::StructuralPartialEq for FnAbi<'a, Ty> {
}
#[automatically_derived]impl<'a, Ty: ::core::cmp::PartialEq>
::core::cmp::PartialEq for FnAbi<'a, Ty> {
#[inline]
fn eq(&self, other: &Self) -> bool {
self.c_variadic == other.c_variadic &&
self.fixed_count == other.fixed_count &&
self.can_unwind == other.can_unwind &&
self.args == other.args && self.ret == other.ret &&
self.conv == other.conv
}
}PartialEq, #[automatically_derived]impl<'a, Ty: ::core::cmp::Eq> ::core::cmp::Eq for
FnAbi<'a, Ty> {
#[inline]
#[doc(hidden)]
#[coverage(off)]
fn assert_fields_are_eq(&self) {
let _: ::core::cmp::AssertParamIsEq<Box<[ArgAbi<'a, Ty>]>>;
let _: ::core::cmp::AssertParamIsEq<ArgAbi<'a, Ty>>;
let _: ::core::cmp::AssertParamIsEq<bool>;
let _: ::core::cmp::AssertParamIsEq<u32>;
let _: ::core::cmp::AssertParamIsEq<CanonAbi>;
}
}Eq, #[automatically_derived]impl<'a, Ty: ::core::hash::Hash> ::core::hash::Hash
for FnAbi<'a, Ty> {
#[inline]
fn hash<__H: ::core::hash::Hasher>(&self, state: &mut __H) {
::core::hash::Hash::hash(&self.args, state);
::core::hash::Hash::hash(&self.ret, state);
::core::hash::Hash::hash(&self.c_variadic, state);
::core::hash::Hash::hash(&self.fixed_count, state);
::core::hash::Hash::hash(&self.conv, state);
::core::hash::Hash::hash(&self.can_unwind, state)
}
}Hash, const _: () =
{
impl<'a, Ty> ::rustc_data_structures::stable_hash::StableHash for
FnAbi<'a, Ty> where
Ty: ::rustc_data_structures::stable_hash::StableHash {
#[inline]
fn stable_hash<__Hcx: ::rustc_data_structures::stable_hash::StableHashCtxt>(&self,
__hcx: &mut __Hcx,
__hasher:
&mut ::rustc_data_structures::stable_hash::StableHasher) {
match *self {
FnAbi {
args: ref __binding_0,
ret: ref __binding_1,
c_variadic: ref __binding_2,
fixed_count: ref __binding_3,
conv: ref __binding_4,
can_unwind: ref __binding_5 } => {
{ __binding_0.stable_hash(__hcx, __hasher); }
{ __binding_1.stable_hash(__hcx, __hasher); }
{ __binding_2.stable_hash(__hcx, __hasher); }
{ __binding_3.stable_hash(__hcx, __hasher); }
{ __binding_4.stable_hash(__hcx, __hasher); }
{ __binding_5.stable_hash(__hcx, __hasher); }
}
}
}
}
};StableHash)]
648pub struct FnAbi<'a, Ty> {
649/// The type, layout, and information about how each argument is passed.
650pub args: Box<[ArgAbi<'a, Ty>]>,
651652/// The layout, type, and the way a value is returned from this function.
653pub ret: ArgAbi<'a, Ty>,
654655/// Marks this function as variadic (accepting a variable number of arguments).
656pub c_variadic: bool,
657658/// The count of non-variadic arguments.
659 ///
660 /// Should only be different from args.len() when c_variadic is true.
661 /// This can be used to know whether an argument is variadic or not.
662pub fixed_count: u32,
663/// The calling convention of this function.
664pub conv: CanonAbi,
665/// Indicates if an unwind may happen across a call to this function.
666pub can_unwind: bool,
667}
668669// Needs to be a custom impl because of the bounds on the `TyAndLayout` debug impl.
670impl<'a, Ty: fmt::Display> fmt::Debugfor FnAbi<'a, Ty> {
671fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
672let FnAbi { args, ret, c_variadic, fixed_count, conv, can_unwind } = self;
673f.debug_struct("FnAbi")
674 .field("args", args)
675 .field("ret", ret)
676 .field("c_variadic", c_variadic)
677 .field("fixed_count", fixed_count)
678 .field("conv", conv)
679 .field("can_unwind", can_unwind)
680 .finish()
681 }
682}
683684impl<'a, Ty> FnAbi<'a, Ty> {
685pub fn adjust_for_foreign_abi<C>(&mut self, cx: &C, abi: ExternAbi)
686where
687Ty: TyAbiInterface<'a, C> + Copy,
688 C: HasDataLayout + HasTargetSpec + HasX86AbiOpt,
689 {
690if abi == ExternAbi::X86Interrupt {
691if let Some(arg) = self.args.first_mut() {
692arg.pass_by_stack_offset(None);
693 }
694return;
695 }
696697let spec = cx.target_spec();
698match &spec.arch {
699 Arch::X86 => {
700let flavor = match abi {
701 ExternAbi::Fastcall { .. } => x86::Flavor::Fastcall,
702 ExternAbi::Vectorcall { .. } => x86::Flavor::Vectorcall,
703 ExternAbi::C { .. } | ExternAbi::Cdecl { .. } | ExternAbi::Stdcall { .. } => {
704 x86::Flavor::General { regparam: cx.x86_abi_opt().regparm }
705 }
706_ => x86::Flavor::General { regparam: None },
707 };
708let reg_struct_return = cx.x86_abi_opt().reg_struct_return;
709let opts = x86::X86Options { flavor, reg_struct_return };
710if spec.is_like_msvc {
711 x86_win32::compute_abi_info(cx, self, opts);
712 } else {
713 x86::compute_abi_info(cx, self, opts);
714 }
715 }
716 Arch::X86_64 => match abi {
717 ExternAbi::SysV64 { .. } => x86_64::compute_abi_info(cx, self),
718 ExternAbi::Win64 { .. } | ExternAbi::Vectorcall { .. } => {
719 x86_win64::compute_abi_info(cx, self)
720 }
721_ => {
722if cx.target_spec().is_like_windows {
723 x86_win64::compute_abi_info(cx, self)
724 } else {
725 x86_64::compute_abi_info(cx, self)
726 }
727 }
728 },
729 Arch::AArch64 | Arch::Arm64EC => {
730let kind = if cx.target_spec().is_like_darwin {
731 aarch64::AbiKind::DarwinPCS732 } else if cx.target_spec().is_like_windows {
733 aarch64::AbiKind::Win64734 } else {
735 aarch64::AbiKind::AAPCS736 };
737 aarch64::compute_abi_info(cx, self, kind)
738 }
739 Arch::AmdGpu => amdgpu::compute_abi_info(cx, self),
740 Arch::Arm => arm::compute_abi_info(cx, self),
741 Arch::Avr => avr::compute_abi_info(cx, self),
742 Arch::LoongArch32 | Arch::LoongArch64 => loongarch::compute_abi_info(cx, self),
743 Arch::M68k => m68k::compute_abi_info(cx, self),
744 Arch::CSky => csky::compute_abi_info(cx, self),
745 Arch::Mips | Arch::Mips32r6 => mips::compute_abi_info(cx, self),
746 Arch::Mips64 | Arch::Mips64r6 => mips64::compute_abi_info(cx, self),
747 Arch::PowerPC => powerpc::compute_abi_info(cx, self),
748 Arch::PowerPC64 => powerpc64::compute_abi_info(cx, self),
749 Arch::S390x => s390x::compute_abi_info(cx, self),
750 Arch::Msp430 => msp430::compute_abi_info(cx, self),
751 Arch::Sparc => sparc::compute_abi_info(cx, self),
752 Arch::Sparc64 => sparc64::compute_abi_info(cx, self),
753 Arch::Nvptx64 => {
754if abi == ExternAbi::PtxKernel || abi == ExternAbi::GpuKernel {
755 nvptx64::compute_ptx_kernel_abi_info(cx, self)
756 } else {
757 nvptx64::compute_abi_info(cx, self)
758 }
759 }
760 Arch::Hexagon => hexagon::compute_abi_info(cx, self),
761 Arch::Xtensa => xtensa::compute_abi_info(cx, self),
762 Arch::RiscV32 | Arch::RiscV64 => riscv::compute_abi_info(cx, self),
763 Arch::Wasm32 | Arch::Wasm64 => wasm::compute_abi_info(cx, self),
764 Arch::Bpf => bpf::compute_abi_info(cx, self),
765 arch @ (Arch::SpirV | Arch::Other(_)) => {
766{
::core::panicking::panic_fmt(format_args!("no lowering implemented for {0}",
arch));
}panic!("no lowering implemented for {arch}")767 }
768 }
769 }
770771pub fn adjust_for_rust_abi<C>(&mut self, cx: &C)
772where
773Ty: TyAbiInterface<'a, C> + Copy,
774 C: HasDataLayout + HasTargetSpec,
775 {
776let spec = cx.target_spec();
777match &spec.arch {
778 Arch::X86 => x86::compute_rust_abi_info(cx, self),
779 Arch::RiscV32 | Arch::RiscV64 => riscv::compute_rust_abi_info(cx, self),
780 Arch::LoongArch32 | Arch::LoongArch64 => loongarch::compute_rust_abi_info(cx, self),
781 Arch::AArch64 => aarch64::compute_rust_abi_info(cx, self),
782 Arch::Bpf => bpf::compute_rust_abi_info(self),
783_ => {}
784 };
785786// Decides whether we can pass the given SIMD argument via `PassMode::Direct`.
787 // May only return `true` if the target will always pass those arguments the same way,
788 // no matter what the user does with `-Ctarget-feature`! In other words, whatever
789 // target features are required to pass a SIMD value in registers must be listed in
790 // the `abi_required_features` for the current target and ABI.
791let can_pass_simd_directly = |arg: &ArgAbi<'_, Ty>| match &spec.arch {
792// On x86, if we have SSE2 (which we have by default for x86_64), we can always pass up
793 // to 128-bit-sized vectors.
794Arch::X86if spec.rustc_abi == Some(RustcAbi::X86Sse2) => arg.layout.size.bits() <= 128,
795 Arch::X86_64if spec.rustc_abi != Some(RustcAbi::Softfloat) => {
796// x86-64 non-softfloat targets all require SSE2 so we can use SSE registers.
797arg.layout.size.bits() <= 128
798}
799 Arch::AArch64if spec.rustc_abi != Some(RustcAbi::Softfloat) => {
800// aarch64 non-softfloat targets all require Neon
801arg.layout.size.bits() <= 128
802}
803_ => false,
804 };
805806for (arg_idx, arg) in self
807.args
808 .iter_mut()
809 .enumerate()
810 .map(|(idx, arg)| (Some(idx), arg))
811 .chain(iter::once((None, &mut self.ret)))
812 {
813// If the logic above already picked a specific type to cast the argument to, leave that
814 // in place.
815if #[allow(non_exhaustive_omitted_patterns)] match arg.mode {
PassMode::Ignore | PassMode::Cast { .. } => true,
_ => false,
}matches!(arg.mode, PassMode::Ignore | PassMode::Cast { .. }) {
816continue;
817 }
818819// Always extend `bool` in the Rust ABI
820let extend_bool = |attrs: &mut ArgAttributes, scalar: Scalar| {
821if scalar.is_bool() {
822 attrs.ext(ArgExtension::Zext);
823 }
824 };
825826if let PassMode::Direct(attrs) = &mut arg.mode
827 && let BackendRepr::Scalar(scalar) = arg.layout.backend_repr
828 {
829 extend_bool(attrs, scalar);
830 } else if let PassMode::Pair(a_attrs, b_attrs) = &mut arg.mode
831 && let BackendRepr::ScalarPair { a, b, b_offset: _ } = arg.layout.backend_repr
832 {
833 extend_bool(a_attrs, a);
834 extend_bool(b_attrs, b);
835 }
836837if arg_idx.is_none()
838 && arg.layout.size > Primitive::Pointer(AddressSpace::ZERO).size(cx) * 2
839 && !#[allow(non_exhaustive_omitted_patterns)] match arg.layout.backend_repr {
BackendRepr::SimdVector { .. } | BackendRepr::SimdScalableVector { .. } =>
true,
_ => false,
}matches!(
840 arg.layout.backend_repr,
841 BackendRepr::SimdVector { .. } | BackendRepr::SimdScalableVector { .. }
842 )843 {
844// Return values larger than 2 registers using a return area
845 // pointer. LLVM and Cranelift disagree about how to return
846 // values that don't fit in the registers designated for return
847 // values. LLVM will force the entire return value to be passed
848 // by return area pointer, while Cranelift will look at each IR level
849 // return value independently and decide to pass it in a
850 // register or not, which would result in the return value
851 // being passed partially in registers and partially through a
852 // return area pointer. For large IR-level values such as `i128`,
853 // cranelift will even split up the value into smaller chunks.
854 //
855 // While Cranelift may need to be fixed as the LLVM behavior is
856 // generally more correct with respect to the surface language,
857 // forcing this behavior in rustc itself makes it easier for
858 // other backends to conform to the Rust ABI and for the C ABI
859 // rustc already handles this behavior anyway.
860 //
861 // In addition LLVM's decision to pass the return value in
862 // registers or using a return area pointer depends on how
863 // exactly the return type is lowered to an LLVM IR type. For
864 // example `Option<u128>` can be lowered as `{ i128, i128 }`
865 // in which case the x86_64 backend would use a return area
866 // pointer, or it could be passed as `{ i32, i128 }` in which
867 // case the x86_64 backend would pass it in registers by taking
868 // advantage of an LLVM ABI extension that allows using 3
869 // registers for the x86_64 sysv call conv rather than the
870 // officially specified 2 registers.
871 //
872 // FIXME: Technically we should look at the amount of available
873 // return registers rather than guessing that there are 2
874 // registers for return values. In practice only a couple of
875 // architectures have less than 2 return registers. None of
876 // which supported by Cranelift.
877 //
878 // NOTE: This adjustment is only necessary for the Rust ABI as
879 // for other ABI's the calling convention implementations in
880 // rustc_target already ensure any return value which doesn't
881 // fit in the available amount of return registers is passed in
882 // the right way for the current target.
883 //
884 // The adjustment is not necessary nor desired for types with a vector
885 // representation; those are handled below.
886arg.make_indirect();
887continue;
888 }
889890match arg.layout.backend_repr {
891 BackendRepr::Memory { .. } => {
892// Compute `Aggregate` ABI.
893894let is_indirect_not_on_stack = #[allow(non_exhaustive_omitted_patterns)] match arg.mode {
PassMode::Indirect { mode: IndirectMode::Pointer, .. } |
PassMode::IndirectUnsized { .. } => true,
_ => false,
}matches!(
895 arg.mode,
896 PassMode::Indirect { mode: IndirectMode::Pointer, .. }
897 | PassMode::IndirectUnsized { .. }
898 );
899if !is_indirect_not_on_stack {
::core::panicking::panic("assertion failed: is_indirect_not_on_stack")
};assert!(is_indirect_not_on_stack);
900901let size = arg.layout.size;
902if arg.layout.is_sized()
903 && size <= Primitive::Pointer(AddressSpace::ZERO).size(cx)
904 {
905// We want to pass small aggregates as immediates, but using
906 // an LLVM aggregate type for this leads to bad optimizations,
907 // so we pick an appropriately sized integer type instead.
908arg.cast_to_maybe_noundef(Reg { kind: RegKind::Integer, size }, cx);
909 } else if self.conv == CanonAbi::RustTail && arg_idx.is_some() {
910if !arg.layout.is_sized() {
{
::core::panicking::panic_fmt(format_args!("extern \"tail\" arguments must be sized"));
}
};assert!(arg.layout.is_sized(), "extern \"tail\" arguments must be sized");
911 arg.pass_by_stack_offset(None);
912 }
913 }
914915 BackendRepr::SimdVector { .. } => {
916// This is a fun case! The gist of what this is doing is
917 // that we want callers and callees to always agree on the
918 // ABI of how they pass SIMD arguments. If we were to *not*
919 // make these arguments indirect then they'd be immediates
920 // in LLVM, which means that they'd used whatever the
921 // appropriate ABI is for the callee and the caller. That
922 // means, for example, if the caller doesn't have AVX
923 // enabled but the callee does, then passing an AVX argument
924 // across this boundary would cause corrupt data to show up.
925 //
926 // This problem is fixed by passing most SIMD arguments through memory between
927 // callers and callees which should get them all to agree on ABI regardless of
928 // target feature sets, except for those that rely on target features that we
929 // know to be always available. Some more information about this issue can be
930 // found in #44367 and the comment on `can_pass_simd_directly`.
931if spec.simd_types_indirect && !can_pass_simd_directly(arg) {
932 arg.make_indirect();
933 }
934 }
935936_ => {}
937 }
938 }
939 }
940}
941942/// Determines whether `layout` contains no uninit bytes (no padding, no unions),
943/// using only the computed layout.
944///
945/// Conservative: returns `false` for anything it cannot prove fully initialized,
946/// including multi-variant enums and SIMD vectors.
947// FIXME: extend to multi-variant enums (per-variant padding analysis needed).
948fn layout_is_noundef<'a, Ty, C>(layout: TyAndLayout<'a, Ty>, cx: &C) -> bool949where
950Ty: TyAbiInterface<'a, C> + Copy,
951 C: HasDataLayout,
952{
953match layout.backend_repr {
954 BackendRepr::Scalar(scalar) => !scalar.is_uninit_valid(),
955 BackendRepr::ScalarPair { a: s1, b: s2, b_offset: _ } => {
956 !s1.is_uninit_valid()
957 && !s2.is_uninit_valid()
958// Ensure there is no padding.
959&& s1.size(cx) + s2.size(cx) == layout.size
960 }
961 BackendRepr::Memory { .. } => match layout.fields {
962 FieldsShape::Primitive | FieldsShape::Union(_) => false,
963// Array elements are at stride offsets with no inter-element gaps.
964FieldsShape::Array { stride: _, count } => {
965count == 0 || layout_is_noundef(layout.field(cx, 0), cx)
966 }
967 FieldsShape::Arbitrary { .. } => {
968// With `Variants::Multiple`, `layout.fields` only covers shared
969 // bytes (niche/discriminant); per-variant data is absent, so
970 // full coverage cannot be proven.
971#[allow(non_exhaustive_omitted_patterns)] match layout.variants {
Variants::Single { .. } => true,
_ => false,
}matches!(layout.variants, Variants::Single { .. }) && fields_are_noundef(layout, cx)
972 }
973 },
974 BackendRepr::SimdVector { .. } | BackendRepr::SimdScalableVector { .. } => false,
975 }
976}
977978/// Returns `true` if the fields of `layout` contiguously cover bytes `0..layout.size`
979/// with no padding gaps and each field is recursively `layout_is_noundef`.
980fn fields_are_noundef<'a, Ty, C>(layout: TyAndLayout<'a, Ty>, cx: &C) -> bool981where
982Ty: TyAbiInterface<'a, C> + Copy,
983 C: HasDataLayout,
984{
985let mut cursor = Size::ZERO;
986for i in layout.fields.index_by_increasing_offset() {
987let field = layout.field(cx, i);
988if field.size == Size::ZERO {
989continue;
990 }
991if layout.fields.offset(i) != cursor {
992return false;
993 }
994if !layout_is_noundef(field, cx) {
995return false;
996 }
997 cursor += field.size;
998 }
999cursor == layout.size
1000}
10011002// Some types are used a lot. Make sure they don't unintentionally get bigger.
1003#[cfg(target_pointer_width = "64")]
1004mod size_asserts {
1005use rustc_data_structures::static_assert_size;
10061007use super::*;
1008// tidy-alphabetical-start
1009const _: [(); 56] = [(); ::std::mem::size_of::<ArgAbi<'_, usize>>()];static_assert_size!(ArgAbi<'_, usize>, 56);
1010const _: [(); 80] = [(); ::std::mem::size_of::<FnAbi<'_, usize>>()];static_assert_size!(FnAbi<'_, usize>, 80);
1011// tidy-alphabetical-end
1012}