1use rustc_abi::{
2 AddressSpace, Align, BackendRepr, Float, HasDataLayout, Primitive, Reg, RegKind, TyAndLayout,
3};
4
5use crate::callconv::{
6 ArgAbi, ArgAttribute, ArgAttributes, CastTarget, FnAbi, PassMode, TyAbiInterface,
7};
8use crate::spec::{HasTargetSpec, RustcAbi};
9
10fn is_single_fp_element<'a, Ty, C>(mut layout: TyAndLayout<'a, Ty>, cx: &C) -> bool
12where
13 Ty: TyAbiInterface<'a, C> + Copy,
14 C: HasDataLayout,
15{
16 let outer_size = layout.layout.size();
18
19 loop {
20 layout = layout.peel_transparent_wrappers_from_non_1zst(cx);
22
23 return match layout.backend_repr {
24 BackendRepr::Scalar(scalar) => match scalar.primitive() {
25 Primitive::Float(float) => float.size() == outer_size,
26 Primitive::Int(_, _) | Primitive::Pointer(_) => false,
27 },
28 BackendRepr::Memory { .. } => {
29 if let Some((_idx, field)) = layout.non_zst_field_ignore_alignment(cx) {
31 layout = field;
33 continue;
34 } else {
35 false
36 }
37 }
38 _ => false,
39 };
40 }
41}
42
43#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for Flavor {
}
#[automatically_derived]impl ::core::clone::Clone for Flavor {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
let _: ::core::clone::AssertParamIsClone<Option<u32>>;
*self
}
}Clone, #[automatically_derived]impl ::core::marker::Copy for Flavor { }Copy, #[automatically_derived]impl ::core::marker::StructuralPartialEq for Flavor {
}
#[automatically_derived]impl ::core::cmp::PartialEq for Flavor {
#[inline]
fn eq(&self, other: &Self) -> bool {
::core::intrinsics::discriminant_value(self) ==
::core::intrinsics::discriminant_value(other) &&
match (self, other) {
(Self::General { regparam: __self_0 }, Self::General {
regparam: __arg1_0 }) => __self_0 == __arg1_0,
_ => true,
}
}
}PartialEq)]
44pub(crate) enum Flavor {
45 General { regparam: Option<u32> },
46 Fastcall,
47 Vectorcall,
48}
49
50pub(crate) fn pass_on_x87_floating_point_stack<'a, C, Ty>(cx: &C, arg_abi: &mut ArgAbi<'a, Ty>)
51where
52 Ty: TyAbiInterface<'a, C> + Copy,
53 C: HasDataLayout,
54{
55 let mut cast: CastTarget = match arg_abi.layout.size.bytes() {
56 4 => Reg::f32().into(),
57 8 => Reg::f64().into(),
58 _ => {
::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
format_args!("arg must be the size of a `f32` or `f64`")));
}unreachable!("arg must be the size of a `f32` or `f64`"),
59 };
60 cast.pass_via_x87_floating_point_stack = true;
61 cast.attrs = if let PassMode::Direct(attrs) = arg_abi.mode {
63 attrs
64 } else if super::layout_is_noundef(arg_abi.layout, cx) {
65 ArgAttribute::NoUndef.into()
66 } else {
67 ArgAttributes::new()
68 };
69 arg_abi.mode = PassMode::Cast { pad_i32_count: 0, cast: Box::new(cast) };
70}
71
72#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
for X86Options {
}
#[automatically_derived]impl ::core::clone::Clone for X86Options {
#[inline]
#[rustc_trivial_field_reads]
fn clone(&self) -> Self {
let _: ::core::clone::AssertParamIsClone<Flavor>;
let _: ::core::clone::AssertParamIsClone<bool>;
*self
}
}Clone, #[automatically_derived]impl ::core::marker::Copy for X86Options { }Copy)]
73pub(crate) struct X86Options {
74 pub flavor: Flavor,
75 pub reg_struct_return: bool,
76}
77
78fn classify_ret<'a, Ty, C>(cx: &C, opts: X86Options, ret: &mut ArgAbi<'a, Ty>)
79where
80 Ty: TyAbiInterface<'a, C> + Copy,
81 C: HasDataLayout + HasTargetSpec,
82{
83 if ret.layout.is_x87_f80(cx) {
84 ret.cast_to(Reg::x87_f80());
85 return;
86 }
87 let uses_x87_return = cx.target_spec().rustc_abi != Some(RustcAbi::Softfloat)
89 && opts.flavor != Flavor::Vectorcall;
90 if ret.layout.is_aggregate() && ret.layout.is_sized() {
91 let t = cx.target_spec();
99 if let Some(Float::F16) = ret.layout.complex_float(cx) {
100 let kind = RegKind::Vector { hint_vector_elem: Primitive::Float(Float::F16) };
102 ret.cast_to(Reg { kind, size: ret.layout.size });
103 } else if t.abi_return_struct_as_int
104 || opts.reg_struct_return
105 || ret.layout.is_complex_number(cx)
106 {
107 if is_single_fp_element(ret.layout, cx) {
110 match ret.layout.size.bytes() {
111 2 => ret.cast_to(Reg::f16()),
112 4 | 8 if uses_x87_return => pass_on_x87_floating_point_stack(cx, ret),
116 4 => ret.cast_to(Reg::f32()),
117 8 => ret.cast_to(Reg::f64()),
118 _ => ret.make_indirect(),
119 }
120 } else {
121 match ret.layout.size.bytes() {
122 1 => ret.cast_to(Reg::i8()),
123 2 => ret.cast_to(Reg::i16()),
124 4 => ret.cast_to(Reg::i32()),
125 8 => ret.cast_to(Reg::i64()),
126 _ => ret.make_indirect(),
127 }
128 }
129 } else {
130 ret.make_indirect();
131 }
132 } else if uses_x87_return
133 && let BackendRepr::Scalar(scalar) = ret.layout.backend_repr
134 && #[allow(non_exhaustive_omitted_patterns)] match scalar.primitive() {
Primitive::Float(Float::F32 | Float::F64) => true,
_ => false,
}matches!(scalar.primitive(), Primitive::Float(Float::F32 | Float::F64))
135 {
136 pass_on_x87_floating_point_stack(cx, ret);
137 } else {
138 ret.extend_integer_width_to(32);
139 }
140}
141
142fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
143where
144 Ty: TyAbiInterface<'a, C> + Copy,
145 C: HasDataLayout + HasTargetSpec,
146{
147 let t = cx.target_spec();
148 let align_4 = Align::from_bytes(4).unwrap();
149 let align_16 = Align::from_bytes(16).unwrap();
150
151 if arg.layout.is_x87_f80(cx) {
152 arg.cast_to(Reg::x87_f80());
154 return;
155 } else if arg.layout.is_aggregate() {
156 fn contains_vector<'a, Ty, C>(cx: &C, layout: TyAndLayout<'a, Ty>) -> bool
171 where
172 Ty: TyAbiInterface<'a, C> + Copy,
173 {
174 match layout.backend_repr {
175 BackendRepr::Scalar(_) | BackendRepr::ScalarPair { .. } => false,
176 BackendRepr::SimdVector { .. } => true,
177 BackendRepr::Memory { .. } => {
178 for i in 0..layout.fields.count() {
179 if contains_vector(cx, layout.field(cx, i)) {
180 return true;
181 }
182 }
183 false
184 }
185 BackendRepr::SimdScalableVector { .. } => {
186 {
::core::panicking::panic_fmt(format_args!("scalable vectors are unsupported"));
}panic!("scalable vectors are unsupported")
187 }
188 }
189 }
190
191 let byval_align = if arg.layout.align < align_4 {
192 align_4
194 } else if t.is_like_darwin && contains_vector(cx, arg.layout) {
195 align_16
197 } else {
198 align_4
200 };
201
202 arg.pass_by_stack_offset(Some(byval_align));
203 } else {
204 arg.extend_integer_width_to(32);
205 }
206}
207
208pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>, opts: X86Options)
209where
210 Ty: TyAbiInterface<'a, C> + Copy,
211 C: HasDataLayout + HasTargetSpec,
212{
213 if !fn_abi.ret.is_ignore() {
214 classify_ret(cx, opts, &mut fn_abi.ret);
215 }
216
217 for arg in fn_abi.args.iter_mut() {
218 if arg.is_ignore() || !arg.layout.is_sized() {
219 continue;
220 }
221
222 if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
223 arg.make_indirect();
224 continue;
225 }
226
227 classify_arg(cx, arg);
228 }
229
230 fill_inregs(cx, fn_abi, opts, false);
231}
232
233pub(crate) fn fill_inregs<'a, Ty, C>(
234 cx: &C,
235 fn_abi: &mut FnAbi<'a, Ty>,
236 opts: X86Options,
237 rust_abi: bool,
238) where
239 Ty: TyAbiInterface<'a, C> + Copy,
240{
241 let mut free_regs = match opts.flavor {
259 _ if fn_abi.c_variadic => 0,
260 Flavor::Fastcall | Flavor::Vectorcall => 2,
261 Flavor::General { regparam } => u64::from(regparam.unwrap_or(0)),
262 };
263
264 if free_regs == 0 {
265 return;
266 }
267
268 let has_casts = fn_abi.args.iter().any(|arg| #[allow(non_exhaustive_omitted_patterns)] match arg.mode {
PassMode::Cast { .. } => true,
_ => false,
}matches!(arg.mode, PassMode::Cast { .. }));
271 if has_casts && rust_abi {
272 return;
273 }
274
275 for arg in fn_abi.args.iter_mut() {
276 let attrs = match arg.mode {
277 PassMode::Ignore | PassMode::Indirect { attrs: _, address_space: _, mode: _ } => {
278 continue;
279 }
280 PassMode::Cast { .. } if arg.layout.is_x87_f80(cx) => continue,
281 PassMode::Direct(ref mut attrs) => attrs,
282 PassMode::Pair(..)
283 | PassMode::IndirectUnsized { attrs: _, meta_attrs: _ }
284 | PassMode::Cast { .. } => {
285 {
::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
format_args!("x86 shouldn\'t be passing arguments by {0:?}",
arg.mode)));
}unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode)
286 }
287 };
288
289 let unit = arg.layout.homogeneous_aggregate(cx).unwrap().unit().unwrap();
291 {
match (&unit.size, &arg.layout.size) {
(left_val, right_val) => {
if !(*left_val == *right_val) {
let kind = ::core::panicking::AssertKind::Eq;
::core::panicking::assert_failed(kind, &*left_val,
&*right_val, ::core::option::Option::None);
}
}
}
};assert_eq!(unit.size, arg.layout.size);
292 if #[allow(non_exhaustive_omitted_patterns)] match unit.kind {
RegKind::Float | RegKind::Vector { .. } => true,
_ => false,
}matches!(unit.kind, RegKind::Float | RegKind::Vector { .. }) {
293 continue;
294 }
295
296 let size_in_regs = arg.layout.size.bits().div_ceil(32);
297
298 if size_in_regs == 0 {
299 continue;
300 }
301
302 if size_in_regs > free_regs {
303 break;
304 }
305
306 free_regs -= size_in_regs;
307
308 if arg.layout.size.bits() <= 32 && unit.kind == RegKind::Integer {
309 attrs.set(ArgAttribute::InReg);
310 }
311
312 if free_regs == 0 {
313 break;
314 }
315 }
316}
317
318pub(crate) fn compute_rust_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
319where
320 Ty: TyAbiInterface<'a, C> + Copy,
321 C: HasDataLayout + HasTargetSpec,
322{
323 if !fn_abi.ret.is_ignore() {
327 let has_float = match fn_abi.ret.layout.backend_repr {
328 BackendRepr::Scalar(s) => #[allow(non_exhaustive_omitted_patterns)] match s.primitive() {
Primitive::Float(_) => true,
_ => false,
}matches!(s.primitive(), Primitive::Float(_)),
329 BackendRepr::ScalarPair { a: s1, b: s2, b_offset: _ } => {
330 #[allow(non_exhaustive_omitted_patterns)] match s1.primitive() {
Primitive::Float(_) => true,
_ => false,
}matches!(s1.primitive(), Primitive::Float(_))
331 || #[allow(non_exhaustive_omitted_patterns)] match s2.primitive() {
Primitive::Float(_) => true,
_ => false,
}matches!(s2.primitive(), Primitive::Float(_))
332 }
333 _ => false, };
335 if has_float {
336 if cx.target_spec().rustc_abi == Some(RustcAbi::X86Sse2)
337 && fn_abi.ret.layout.backend_repr.is_scalar()
338 && fn_abi.ret.layout.size.bits() <= 128
339 {
340 fn_abi.ret.cast_to(Reg::opaque_vector(fn_abi.ret.layout.size));
344 } else if fn_abi.ret.layout.size <= Primitive::Pointer(AddressSpace::ZERO).size(cx) {
345 fn_abi.ret.cast_to(Reg { kind: RegKind::Integer, size: fn_abi.ret.layout.size });
347 } else {
348 fn_abi.ret.make_indirect();
350 }
351 return;
352 }
353 }
354}