Skip to main content

rustc_target/callconv/
x86.rs

1use rustc_abi::{
2    AddressSpace, Align, BackendRepr, Float, HasDataLayout, Primitive, Reg, RegKind, TyAndLayout,
3};
4
5use crate::callconv::{
6    ArgAbi, ArgAttribute, ArgAttributes, CastTarget, FnAbi, PassMode, TyAbiInterface,
7};
8use crate::spec::{HasTargetSpec, RustcAbi};
9
10/// Is this a struct with a single float field?
11fn is_single_fp_element<'a, Ty, C>(mut layout: TyAndLayout<'a, Ty>, cx: &C) -> bool
12where
13    Ty: TyAbiInterface<'a, C> + Copy,
14    C: HasDataLayout,
15{
16    // On X86 over-aligned structs are disqualified.
17    let outer_size = layout.layout.size();
18
19    loop {
20        // We're only looking for scalar types that are non-ZST.
21        layout = layout.peel_transparent_wrappers_from_non_1zst(cx);
22
23        return match layout.backend_repr {
24            BackendRepr::Scalar(scalar) => match scalar.primitive() {
25                Primitive::Float(float) => float.size() == outer_size,
26                Primitive::Int(_, _) | Primitive::Pointer(_) => false,
27            },
28            BackendRepr::Memory { .. } => {
29                // Structs, unions and arrays all qualify.
30                if let Some((_idx, field)) = layout.non_zst_field_ignore_alignment(cx) {
31                    // NOTE: alignment is not relevant here, checking for 1-ZST is incorrect.
32                    layout = field;
33                    continue;
34                } else {
35                    false
36                }
37            }
38            _ => false,
39        };
40    }
41}
42
43#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for Flavor {
}
#[automatically_derived]impl ::core::clone::Clone for Flavor {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<Option<u32>>;
        *self
    }
}Clone, #[automatically_derived]impl ::core::marker::Copy for Flavor { }Copy, #[automatically_derived]impl ::core::marker::StructuralPartialEq for Flavor {
}
#[automatically_derived]impl ::core::cmp::PartialEq for Flavor {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
                ::core::intrinsics::discriminant_value(other) &&
            match (self, other) {
                (Self::General { regparam: __self_0 }, Self::General {
                    regparam: __arg1_0 }) => __self_0 == __arg1_0,
                _ => true,
            }
    }
}PartialEq)]
44pub(crate) enum Flavor {
45    General { regparam: Option<u32> },
46    Fastcall,
47    Vectorcall,
48}
49
50pub(crate) fn pass_on_x87_floating_point_stack<'a, C, Ty>(cx: &C, arg_abi: &mut ArgAbi<'a, Ty>)
51where
52    Ty: TyAbiInterface<'a, C> + Copy,
53    C: HasDataLayout,
54{
55    let mut cast: CastTarget = match arg_abi.layout.size.bytes() {
56        4 => Reg::f32().into(),
57        8 => Reg::f64().into(),
58        _ => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("arg must be the size of a `f32` or `f64`")));
}unreachable!("arg must be the size of a `f32` or `f64`"),
59    };
60    cast.pass_via_x87_floating_point_stack = true;
61    // Forward whether the argument is `NoUndef` or not to improve codegen.
62    cast.attrs = if let PassMode::Direct(attrs) = arg_abi.mode {
63        attrs
64    } else if super::layout_is_noundef(arg_abi.layout, cx) {
65        ArgAttribute::NoUndef.into()
66    } else {
67        ArgAttributes::new()
68    };
69    arg_abi.mode = PassMode::Cast { pad_i32_count: 0, cast: Box::new(cast) };
70}
71
72#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for X86Options {
}
#[automatically_derived]impl ::core::clone::Clone for X86Options {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<Flavor>;
        let _: ::core::clone::AssertParamIsClone<bool>;
        *self
    }
}Clone, #[automatically_derived]impl ::core::marker::Copy for X86Options { }Copy)]
73pub(crate) struct X86Options {
74    pub flavor: Flavor,
75    pub reg_struct_return: bool,
76}
77
78fn classify_ret<'a, Ty, C>(cx: &C, opts: X86Options, ret: &mut ArgAbi<'a, Ty>)
79where
80    Ty: TyAbiInterface<'a, C> + Copy,
81    C: HasDataLayout + HasTargetSpec,
82{
83    if ret.layout.is_x87_f80(cx) {
84        ret.cast_to(Reg::x87_f80());
85        return;
86    }
87    // "vectorcall" returns floats in `xmm0`, and soft float also does not use the x87 stack.
88    let uses_x87_return = cx.target_spec().rustc_abi != Some(RustcAbi::Softfloat)
89        && opts.flavor != Flavor::Vectorcall;
90    if ret.layout.is_aggregate() && ret.layout.is_sized() {
91        // Returning a structure. Most often, this will use
92        // a hidden first argument. On some platforms, though,
93        // small structs are returned as integers.
94        //
95        // Some links:
96        // https://www.angelcode.com/dev/callconv/callconv.html
97        // Clang's ABI handling is in lib/CodeGen/TargetInfo.cpp
98        let t = cx.target_spec();
99        if let Some(Float::F16) = ret.layout.complex_float(cx) {
100            // `_Complex _Float16` is returned as `<2 x half>`.
101            let kind = RegKind::Vector { hint_vector_elem: Primitive::Float(Float::F16) };
102            ret.cast_to(Reg { kind, size: ret.layout.size });
103        } else if t.abi_return_struct_as_int
104            || opts.reg_struct_return
105            || ret.layout.is_complex_number(cx)
106        {
107            // According to Clang, everyone but MSVC returns single-element
108            // float aggregates directly in a floating-point register.
109            if is_single_fp_element(ret.layout, cx) {
110                match ret.layout.size.bytes() {
111                    2 => ret.cast_to(Reg::f16()),
112                    // The calling convention passes `f32`/`f64` returns via the x87 stack. Tell
113                    // the backend to convert to an `x87_f80` manually to avoid LLVM quieting
114                    // signalling NaNs when loading/storing to/from the x87 stack.
115                    4 | 8 if uses_x87_return => pass_on_x87_floating_point_stack(cx, ret),
116                    4 => ret.cast_to(Reg::f32()),
117                    8 => ret.cast_to(Reg::f64()),
118                    _ => ret.make_indirect(),
119                }
120            } else {
121                match ret.layout.size.bytes() {
122                    1 => ret.cast_to(Reg::i8()),
123                    2 => ret.cast_to(Reg::i16()),
124                    4 => ret.cast_to(Reg::i32()),
125                    8 => ret.cast_to(Reg::i64()),
126                    _ => ret.make_indirect(),
127                }
128            }
129        } else {
130            ret.make_indirect();
131        }
132    } else if uses_x87_return
133        && let BackendRepr::Scalar(scalar) = ret.layout.backend_repr
134        && #[allow(non_exhaustive_omitted_patterns)] match scalar.primitive() {
    Primitive::Float(Float::F32 | Float::F64) => true,
    _ => false,
}matches!(scalar.primitive(), Primitive::Float(Float::F32 | Float::F64))
135    {
136        pass_on_x87_floating_point_stack(cx, ret);
137    } else {
138        ret.extend_integer_width_to(32);
139    }
140}
141
142fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
143where
144    Ty: TyAbiInterface<'a, C> + Copy,
145    C: HasDataLayout + HasTargetSpec,
146{
147    let t = cx.target_spec();
148    let align_4 = Align::from_bytes(4).unwrap();
149    let align_16 = Align::from_bytes(16).unwrap();
150
151    if arg.layout.is_x87_f80(cx) {
152        // This branch and cast are not technically required, but it makes the IR more readable.
153        arg.cast_to(Reg::x87_f80());
154        return;
155    } else if arg.layout.is_aggregate() {
156        // We need to compute the alignment of the `byval` argument. The rules can be found in
157        // `X86_32ABIInfo::getTypeStackAlignInBytes` in Clang's `TargetInfo.cpp`. Summarized
158        // here, they are:
159        //
160        // 1. If the natural alignment of the type is <= 4, the alignment is 4.
161        //
162        // 2. Otherwise, on Linux, the alignment of any vector type is the natural alignment.
163        // This doesn't matter here because we only pass aggregates via `byval`, not vectors.
164        //
165        // 3. Otherwise, on Apple platforms, the alignment of anything that contains a vector
166        // type is 16.
167        //
168        // 4. If none of these conditions are true, the alignment is 4.
169
170        fn contains_vector<'a, Ty, C>(cx: &C, layout: TyAndLayout<'a, Ty>) -> bool
171        where
172            Ty: TyAbiInterface<'a, C> + Copy,
173        {
174            match layout.backend_repr {
175                BackendRepr::Scalar(_) | BackendRepr::ScalarPair { .. } => false,
176                BackendRepr::SimdVector { .. } => true,
177                BackendRepr::Memory { .. } => {
178                    for i in 0..layout.fields.count() {
179                        if contains_vector(cx, layout.field(cx, i)) {
180                            return true;
181                        }
182                    }
183                    false
184                }
185                BackendRepr::SimdScalableVector { .. } => {
186                    {
    ::core::panicking::panic_fmt(format_args!("scalable vectors are unsupported"));
}panic!("scalable vectors are unsupported")
187                }
188            }
189        }
190
191        let byval_align = if arg.layout.align < align_4 {
192            // (1.)
193            align_4
194        } else if t.is_like_darwin && contains_vector(cx, arg.layout) {
195            // (3.)
196            align_16
197        } else {
198            // (4.)
199            align_4
200        };
201
202        arg.pass_by_stack_offset(Some(byval_align));
203    } else {
204        arg.extend_integer_width_to(32);
205    }
206}
207
208pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>, opts: X86Options)
209where
210    Ty: TyAbiInterface<'a, C> + Copy,
211    C: HasDataLayout + HasTargetSpec,
212{
213    if !fn_abi.ret.is_ignore() {
214        classify_ret(cx, opts, &mut fn_abi.ret);
215    }
216
217    for arg in fn_abi.args.iter_mut() {
218        if arg.is_ignore() || !arg.layout.is_sized() {
219            continue;
220        }
221
222        if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
223            arg.make_indirect();
224            continue;
225        }
226
227        classify_arg(cx, arg);
228    }
229
230    fill_inregs(cx, fn_abi, opts, false);
231}
232
233pub(crate) fn fill_inregs<'a, Ty, C>(
234    cx: &C,
235    fn_abi: &mut FnAbi<'a, Ty>,
236    opts: X86Options,
237    rust_abi: bool,
238) where
239    Ty: TyAbiInterface<'a, C> + Copy,
240{
241    // Mark arguments as InReg like clang does it,
242    // so our fastcall/vectorcall is compatible with C/C++ fastcall/vectorcall.
243
244    // Clang reference: lib/CodeGen/TargetInfo.cpp
245    // See X86_32ABIInfo::shouldPrimitiveUseInReg(), X86_32ABIInfo::updateFreeRegs()
246
247    // IsSoftFloatABI is only set to true on ARM platforms,
248    // which in turn can't be x86?
249
250    // The number of registers available for argument passing.
251    //
252    // An `extern "fastcall"` and `extern "vectorcall"` function always have 2 registers available.
253    // Otherwise the `regparam` count (in the range 0..=3) determines the number of available
254    // registers. If unspecified, no registers are used for argument passing.
255    //
256    // Functions that take a variable number of arguments continue to be passed all of their
257    // arguments on the stack.
258    let mut free_regs = match opts.flavor {
259        _ if fn_abi.c_variadic => 0,
260        Flavor::Fastcall | Flavor::Vectorcall => 2,
261        Flavor::General { regparam } => u64::from(regparam.unwrap_or(0)),
262    };
263
264    if free_regs == 0 {
265        return;
266    }
267
268    // For types generating PassMode::Cast, InRegs will not be set.
269    // Maybe, this is a FIXME
270    let has_casts = fn_abi.args.iter().any(|arg| #[allow(non_exhaustive_omitted_patterns)] match arg.mode {
    PassMode::Cast { .. } => true,
    _ => false,
}matches!(arg.mode, PassMode::Cast { .. }));
271    if has_casts && rust_abi {
272        return;
273    }
274
275    for arg in fn_abi.args.iter_mut() {
276        let attrs = match arg.mode {
277            PassMode::Ignore | PassMode::Indirect { attrs: _, address_space: _, mode: _ } => {
278                continue;
279            }
280            PassMode::Cast { .. } if arg.layout.is_x87_f80(cx) => continue,
281            PassMode::Direct(ref mut attrs) => attrs,
282            PassMode::Pair(..)
283            | PassMode::IndirectUnsized { attrs: _, meta_attrs: _ }
284            | PassMode::Cast { .. } => {
285                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("x86 shouldn\'t be passing arguments by {0:?}",
                arg.mode)));
}unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode)
286            }
287        };
288
289        // At this point we know this must be a primitive of sorts.
290        let unit = arg.layout.homogeneous_aggregate(cx).unwrap().unit().unwrap();
291        {
    match (&unit.size, &arg.layout.size) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(unit.size, arg.layout.size);
292        if #[allow(non_exhaustive_omitted_patterns)] match unit.kind {
    RegKind::Float | RegKind::Vector { .. } => true,
    _ => false,
}matches!(unit.kind, RegKind::Float | RegKind::Vector { .. }) {
293            continue;
294        }
295
296        let size_in_regs = arg.layout.size.bits().div_ceil(32);
297
298        if size_in_regs == 0 {
299            continue;
300        }
301
302        if size_in_regs > free_regs {
303            break;
304        }
305
306        free_regs -= size_in_regs;
307
308        if arg.layout.size.bits() <= 32 && unit.kind == RegKind::Integer {
309            attrs.set(ArgAttribute::InReg);
310        }
311
312        if free_regs == 0 {
313            break;
314        }
315    }
316}
317
318pub(crate) fn compute_rust_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
319where
320    Ty: TyAbiInterface<'a, C> + Copy,
321    C: HasDataLayout + HasTargetSpec,
322{
323    // Avoid returning floats in x87 registers on x86 as loading and storing from x87
324    // registers will quiet signalling NaNs. Also avoid using SSE registers since they
325    // are not always available (depending on target features).
326    if !fn_abi.ret.is_ignore() {
327        let has_float = match fn_abi.ret.layout.backend_repr {
328            BackendRepr::Scalar(s) => #[allow(non_exhaustive_omitted_patterns)] match s.primitive() {
    Primitive::Float(_) => true,
    _ => false,
}matches!(s.primitive(), Primitive::Float(_)),
329            BackendRepr::ScalarPair { a: s1, b: s2, b_offset: _ } => {
330                #[allow(non_exhaustive_omitted_patterns)] match s1.primitive() {
    Primitive::Float(_) => true,
    _ => false,
}matches!(s1.primitive(), Primitive::Float(_))
331                    || #[allow(non_exhaustive_omitted_patterns)] match s2.primitive() {
    Primitive::Float(_) => true,
    _ => false,
}matches!(s2.primitive(), Primitive::Float(_))
332            }
333            _ => false, // anyway not passed via registers on x86
334        };
335        if has_float {
336            if cx.target_spec().rustc_abi == Some(RustcAbi::X86Sse2)
337                && fn_abi.ret.layout.backend_repr.is_scalar()
338                && fn_abi.ret.layout.size.bits() <= 128
339            {
340                // This is a single scalar that fits into an SSE register, and the target uses the
341                // SSE ABI. We prefer this over integer registers as float scalars need to be in SSE
342                // registers for float operations, so that's the best place to pass them around.
343                fn_abi.ret.cast_to(Reg::opaque_vector(fn_abi.ret.layout.size));
344            } else if fn_abi.ret.layout.size <= Primitive::Pointer(AddressSpace::ZERO).size(cx) {
345                // Same size or smaller than pointer, return in an integer register.
346                fn_abi.ret.cast_to(Reg { kind: RegKind::Integer, size: fn_abi.ret.layout.size });
347            } else {
348                // Larger than a pointer, return indirectly.
349                fn_abi.ret.make_indirect();
350            }
351            return;
352        }
353    }
354}