rustc_codegen_llvm/
abi.rs

Help
1use std::borrow::Borrow;
2use std::cmp;
3
4use libc::c_uint;
5use rustc_abi::{BackendRepr, HasDataLayout, Primitive, Reg, RegKind, Size};
6use rustc_codegen_ssa::MemFlags;
7use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
8use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
9use rustc_codegen_ssa::traits::*;
10use rustc_middle::ty::Ty;
11use rustc_middle::ty::layout::LayoutOf;
12use rustc_middle::{bug, ty};
13use rustc_session::config;
14use rustc_target::callconv::{
15    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, Conv, FnAbi, PassMode,
16};
17use rustc_target::spec::SanitizerSet;
18use smallvec::SmallVec;
19
20use crate::attributes::llfn_attrs_from_instance;
21use crate::builder::Builder;
22use crate::context::CodegenCx;
23use crate::llvm::{self, Attribute, AttributePlace};
24use crate::type_::Type;
25use crate::type_of::LayoutLlvmExt;
26use crate::value::Value;
27use crate::{attributes, llvm_util};
28
29trait ArgAttributesExt {
30    fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value);
31    fn apply_attrs_to_callsite(
32        &self,
33        idx: AttributePlace,
34        cx: &CodegenCx<'_, '_>,
35        callsite: &Value,
36    );
37}
38
39const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] =
40    [(ArgAttribute::InReg, llvm::AttributeKind::InReg)];
41
42const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [
43    (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias),
44    (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture),
45    (ArgAttribute::NonNull, llvm::AttributeKind::NonNull),
46    (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly),
47    (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef),
48];
49
50fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> {
51    let mut regular = this.regular;
52
53    let mut attrs = SmallVec::new();
54
55    // ABI-affecting attributes must always be applied
56    for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES {
57        if regular.contains(attr) {
58            attrs.push(llattr.create_attr(cx.llcx));
59        }
60    }
61    if let Some(align) = this.pointee_align {
62        attrs.push(llvm::CreateAlignmentAttr(cx.llcx, align.bytes()));
63    }
64    match this.arg_ext {
65        ArgExtension::None => {}
66        ArgExtension::Zext => attrs.push(llvm::AttributeKind::ZExt.create_attr(cx.llcx)),
67        ArgExtension::Sext => attrs.push(llvm::AttributeKind::SExt.create_attr(cx.llcx)),
68    }
69
70    // Only apply remaining attributes when optimizing
71    if cx.sess().opts.optimize != config::OptLevel::No {
72        let deref = this.pointee_size.bytes();
73        if deref != 0 {
74            if regular.contains(ArgAttribute::NonNull) {
75                attrs.push(llvm::CreateDereferenceableAttr(cx.llcx, deref));
76            } else {
77                attrs.push(llvm::CreateDereferenceableOrNullAttr(cx.llcx, deref));
78            }
79            regular -= ArgAttribute::NonNull;
80        }
81        for (attr, llattr) in OPTIMIZATION_ATTRIBUTES {
82            if regular.contains(attr) {
83                attrs.push(llattr.create_attr(cx.llcx));
84            }
85        }
86    } else if cx.tcx.sess.opts.unstable_opts.sanitizer.contains(SanitizerSet::MEMORY) {
87        // If we're not optimising, *but* memory sanitizer is on, emit noundef, since it affects
88        // memory sanitizer's behavior.
89
90        if regular.contains(ArgAttribute::NoUndef) {
91            attrs.push(llvm::AttributeKind::NoUndef.create_attr(cx.llcx));
92        }
93    }
94
95    attrs
96}
97
98impl ArgAttributesExt for ArgAttributes {
99    fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) {
100        let attrs = get_attrs(self, cx);
101        attributes::apply_to_llfn(llfn, idx, &attrs);
102    }
103
104    fn apply_attrs_to_callsite(
105        &self,
106        idx: AttributePlace,
107        cx: &CodegenCx<'_, '_>,
108        callsite: &Value,
109    ) {
110        let attrs = get_attrs(self, cx);
111        attributes::apply_to_callsite(callsite, idx, &attrs);
112    }
113}
114
115pub(crate) trait LlvmType {
116    fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type;
117}
118
119impl LlvmType for Reg {
120    fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type {
121        match self.kind {
122            RegKind::Integer => cx.type_ix(self.size.bits()),
123            RegKind::Float => match self.size.bits() {
124                16 => cx.type_f16(),
125                32 => cx.type_f32(),
126                64 => cx.type_f64(),
127                128 => cx.type_f128(),
128                _ => bug!("unsupported float: {:?}", self),
129            },
130            RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()),
131        }
132    }
133}
134
135impl LlvmType for CastTarget {
136    fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type {
137        let rest_ll_unit = self.rest.unit.llvm_type(cx);
138        let rest_count = if self.rest.total == Size::ZERO {
139            0
140        } else {
141            assert_ne!(
142                self.rest.unit.size,
143                Size::ZERO,
144                "total size {:?} cannot be divided into units of zero size",
145                self.rest.total
146            );
147            if self.rest.total.bytes() % self.rest.unit.size.bytes() != 0 {
148                assert_eq!(self.rest.unit.kind, RegKind::Integer, "only int regs can be split");
149            }
150            self.rest.total.bytes().div_ceil(self.rest.unit.size.bytes())
151        };
152
153        // Simplify to a single unit or an array if there's no prefix.
154        // This produces the same layout, but using a simpler type.
155        if self.prefix.iter().all(|x| x.is_none()) {
156            // We can't do this if is_consecutive is set and the unit would get
157            // split on the target. Currently, this is only relevant for i128
158            // registers.
159            if rest_count == 1 && (!self.rest.is_consecutive || self.rest.unit != Reg::i128()) {
160                return rest_ll_unit;
161            }
162
163            return cx.type_array(rest_ll_unit, rest_count);
164        }
165
166        // Generate a struct type with the prefix and the "rest" arguments.
167        let prefix_args =
168            self.prefix.iter().flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx)));
169        let rest_args = (0..rest_count).map(|_| rest_ll_unit);
170        let args: Vec<_> = prefix_args.chain(rest_args).collect();
171        cx.type_struct(&args, false)
172    }
173}
174
175trait ArgAbiExt<'ll, 'tcx> {
176    fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
177    fn store(
178        &self,
179        bx: &mut Builder<'_, 'll, 'tcx>,
180        val: &'ll Value,
181        dst: PlaceRef<'tcx, &'ll Value>,
182    );
183    fn store_fn_arg(
184        &self,
185        bx: &mut Builder<'_, 'll, 'tcx>,
186        idx: &mut usize,
187        dst: PlaceRef<'tcx, &'ll Value>,
188    );
189}
190
191impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
192    /// Gets the LLVM type for a place of the original Rust type of
193    /// this argument/return, i.e., the result of `type_of::type_of`.
194    fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
195        self.layout.llvm_type(cx)
196    }
197
198    /// Stores a direct/indirect value described by this ArgAbi into a
199    /// place for the original Rust type of this argument/return.
200    /// Can be used for both storing formal arguments into Rust variables
201    /// or results of call/invoke instructions into their destinations.
202    fn store(
203        &self,
204        bx: &mut Builder<'_, 'll, 'tcx>,
205        val: &'ll Value,
206        dst: PlaceRef<'tcx, &'ll Value>,
207    ) {
208        match &self.mode {
209            PassMode::Ignore => {}
210            // Sized indirect arguments
211            PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
212                let align = attrs.pointee_align.unwrap_or(self.layout.align.abi);
213                OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst);
214            }
215            // Unsized indirect qrguments
216            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
217                bug!("unsized `ArgAbi` must be handled through `store_fn_arg`");
218            }
219            PassMode::Cast { cast, pad_i32: _ } => {
220                // The ABI mandates that the value is passed as a different struct representation.
221                // Spill and reload it from the stack to convert from the ABI representation to
222                // the Rust representation.
223                let scratch_size = cast.size(bx);
224                let scratch_align = cast.align(bx);
225                // Note that the ABI type may be either larger or smaller than the Rust type,
226                // due to the presence or absence of trailing padding. For example:
227                // - On some ABIs, the Rust layout { f64, f32, <f32 padding> } may omit padding
228                //   when passed by value, making it smaller.
229                // - On some ABIs, the Rust layout { u16, u16, u16 } may be padded up to 8 bytes
230                //   when passed by value, making it larger.
231                let copy_bytes =
232                    cmp::min(cast.unaligned_size(bx).bytes(), self.layout.size.bytes());
233                // Allocate some scratch space...
234                let llscratch = bx.alloca(scratch_size, scratch_align);
235                bx.lifetime_start(llscratch, scratch_size);
236                // ...store the value...
237                bx.store(val, llscratch, scratch_align);
238                // ... and then memcpy it to the intended destination.
239                bx.memcpy(
240                    dst.val.llval,
241                    self.layout.align.abi,
242                    llscratch,
243                    scratch_align,
244                    bx.const_usize(copy_bytes),
245                    MemFlags::empty(),
246                );
247                bx.lifetime_end(llscratch, scratch_size);
248            }
249            _ => {
250                OperandRef::from_immediate_or_packed_pair(bx, val, self.layout).val.store(bx, dst);
251            }
252        }
253    }
254
255    fn store_fn_arg(
256        &self,
257        bx: &mut Builder<'_, 'll, 'tcx>,
258        idx: &mut usize,
259        dst: PlaceRef<'tcx, &'ll Value>,
260    ) {
261        let mut next = || {
262            let val = llvm::get_param(bx.llfn(), *idx as c_uint);
263            *idx += 1;
264            val
265        };
266        match self.mode {
267            PassMode::Ignore => {}
268            PassMode::Pair(..) => {
269                OperandValue::Pair(next(), next()).store(bx, dst);
270            }
271            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
272                let place_val = PlaceValue {
273                    llval: next(),
274                    llextra: Some(next()),
275                    align: self.layout.align.abi,
276                };
277                OperandValue::Ref(place_val).store(bx, dst);
278            }
279            PassMode::Direct(_)
280            | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ }
281            | PassMode::Cast { .. } => {
282                let next_arg = next();
283                self.store(bx, next_arg, dst);
284            }
285        }
286    }
287}
288
289impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
290    fn store_fn_arg(
291        &mut self,
292        arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
293        idx: &mut usize,
294        dst: PlaceRef<'tcx, Self::Value>,
295    ) {
296        arg_abi.store_fn_arg(self, idx, dst)
297    }
298    fn store_arg(
299        &mut self,
300        arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
301        val: &'ll Value,
302        dst: PlaceRef<'tcx, &'ll Value>,
303    ) {
304        arg_abi.store(self, val, dst)
305    }
306    fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
307        arg_abi.memory_ty(self)
308    }
309}
310
311pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
312    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
313    fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
314    fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv;
315
316    /// Apply attributes to a function declaration/definition.
317    fn apply_attrs_llfn(
318        &self,
319        cx: &CodegenCx<'ll, 'tcx>,
320        llfn: &'ll Value,
321        instance: Option<ty::Instance<'tcx>>,
322    );
323
324    /// Apply attributes to a function call.
325    fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value);
326}
327
328impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
329    fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
330        // Ignore "extra" args from the call site for C variadic functions.
331        // Only the "fixed" args are part of the LLVM function signature.
332        let args =
333            if self.c_variadic { &self.args[..self.fixed_count as usize] } else { &self.args };
334
335        // This capacity calculation is approximate.
336        let mut llargument_tys = Vec::with_capacity(
337            self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 },
338        );
339
340        let llreturn_ty = match &self.ret.mode {
341            PassMode::Ignore => cx.type_void(),
342            PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
343            PassMode::Cast { cast, pad_i32: _ } => cast.llvm_type(cx),
344            PassMode::Indirect { .. } => {
345                llargument_tys.push(cx.type_ptr());
346                cx.type_void()
347            }
348        };
349
350        for arg in args {
351            // Note that the exact number of arguments pushed here is carefully synchronized with
352            // code all over the place, both in the codegen_llvm and codegen_ssa crates. That's how
353            // other code then knows which LLVM argument(s) correspond to the n-th Rust argument.
354            let llarg_ty = match &arg.mode {
355                PassMode::Ignore => continue,
356                PassMode::Direct(_) => {
357                    // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges),
358                    // and for Scalar ABIs the LLVM type is fully determined by `layout.abi`,
359                    // guaranteeing that we generate ABI-compatible LLVM IR.
360                    arg.layout.immediate_llvm_type(cx)
361                }
362                PassMode::Pair(..) => {
363                    // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges),
364                    // so for ScalarPair we can easily be sure that we are generating ABI-compatible
365                    // LLVM IR.
366                    llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true));
367                    llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true));
368                    continue;
369                }
370                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
371                    // Construct the type of a (wide) pointer to `ty`, and pass its two fields.
372                    // Any two ABI-compatible unsized types have the same metadata type and
373                    // moreover the same metadata value leads to the same dynamic size and
374                    // alignment, so this respects ABI compatibility.
375                    let ptr_ty = Ty::new_mut_ptr(cx.tcx, arg.layout.ty);
376                    let ptr_layout = cx.layout_of(ptr_ty);
377                    llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true));
378                    llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true));
379                    continue;
380                }
381                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(),
382                PassMode::Cast { cast, pad_i32 } => {
383                    // add padding
384                    if *pad_i32 {
385                        llargument_tys.push(Reg::i32().llvm_type(cx));
386                    }
387                    // Compute the LLVM type we use for this function from the cast type.
388                    // We assume here that ABI-compatible Rust types have the same cast type.
389                    cast.llvm_type(cx)
390                }
391            };
392            llargument_tys.push(llarg_ty);
393        }
394
395        if self.c_variadic {
396            cx.type_variadic_func(&llargument_tys, llreturn_ty)
397        } else {
398            cx.type_func(&llargument_tys, llreturn_ty)
399        }
400    }
401
402    fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
403        cx.type_ptr_ext(cx.data_layout().instruction_address_space)
404    }
405
406    fn llvm_cconv(&self, cx: &CodegenCx<'ll, 'tcx>) -> llvm::CallConv {
407        llvm::CallConv::from_conv(self.conv, cx.tcx.sess.target.arch.borrow())
408    }
409
410    fn apply_attrs_llfn(
411        &self,
412        cx: &CodegenCx<'ll, 'tcx>,
413        llfn: &'ll Value,
414        instance: Option<ty::Instance<'tcx>>,
415    ) {
416        let mut func_attrs = SmallVec::<[_; 3]>::new();
417        if self.ret.layout.is_uninhabited() {
418            func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx));
419        }
420        if !self.can_unwind {
421            func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx));
422        }
423        if let Conv::RiscvInterrupt { kind } = self.conv {
424            func_attrs.push(llvm::CreateAttrStringValue(cx.llcx, "interrupt", kind.as_str()));
425        }
426        if let Conv::CCmseNonSecureEntry = self.conv {
427            func_attrs.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry"))
428        }
429        attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs });
430
431        let mut i = 0;
432        let mut apply = |attrs: &ArgAttributes| {
433            attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn);
434            i += 1;
435            i - 1
436        };
437
438        let apply_range_attr = |idx: AttributePlace, scalar: rustc_abi::Scalar| {
439            if cx.sess().opts.optimize != config::OptLevel::No
440                && llvm_util::get_version() >= (19, 0, 0)
441                && matches!(scalar.primitive(), Primitive::Int(..))
442                // If the value is a boolean, the range is 0..2 and that ultimately
443                // become 0..0 when the type becomes i1, which would be rejected
444                // by the LLVM verifier.
445                && !scalar.is_bool()
446                // LLVM also rejects full range.
447                && !scalar.is_always_valid(cx)
448            {
449                attributes::apply_to_llfn(
450                    llfn,
451                    idx,
452                    &[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))],
453                );
454            }
455        };
456
457        match &self.ret.mode {
458            PassMode::Direct(attrs) => {
459                attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
460                if let BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr {
461                    apply_range_attr(llvm::AttributePlace::ReturnValue, scalar);
462                }
463            }
464            PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
465                assert!(!on_stack);
466                let i = apply(attrs);
467                let sret = llvm::CreateStructRetAttr(
468                    cx.llcx,
469                    cx.type_array(cx.type_i8(), self.ret.layout.size.bytes()),
470                );
471                attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]);
472                if cx.sess().opts.optimize != config::OptLevel::No {
473                    attributes::apply_to_llfn(
474                        llfn,
475                        llvm::AttributePlace::Argument(i),
476                        &[
477                            llvm::AttributeKind::Writable.create_attr(cx.llcx),
478                            llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx),
479                        ],
480                    );
481                }
482            }
483            PassMode::Cast { cast, pad_i32: _ } => {
484                cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
485            }
486            _ => {}
487        }
488        for arg in self.args.iter() {
489            match &arg.mode {
490                PassMode::Ignore => {}
491                PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => {
492                    let i = apply(attrs);
493                    let byval = llvm::CreateByValAttr(
494                        cx.llcx,
495                        cx.type_array(cx.type_i8(), arg.layout.size.bytes()),
496                    );
497                    attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
498                }
499                PassMode::Direct(attrs) => {
500                    let i = apply(attrs);
501                    if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
502                        apply_range_attr(llvm::AttributePlace::Argument(i), scalar);
503                    }
504                }
505                PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
506                    apply(attrs);
507                }
508                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
509                    assert!(!on_stack);
510                    apply(attrs);
511                    apply(meta_attrs);
512                }
513                PassMode::Pair(a, b) => {
514                    let i = apply(a);
515                    let ii = apply(b);
516                    if let BackendRepr::ScalarPair(scalar_a, scalar_b) = arg.layout.backend_repr {
517                        apply_range_attr(llvm::AttributePlace::Argument(i), scalar_a);
518                        apply_range_attr(llvm::AttributePlace::Argument(ii), scalar_b);
519                    }
520                }
521                PassMode::Cast { cast, pad_i32 } => {
522                    if *pad_i32 {
523                        apply(&ArgAttributes::new());
524                    }
525                    apply(&cast.attrs);
526                }
527            }
528        }
529
530        // If the declaration has an associated instance, compute extra attributes based on that.
531        if let Some(instance) = instance {
532            llfn_attrs_from_instance(cx, llfn, instance);
533        }
534    }
535
536    fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) {
537        let mut func_attrs = SmallVec::<[_; 2]>::new();
538        if self.ret.layout.is_uninhabited() {
539            func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx));
540        }
541        if !self.can_unwind {
542            func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(bx.cx.llcx));
543        }
544        attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &{ func_attrs });
545
546        let mut i = 0;
547        let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| {
548            attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite);
549            i += 1;
550            i - 1
551        };
552        match &self.ret.mode {
553            PassMode::Direct(attrs) => {
554                attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite);
555            }
556            PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
557                assert!(!on_stack);
558                let i = apply(bx.cx, attrs);
559                let sret = llvm::CreateStructRetAttr(
560                    bx.cx.llcx,
561                    bx.cx.type_array(bx.cx.type_i8(), self.ret.layout.size.bytes()),
562                );
563                attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]);
564            }
565            PassMode::Cast { cast, pad_i32: _ } => {
566                cast.attrs.apply_attrs_to_callsite(
567                    llvm::AttributePlace::ReturnValue,
568                    bx.cx,
569                    callsite,
570                );
571            }
572            _ => {}
573        }
574        if bx.cx.sess().opts.optimize != config::OptLevel::No
575                && llvm_util::get_version() < (19, 0, 0)
576                && let BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr
577                && matches!(scalar.primitive(), Primitive::Int(..))
578                // If the value is a boolean, the range is 0..2 and that ultimately
579                // become 0..0 when the type becomes i1, which would be rejected
580                // by the LLVM verifier.
581                && !scalar.is_bool()
582                // LLVM also rejects full range.
583                && !scalar.is_always_valid(bx)
584        {
585            bx.range_metadata(callsite, scalar.valid_range(bx));
586        }
587        for arg in self.args.iter() {
588            match &arg.mode {
589                PassMode::Ignore => {}
590                PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => {
591                    let i = apply(bx.cx, attrs);
592                    let byval = llvm::CreateByValAttr(
593                        bx.cx.llcx,
594                        bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()),
595                    );
596                    attributes::apply_to_callsite(
597                        callsite,
598                        llvm::AttributePlace::Argument(i),
599                        &[byval],
600                    );
601                }
602                PassMode::Direct(attrs)
603                | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
604                    apply(bx.cx, attrs);
605                }
606                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack: _ } => {
607                    apply(bx.cx, attrs);
608                    apply(bx.cx, meta_attrs);
609                }
610                PassMode::Pair(a, b) => {
611                    apply(bx.cx, a);
612                    apply(bx.cx, b);
613                }
614                PassMode::Cast { cast, pad_i32 } => {
615                    if *pad_i32 {
616                        apply(bx.cx, &ArgAttributes::new());
617                    }
618                    apply(bx.cx, &cast.attrs);
619                }
620            }
621        }
622
623        let cconv = self.llvm_cconv(&bx.cx);
624        if cconv != llvm::CCallConv {
625            llvm::SetInstructionCallConv(callsite, cconv);
626        }
627
628        if self.conv == Conv::CCmseNonSecureCall {
629            // This will probably get ignored on all targets but those supporting the TrustZone-M
630            // extension (thumbv8m targets).
631            let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call");
632            attributes::apply_to_callsite(
633                callsite,
634                llvm::AttributePlace::Function,
635                &[cmse_nonsecure_call],
636            );
637        }
638
639        // Some intrinsics require that an elementtype attribute (with the pointee type of a
640        // pointer argument) is added to the callsite.
641        let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) };
642        if element_type_index >= 0 {
643            let arg_ty = self.args[element_type_index as usize].layout.ty;
644            let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument");
645            let element_type_attr = unsafe {
646                llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx))
647            };
648            attributes::apply_to_callsite(
649                callsite,
650                llvm::AttributePlace::Argument(element_type_index as u32),
651                &[element_type_attr],
652            );
653        }
654    }
655}
656
657impl AbiBuilderMethods for Builder<'_, '_, '_> {
658    fn get_param(&mut self, index: usize) -> Self::Value {
659        llvm::get_param(self.llfn(), index as c_uint)
660    }
661}
662
663impl llvm::CallConv {
664    pub(crate) fn from_conv(conv: Conv, arch: &str) -> Self {
665        match conv {
666            Conv::C
667            | Conv::Rust
668            | Conv::CCmseNonSecureCall
669            | Conv::CCmseNonSecureEntry
670            | Conv::RiscvInterrupt { .. } => llvm::CCallConv,
671            Conv::Cold => llvm::ColdCallConv,
672            Conv::PreserveMost => llvm::PreserveMost,
673            Conv::PreserveAll => llvm::PreserveAll,
674            Conv::GpuKernel => {
675                if arch == "amdgpu" {
676                    llvm::AmdgpuKernel
677                } else if arch == "nvptx64" {
678                    llvm::PtxKernel
679                } else {
680                    panic!("Architecture {arch} does not support GpuKernel calling convention");
681                }
682            }
683            Conv::AvrInterrupt => llvm::AvrInterrupt,
684            Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt,
685            Conv::ArmAapcs => llvm::ArmAapcsCallConv,
686            Conv::Msp430Intr => llvm::Msp430Intr,
687            Conv::X86Fastcall => llvm::X86FastcallCallConv,
688            Conv::X86Intr => llvm::X86_Intr,
689            Conv::X86Stdcall => llvm::X86StdcallCallConv,
690            Conv::X86ThisCall => llvm::X86_ThisCall,
691            Conv::X86VectorCall => llvm::X86_VectorCall,
692            Conv::X86_64SysV => llvm::X86_64_SysV,
693            Conv::X86_64Win64 => llvm::X86_64_Win64,
694        }
695    }
696}
rustc_codegen_llvm/abi.rs

rustc_codegen_llvm/
abi.rs
