rustc_codegen_ssa/mir/
naked_asm.rs

1use rustc_abi::{BackendRepr, Float, Integer, Primitive, RegKind};
2use rustc_hir::attrs::{InstructionSetAttr, Linkage};
3use rustc_middle::mir::mono::{MonoItemData, Visibility};
4use rustc_middle::mir::{InlineAsmOperand, START_BLOCK};
5use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
6use rustc_middle::ty::{Instance, Ty, TyCtxt, TypeVisitableExt};
7use rustc_middle::{bug, ty};
8use rustc_span::sym;
9use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
10use rustc_target::spec::{Arch, BinaryFormat};
11
12use crate::common;
13use crate::mir::AsmCodegenMethods;
14use crate::traits::GlobalAsmOperandRef;
15
16pub fn codegen_naked_asm<
17    'a,
18    'tcx,
19    Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
20        + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>
21        + AsmCodegenMethods<'tcx>,
22>(
23    cx: &'a mut Cx,
24    instance: Instance<'tcx>,
25    item_data: MonoItemData,
26) {
27    assert!(!instance.args.has_infer());
28    let mir = cx.tcx().instance_mir(instance.def);
29
30    let rustc_middle::mir::TerminatorKind::InlineAsm {
31        asm_macro: _,
32        template,
33        ref operands,
34        options,
35        line_spans,
36        targets: _,
37        unwind: _,
38    } = mir.basic_blocks[START_BLOCK].terminator().kind
39    else {
40        bug!("#[naked] functions should always terminate with an asm! block")
41    };
42
43    let operands: Vec<_> =
44        operands.iter().map(|op| inline_to_global_operand::<Cx>(cx, instance, op)).collect();
45
46    let name = cx.mangled_name(instance);
47    let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
48    let (begin, end) = prefix_and_suffix(cx.tcx(), instance, &name, item_data, fn_abi);
49
50    let mut template_vec = Vec::new();
51    template_vec.push(rustc_ast::ast::InlineAsmTemplatePiece::String(begin.into()));
52    template_vec.extend(template.iter().cloned());
53    template_vec.push(rustc_ast::ast::InlineAsmTemplatePiece::String(end.into()));
54
55    cx.codegen_global_asm(&template_vec, &operands, options, line_spans);
56}
57
58fn inline_to_global_operand<'a, 'tcx, Cx: LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>>(
59    cx: &'a Cx,
60    instance: Instance<'tcx>,
61    op: &InlineAsmOperand<'tcx>,
62) -> GlobalAsmOperandRef<'tcx> {
63    match op {
64        InlineAsmOperand::Const { value } => {
65            let const_value = instance
66                .instantiate_mir_and_normalize_erasing_regions(
67                    cx.tcx(),
68                    cx.typing_env(),
69                    ty::EarlyBinder::bind(value.const_),
70                )
71                .eval(cx.tcx(), cx.typing_env(), value.span)
72                .expect("erroneous constant missed by mono item collection");
73
74            let mono_type = instance.instantiate_mir_and_normalize_erasing_regions(
75                cx.tcx(),
76                cx.typing_env(),
77                ty::EarlyBinder::bind(value.ty()),
78            );
79
80            let string = common::asm_const_to_str(
81                cx.tcx(),
82                value.span,
83                const_value,
84                cx.layout_of(mono_type),
85            );
86
87            GlobalAsmOperandRef::Const { string }
88        }
89        InlineAsmOperand::SymFn { value } => {
90            let mono_type = instance.instantiate_mir_and_normalize_erasing_regions(
91                cx.tcx(),
92                cx.typing_env(),
93                ty::EarlyBinder::bind(value.ty()),
94            );
95
96            let instance = match mono_type.kind() {
97                &ty::FnDef(def_id, args) => {
98                    Instance::expect_resolve(cx.tcx(), cx.typing_env(), def_id, args, value.span)
99                }
100                _ => bug!("asm sym is not a function"),
101            };
102
103            GlobalAsmOperandRef::SymFn { instance }
104        }
105        InlineAsmOperand::SymStatic { def_id } => {
106            GlobalAsmOperandRef::SymStatic { def_id: *def_id }
107        }
108        InlineAsmOperand::In { .. }
109        | InlineAsmOperand::Out { .. }
110        | InlineAsmOperand::InOut { .. }
111        | InlineAsmOperand::Label { .. } => {
112            bug!("invalid operand type for naked_asm!")
113        }
114    }
115}
116
117fn prefix_and_suffix<'tcx>(
118    tcx: TyCtxt<'tcx>,
119    instance: Instance<'tcx>,
120    asm_name: &str,
121    item_data: MonoItemData,
122    fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
123) -> (String, String) {
124    use std::fmt::Write;
125
126    let asm_binary_format = &tcx.sess.target.binary_format;
127
128    let is_arm = tcx.sess.target.arch == Arch::Arm;
129    let is_thumb = tcx.sess.unstable_target_features.contains(&sym::thumb_mode);
130
131    let attrs = tcx.codegen_instance_attrs(instance.def);
132    let link_section = attrs.link_section.map(|symbol| symbol.as_str().to_string());
133
134    // If no alignment is specified, an alignment of 4 bytes is used.
135    let align_bytes = attrs.alignment.map(|a| a.bytes()).unwrap_or(4);
136
137    // In particular, `.arm` can also be written `.code 32` and `.thumb` as `.code 16`.
138    let (arch_prefix, arch_suffix) = if is_arm {
139        (
140            match attrs.instruction_set {
141                None => match is_thumb {
142                    true => ".thumb\n.thumb_func",
143                    false => ".arm",
144                },
145                Some(InstructionSetAttr::ArmT32) => ".thumb\n.thumb_func",
146                Some(InstructionSetAttr::ArmA32) => ".arm",
147            },
148            match is_thumb {
149                true => ".thumb",
150                false => ".arm",
151            },
152        )
153    } else {
154        ("", "")
155    };
156
157    let emit_fatal = |msg| tcx.dcx().span_fatal(tcx.def_span(instance.def_id()), msg);
158
159    // see https://godbolt.org/z/cPK4sxKor.
160    let write_linkage = |w: &mut String| -> std::fmt::Result {
161        match item_data.linkage {
162            Linkage::External => {
163                writeln!(w, ".globl {asm_name}")?;
164            }
165            Linkage::LinkOnceAny | Linkage::LinkOnceODR | Linkage::WeakAny | Linkage::WeakODR => {
166                match asm_binary_format {
167                    BinaryFormat::Elf | BinaryFormat::Coff | BinaryFormat::Wasm => {
168                        writeln!(w, ".weak {asm_name}")?;
169                    }
170                    BinaryFormat::Xcoff => {
171                        // FIXME: there is currently no way of defining a weak symbol in inline assembly
172                        // for AIX. See https://github.com/llvm/llvm-project/issues/130269
173                        emit_fatal(
174                            "cannot create weak symbols from inline assembly for this target",
175                        )
176                    }
177                    BinaryFormat::MachO => {
178                        writeln!(w, ".globl {asm_name}")?;
179                        writeln!(w, ".weak_definition {asm_name}")?;
180                    }
181                }
182            }
183            Linkage::Internal => {
184                // LTO can fail when internal linkage is used.
185                emit_fatal("naked functions may not have internal linkage")
186            }
187            Linkage::Common => emit_fatal("Functions may not have common linkage"),
188            Linkage::AvailableExternally => {
189                // this would make the function equal an extern definition
190                emit_fatal("Functions may not have available_externally linkage")
191            }
192            Linkage::ExternalWeak => {
193                // FIXME: actually this causes a SIGILL in LLVM
194                emit_fatal("Functions may not have external weak linkage")
195            }
196        }
197
198        Ok(())
199    };
200
201    let mut begin = String::new();
202    let mut end = String::new();
203    match asm_binary_format {
204        BinaryFormat::Elf => {
205            let section = link_section.unwrap_or_else(|| format!(".text.{asm_name}"));
206
207            let progbits = match is_arm {
208                true => "%progbits",
209                false => "@progbits",
210            };
211
212            let function = match is_arm {
213                true => "%function",
214                false => "@function",
215            };
216
217            writeln!(begin, ".pushsection {section},\"ax\", {progbits}").unwrap();
218            writeln!(begin, ".balign {align_bytes}").unwrap();
219            write_linkage(&mut begin).unwrap();
220            match item_data.visibility {
221                Visibility::Default => {}
222                Visibility::Protected => writeln!(begin, ".protected {asm_name}").unwrap(),
223                Visibility::Hidden => writeln!(begin, ".hidden {asm_name}").unwrap(),
224            }
225            writeln!(begin, ".type {asm_name}, {function}").unwrap();
226            if !arch_prefix.is_empty() {
227                writeln!(begin, "{}", arch_prefix).unwrap();
228            }
229            writeln!(begin, "{asm_name}:").unwrap();
230
231            writeln!(end).unwrap();
232            // emit a label starting with `func_end` for `cargo asm` and other tooling that might
233            // pattern match on assembly generated by LLVM.
234            writeln!(end, ".Lfunc_end_{asm_name}:").unwrap();
235            writeln!(end, ".size {asm_name}, . - {asm_name}").unwrap();
236            writeln!(end, ".popsection").unwrap();
237            if !arch_suffix.is_empty() {
238                writeln!(end, "{}", arch_suffix).unwrap();
239            }
240        }
241        BinaryFormat::MachO => {
242            let section = link_section.unwrap_or_else(|| "__TEXT,__text".to_string());
243            writeln!(begin, ".pushsection {},regular,pure_instructions", section).unwrap();
244            writeln!(begin, ".balign {align_bytes}").unwrap();
245            write_linkage(&mut begin).unwrap();
246            match item_data.visibility {
247                Visibility::Default | Visibility::Protected => {}
248                Visibility::Hidden => writeln!(begin, ".private_extern {asm_name}").unwrap(),
249            }
250            writeln!(begin, "{asm_name}:").unwrap();
251
252            writeln!(end).unwrap();
253            writeln!(end, ".Lfunc_end_{asm_name}:").unwrap();
254            writeln!(end, ".popsection").unwrap();
255            if !arch_suffix.is_empty() {
256                writeln!(end, "{}", arch_suffix).unwrap();
257            }
258        }
259        BinaryFormat::Coff => {
260            let section = link_section.unwrap_or_else(|| format!(".text.{asm_name}"));
261            writeln!(begin, ".pushsection {},\"xr\"", section).unwrap();
262            writeln!(begin, ".balign {align_bytes}").unwrap();
263            write_linkage(&mut begin).unwrap();
264            writeln!(begin, ".def {asm_name}").unwrap();
265            writeln!(begin, ".scl 2").unwrap();
266            writeln!(begin, ".type 32").unwrap();
267            writeln!(begin, ".endef").unwrap();
268            writeln!(begin, "{asm_name}:").unwrap();
269
270            writeln!(end).unwrap();
271            writeln!(end, ".Lfunc_end_{asm_name}:").unwrap();
272            writeln!(end, ".popsection").unwrap();
273            if !arch_suffix.is_empty() {
274                writeln!(end, "{}", arch_suffix).unwrap();
275            }
276        }
277        BinaryFormat::Wasm => {
278            let section = link_section.unwrap_or_else(|| format!(".text.{asm_name}"));
279
280            writeln!(begin, ".section {section},\"\",@").unwrap();
281            // wasm functions cannot be aligned, so skip
282            write_linkage(&mut begin).unwrap();
283            if let Visibility::Hidden = item_data.visibility {
284                writeln!(begin, ".hidden {asm_name}").unwrap();
285            }
286            writeln!(begin, ".type {asm_name}, @function").unwrap();
287            if !arch_prefix.is_empty() {
288                writeln!(begin, "{}", arch_prefix).unwrap();
289            }
290            writeln!(begin, "{asm_name}:").unwrap();
291            writeln!(begin, ".functype {asm_name} {}", wasm_functype(tcx, fn_abi)).unwrap();
292
293            writeln!(end).unwrap();
294            // .size is ignored for function symbols, so we can skip it
295            writeln!(end, "end_function").unwrap();
296            writeln!(end, ".Lfunc_end_{asm_name}:").unwrap();
297        }
298        BinaryFormat::Xcoff => {
299            // the LLVM XCOFFAsmParser is extremely incomplete and does not implement many of the
300            // documented directives.
301            //
302            // - https://github.com/llvm/llvm-project/blob/1b25c0c4da968fe78921ce77736e5baef4db75e3/llvm/lib/MC/MCParser/XCOFFAsmParser.cpp
303            // - https://www.ibm.com/docs/en/ssw_aix_71/assembler/assembler_pdf.pdf
304            //
305            // Consequently, we try our best here but cannot do as good a job as for other binary
306            // formats.
307
308            // FIXME: start a section. `.csect` is not currently implemented in LLVM
309
310            // fun fact: according to the assembler documentation, .align takes an exponent,
311            // but LLVM only accepts powers of 2 (but does emit the exponent)
312            // so when we hand `.align 32` to LLVM, the assembly output will contain `.align 5`
313            writeln!(begin, ".align {}", align_bytes).unwrap();
314
315            write_linkage(&mut begin).unwrap();
316            if let Visibility::Hidden = item_data.visibility {
317                // FIXME apparently `.globl {asm_name}, hidden` is valid
318                // but due to limitations with `.weak` (see above) we can't really use that in general yet
319            }
320            writeln!(begin, "{asm_name}:").unwrap();
321
322            writeln!(end).unwrap();
323            // FIXME: end the section?
324        }
325    }
326
327    (begin, end)
328}
329
330/// The webassembly type signature for the given function.
331///
332/// Used by the `.functype` directive on wasm targets.
333fn wasm_functype<'tcx>(tcx: TyCtxt<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> String {
334    let mut signature = String::with_capacity(64);
335
336    let ptr_type = match tcx.data_layout.pointer_size().bits() {
337        32 => "i32",
338        64 => "i64",
339        other => bug!("wasm pointer size cannot be {other} bits"),
340    };
341
342    let hidden_return = matches!(fn_abi.ret.mode, PassMode::Indirect { .. });
343
344    signature.push('(');
345
346    if hidden_return {
347        signature.push_str(ptr_type);
348        if !fn_abi.args.is_empty() {
349            signature.push_str(", ");
350        }
351    }
352
353    let mut it = fn_abi.args.iter().peekable();
354    while let Some(arg_abi) = it.next() {
355        wasm_type(&mut signature, arg_abi, ptr_type);
356        if it.peek().is_some() {
357            signature.push_str(", ");
358        }
359    }
360
361    signature.push_str(") -> (");
362
363    if !hidden_return {
364        wasm_type(&mut signature, &fn_abi.ret, ptr_type);
365    }
366
367    signature.push(')');
368
369    signature
370}
371
372fn wasm_type<'tcx>(signature: &mut String, arg_abi: &ArgAbi<'_, Ty<'tcx>>, ptr_type: &'static str) {
373    match arg_abi.mode {
374        PassMode::Ignore => { /* do nothing */ }
375        PassMode::Direct(_) => {
376            let direct_type = match arg_abi.layout.backend_repr {
377                BackendRepr::Scalar(scalar) => wasm_primitive(scalar.primitive(), ptr_type),
378                BackendRepr::SimdVector { .. } => "v128",
379                other => unreachable!("unexpected BackendRepr: {:?}", other),
380            };
381
382            signature.push_str(direct_type);
383        }
384        PassMode::Pair(_, _) => match arg_abi.layout.backend_repr {
385            BackendRepr::ScalarPair(a, b) => {
386                signature.push_str(wasm_primitive(a.primitive(), ptr_type));
387                signature.push_str(", ");
388                signature.push_str(wasm_primitive(b.primitive(), ptr_type));
389            }
390            other => unreachable!("{other:?}"),
391        },
392        PassMode::Cast { pad_i32, ref cast } => {
393            // For wasm, Cast is used for single-field primitive wrappers like `struct Wrapper(i64);`
394            assert!(!pad_i32, "not currently used by wasm calling convention");
395            assert!(cast.prefix[0].is_none(), "no prefix");
396            assert_eq!(cast.rest.total, arg_abi.layout.size, "single item");
397
398            let wrapped_wasm_type = match cast.rest.unit.kind {
399                RegKind::Integer => match cast.rest.unit.size.bytes() {
400                    ..=4 => "i32",
401                    ..=8 => "i64",
402                    _ => ptr_type,
403                },
404                RegKind::Float => match cast.rest.unit.size.bytes() {
405                    ..=4 => "f32",
406                    ..=8 => "f64",
407                    _ => ptr_type,
408                },
409                RegKind::Vector => "v128",
410            };
411
412            signature.push_str(wrapped_wasm_type);
413        }
414        PassMode::Indirect { .. } => signature.push_str(ptr_type),
415    }
416}
417
418fn wasm_primitive(primitive: Primitive, ptr_type: &'static str) -> &'static str {
419    match primitive {
420        Primitive::Int(integer, _) => match integer {
421            Integer::I8 | Integer::I16 | Integer::I32 => "i32",
422            Integer::I64 => "i64",
423            Integer::I128 => "i64, i64",
424        },
425        Primitive::Float(float) => match float {
426            Float::F16 | Float::F32 => "f32",
427            Float::F64 => "f64",
428            Float::F128 => "i64, i64",
429        },
430        Primitive::Pointer(_) => ptr_type,
431    }
432}
rustc_codegen_ssa/mir/naked_asm.rs

rustc_codegen_ssa/mir/
naked_asm.rs
