rustc_codegen_llvm/
asm.rs

1use std::assert_matches;
2use std::fmt::Write;
3
4use rustc_abi::{BackendRepr, Endian, Float, Integer, Primitive, Scalar, Size};
5use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
6use rustc_codegen_ssa::mir::operand::OperandValue;
7use rustc_codegen_ssa::traits::*;
8use rustc_data_structures::fx::FxHashMap;
9use rustc_middle::mir::interpret::{PointerArithmetic, Scalar as ConstScalar};
10use rustc_middle::ty::Instance;
11use rustc_middle::ty::layout::TyAndLayout;
12use rustc_middle::{bug, span_bug};
13use rustc_span::{Pos, Span, Symbol, sym};
14use rustc_target::asm::*;
15use rustc_target::spec::HasTargetSpec;
16use smallvec::SmallVec;
17use tracing::debug;
18
19use crate::attributes;
20use crate::builder::Builder;
21use crate::common::Funclet;
22use crate::context::CodegenCx;
23use crate::llvm::{self, ToLlvmBool, Type, Value};
24use crate::type_of::LayoutLlvmExt;
25
26impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
27    fn codegen_inline_asm(
28        &mut self,
29        template: &[InlineAsmTemplatePiece],
30        operands: &[InlineAsmOperandRef<'tcx, Self>],
31        options: InlineAsmOptions,
32        line_spans: &[Span],
33        instance: Instance<'_>,
34        dest: Option<Self::BasicBlock>,
35        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
36    ) {
37        let asm_arch = self.tcx.sess.asm_arch.unwrap();
38
39        // Collect the types of output operands
40        let mut constraints = ::alloc::vec::Vec::new()vec![];
41        let mut clobbers = ::alloc::vec::Vec::new()vec![];
42        let mut output_types = ::alloc::vec::Vec::new()vec![];
43        let mut op_idx = FxHashMap::default();
44        let mut clobbered_x87 = false;
45        for (idx, op) in operands.iter().enumerate() {
46            match *op {
47                InlineAsmOperandRef::Out { reg, late, place } => {
48                    let is_target_supported = |reg_class: InlineAsmRegClass| {
49                        for &(_, feature) in reg_class.supported_types(asm_arch, true).as_ref() {
50                            if let Some(feature) = feature {
51                                if self
52                                    .tcx
53                                    .asm_target_features(instance.def_id())
54                                    .contains(&feature)
55                                {
56                                    return true;
57                                }
58                            } else {
59                                // Register class is unconditionally supported
60                                return true;
61                            }
62                        }
63                        false
64                    };
65
66                    let mut layout = None;
67                    let ty = if let Some(ref place) = place {
68                        layout = Some(&place.layout);
69                        llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)
70                    } else if #[allow(non_exhaustive_omitted_patterns)] match reg.reg_class() {
    InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg |
        X86InlineAsmRegClass::x87_reg) => true,
    _ => false,
}matches!(
71                        reg.reg_class(),
72                        InlineAsmRegClass::X86(
73                            X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
74                        )
75                    ) {
76                        // Special handling for x87/mmx registers: we always
77                        // clobber the whole set if one register is marked as
78                        // clobbered. This is due to the way LLVM handles the
79                        // FP stack in inline assembly.
80                        if !clobbered_x87 {
81                            clobbered_x87 = true;
82                            clobbers.push("~{st}".to_string());
83                            for i in 1..=7 {
84                                clobbers.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("~{{st({0})}}", i))
    })format!("~{{st({})}}", i));
85                            }
86                        }
87                        continue;
88                    } else if !is_target_supported(reg.reg_class())
89                        || reg.reg_class().is_clobber_only(asm_arch, true)
90                    {
91                        // We turn discarded outputs into clobber constraints
92                        // if the target feature needed by the register class is
93                        // disabled. This is necessary otherwise LLVM will try
94                        // to actually allocate a register for the dummy output.
95                        {
    match reg {
        InlineAsmRegOrRegClass::Reg(_) => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "InlineAsmRegOrRegClass::Reg(_)",
                ::core::option::Option::None);
        }
    }
};assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_));
96                        clobbers.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("~{0}", reg_to_llvm(reg, None)))
    })format!("~{}", reg_to_llvm(reg, None)));
97                        continue;
98                    } else {
99                        // If the output is discarded, we don't really care what
100                        // type is used. We're just using this to tell LLVM to
101                        // reserve the register.
102                        dummy_output_type(self.cx, reg.reg_class())
103                    };
104                    output_types.push(ty);
105                    op_idx.insert(idx, constraints.len());
106                    let prefix = if late { "=" } else { "=&" };
107                    constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", prefix,
                reg_to_llvm(reg, layout)))
    })format!("{}{}", prefix, reg_to_llvm(reg, layout)));
108                }
109                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
110                    let layout = if let Some(ref out_place) = out_place {
111                        &out_place.layout
112                    } else {
113                        // LLVM required tied operands to have the same type,
114                        // so we just use the type of the input.
115                        &in_value.layout
116                    };
117                    let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);
118                    output_types.push(ty);
119                    op_idx.insert(idx, constraints.len());
120                    let prefix = if late { "=" } else { "=&" };
121                    constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", prefix,
                reg_to_llvm(reg, Some(layout))))
    })format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
122                }
123                _ => {}
124            }
125        }
126
127        // Collect input operands
128        let mut inputs = ::alloc::vec::Vec::new()vec![];
129        for (idx, op) in operands.iter().enumerate() {
130            match *op {
131                InlineAsmOperandRef::In { reg, value } => {
132                    let llval = llvm_fixup_input(
133                        self,
134                        value.immediate(),
135                        reg.reg_class(),
136                        &value.layout,
137                        instance,
138                    );
139                    inputs.push(llval);
140                    op_idx.insert(idx, constraints.len());
141                    constraints.push(reg_to_llvm(reg, Some(&value.layout)));
142                }
143                InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => {
144                    let value = llvm_fixup_input(
145                        self,
146                        in_value.immediate(),
147                        reg.reg_class(),
148                        &in_value.layout,
149                        instance,
150                    );
151                    inputs.push(value);
152
153                    // In the case of fixed registers, we have the choice of
154                    // either using a tied operand or duplicating the constraint.
155                    // We prefer the latter because it matches the behavior of
156                    // Clang.
157                    if late && #[allow(non_exhaustive_omitted_patterns)] match reg {
    InlineAsmRegOrRegClass::Reg(_) => true,
    _ => false,
}matches!(reg, InlineAsmRegOrRegClass::Reg(_)) {
158                        constraints.push(reg_to_llvm(reg, Some(&in_value.layout)));
159                    } else {
160                        constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}", op_idx[&idx]))
    })format!("{}", op_idx[&idx]));
161                    }
162                }
163                InlineAsmOperandRef::Const { value, ty: _ } => match value {
164                    ConstScalar::Int(_) => (),
165                    ConstScalar::Ptr(ptr, _) => {
166                        let (prov, _) = ptr.prov_and_relative_offset();
167                        let global_alloc = self.tcx.global_alloc(prov.alloc_id());
168                        let value = self.cx.alloc_to_backend(global_alloc, false, None).unwrap();
169                        inputs.push(value);
170                        op_idx.insert(idx, constraints.len());
171                        constraints.push("s".to_string());
172                    }
173                },
174                InlineAsmOperandRef::SymThreadLocalStatic { def_id } => {
175                    inputs.push(self.cx.get_static(def_id));
176                    op_idx.insert(idx, constraints.len());
177                    constraints.push("s".to_string());
178                }
179                _ => {}
180            }
181        }
182
183        // Build the template string
184        let mut labels = ::alloc::vec::Vec::new()vec![];
185        let mut template_str = String::new();
186        for piece in template {
187            match *piece {
188                InlineAsmTemplatePiece::String(ref s) => {
189                    if s.contains('$') {
190                        for c in s.chars() {
191                            if c == '$' {
192                                template_str.push_str("$$");
193                            } else {
194                                template_str.push(c);
195                            }
196                        }
197                    } else {
198                        template_str.push_str(s)
199                    }
200                }
201                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span } => {
202                    match operands[operand_idx] {
203                        InlineAsmOperandRef::In { reg, .. }
204                        | InlineAsmOperandRef::Out { reg, .. }
205                        | InlineAsmOperandRef::InOut { reg, .. } => {
206                            let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
207                            if let Some(modifier) = modifier {
208                                template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:{1}}}",
                op_idx[&operand_idx], modifier))
    })format!(
209                                    "${{{}:{}}}",
210                                    op_idx[&operand_idx], modifier
211                                ));
212                            } else {
213                                template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}}}", op_idx[&operand_idx]))
    })format!("${{{}}}", op_idx[&operand_idx]));
214                            }
215                        }
216                        InlineAsmOperandRef::Const { value, ty } => {
217                            match value {
218                                ConstScalar::Int(int) => {
219                                    // Const operands get injected directly into the template
220                                    let string = rustc_codegen_ssa::common::asm_const_to_str(
221                                        self.tcx,
222                                        span,
223                                        int,
224                                        self.layout_of(ty),
225                                    );
226                                    template_str.push_str(&string);
227                                }
228                                ConstScalar::Ptr(ptr, _) => {
229                                    let (_, offset) = ptr.prov_and_relative_offset();
230
231                                    // Only emit the raw symbol name
232                                    template_str
233                                        .push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:c}}", op_idx[&operand_idx]))
    })format!("${{{}:c}}", op_idx[&operand_idx]));
234
235                                    if offset != Size::ZERO {
236                                        let offset =
237                                            self.sign_extend_to_target_isize(offset.bytes());
238                                        template_str.write_fmt(format_args!("{0:+}", offset))write!(template_str, "{offset:+}").unwrap();
239                                    }
240                                }
241                            }
242                        }
243                        InlineAsmOperandRef::SymThreadLocalStatic { .. } => {
244                            // Only emit the raw symbol name
245                            template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:c}}", op_idx[&operand_idx]))
    })format!("${{{}:c}}", op_idx[&operand_idx]));
246                        }
247                        InlineAsmOperandRef::Label { label } => {
248                            template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:l}}", constraints.len()))
    })format!("${{{}:l}}", constraints.len()));
249                            constraints.push("!i".to_owned());
250                            labels.push(label);
251                        }
252                    }
253                }
254            }
255        }
256
257        constraints.append(&mut clobbers);
258        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
259            match asm_arch {
260                InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC | InlineAsmArch::Arm => {
261                    constraints.push("~{cc}".to_string());
262                }
263                InlineAsmArch::Amdgpu => {}
264                InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
265                    constraints.extend_from_slice(&[
266                        "~{dirflag}".to_string(),
267                        "~{fpsr}".to_string(),
268                        "~{flags}".to_string(),
269                    ]);
270                }
271                InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
272                    constraints.extend_from_slice(&[
273                        "~{fflags}".to_string(),
274                        "~{vtype}".to_string(),
275                        "~{vl}".to_string(),
276                        "~{vxsat}".to_string(),
277                        "~{vxrm}".to_string(),
278                    ]);
279                }
280                InlineAsmArch::Avr => {
281                    constraints.push("~{sreg}".to_string());
282                }
283                InlineAsmArch::Nvptx64 => {}
284                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
285                InlineAsmArch::Hexagon => {}
286                InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
287                    constraints.extend_from_slice(&[
288                        "~{$fcc0}".to_string(),
289                        "~{$fcc1}".to_string(),
290                        "~{$fcc2}".to_string(),
291                        "~{$fcc3}".to_string(),
292                        "~{$fcc4}".to_string(),
293                        "~{$fcc5}".to_string(),
294                        "~{$fcc6}".to_string(),
295                        "~{$fcc7}".to_string(),
296                    ]);
297                }
298                InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
299                InlineAsmArch::S390x => {
300                    constraints.push("~{cc}".to_string());
301                }
302                InlineAsmArch::Sparc | InlineAsmArch::Sparc64 => {
303                    // In LLVM, ~{icc} represents icc and xcc in 64-bit code.
304                    // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/SparcRegisterInfo.td#L64
305                    constraints.push("~{icc}".to_string());
306                    constraints.push("~{fcc0}".to_string());
307                    constraints.push("~{fcc1}".to_string());
308                    constraints.push("~{fcc2}".to_string());
309                    constraints.push("~{fcc3}".to_string());
310                }
311                InlineAsmArch::SpirV => {}
312                InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
313                InlineAsmArch::Xtensa => {}
314                InlineAsmArch::Bpf => {}
315                InlineAsmArch::Msp430 => {
316                    constraints.push("~{sr}".to_string());
317                }
318                InlineAsmArch::M68k => {
319                    constraints.push("~{ccr}".to_string());
320                }
321                InlineAsmArch::CSKY => {
322                    constraints.push("~{psr}".to_string());
323                }
324            }
325        }
326        if !options.contains(InlineAsmOptions::NOMEM) {
327            // This is actually ignored by LLVM, but it's probably best to keep
328            // it just in case. LLVM instead uses the ReadOnly/ReadNone
329            // attributes on the call instruction to optimize.
330            constraints.push("~{memory}".to_string());
331        }
332        let volatile = !options.contains(InlineAsmOptions::PURE);
333        let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
334        let output_type = match &output_types[..] {
335            [] => self.type_void(),
336            [ty] => ty,
337            tys => self.type_struct(tys, false),
338        };
339        let dialect = match asm_arch {
340            InlineAsmArch::X86 | InlineAsmArch::X86_64
341                if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
342            {
343                llvm::AsmDialect::Intel
344            }
345            _ => llvm::AsmDialect::Att,
346        };
347        let result = inline_asm_call(
348            self,
349            &template_str,
350            &constraints.join(","),
351            &inputs,
352            output_type,
353            &labels,
354            volatile,
355            alignstack,
356            dialect,
357            line_spans,
358            options.contains(InlineAsmOptions::MAY_UNWIND),
359            dest,
360            catch_funclet,
361        )
362        .unwrap_or_else(|| ::rustc_middle::util::bug::span_bug_fmt(line_spans[0],
    format_args!("LLVM asm constraint validation failed"))span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
363
364        let mut attrs = SmallVec::<[_; 2]>::new();
365        if options.contains(InlineAsmOptions::PURE) {
366            if options.contains(InlineAsmOptions::NOMEM) {
367                attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx));
368            } else if options.contains(InlineAsmOptions::READONLY) {
369                attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx));
370            }
371            attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
372        } else if options.contains(InlineAsmOptions::NOMEM) {
373            attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
374        } else if options.contains(InlineAsmOptions::READONLY) {
375            attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx));
376        }
377        attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
378
379        // Write results to outputs. We need to do this for all possible control flow.
380        //
381        // Note that `dest` maybe populated with unreachable_block when asm goto with outputs
382        // is used (because we need to codegen callbr which always needs a destination), so
383        // here we use the NORETURN option to determine if `dest` should be used.
384        for block in (if options.contains(InlineAsmOptions::NORETURN) { None } else { Some(dest) })
385            .into_iter()
386            .chain(labels.iter().copied().map(Some))
387        {
388            if let Some(block) = block {
389                self.switch_to_block(block);
390            }
391
392            for (idx, op) in operands.iter().enumerate() {
393                if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
394                | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
395                {
396                    let value = if output_types.len() == 1 {
397                        result
398                    } else {
399                        self.extract_value(result, op_idx[&idx] as u64)
400                    };
401                    let value =
402                        llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);
403                    OperandValue::Immediate(value).store(self, place);
404                }
405            }
406        }
407    }
408}
409
410impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
411    fn codegen_global_asm(
412        &mut self,
413        template: &[InlineAsmTemplatePiece],
414        operands: &[GlobalAsmOperandRef<'tcx>],
415        options: InlineAsmOptions,
416        _line_spans: &[Span],
417    ) {
418        let asm_arch = self.tcx.sess.asm_arch.unwrap();
419
420        // Build the template string
421        let mut template_str = String::new();
422
423        // On X86 platforms there are two assembly syntaxes. Rust uses intel by default,
424        // but AT&T can be specified explicitly.
425        if #[allow(non_exhaustive_omitted_patterns)] match asm_arch {
    InlineAsmArch::X86 | InlineAsmArch::X86_64 => true,
    _ => false,
}matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) {
426            if options.contains(InlineAsmOptions::ATT_SYNTAX) {
427                template_str.push_str(".att_syntax\n")
428            } else {
429                template_str.push_str(".intel_syntax\n")
430            }
431        }
432
433        for piece in template {
434            match *piece {
435                InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
436                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span } => {
437                    use rustc_codegen_ssa::back::symbol_export::escape_symbol_name;
438                    match operands[operand_idx] {
439                        GlobalAsmOperandRef::Const { value, ty } => {
440                            match value {
441                                ConstScalar::Int(int) => {
442                                    // Const operands get injected directly into the
443                                    // template. Note that we don't need to escape $
444                                    // here unlike normal inline assembly.
445                                    let string = rustc_codegen_ssa::common::asm_const_to_str(
446                                        self.tcx,
447                                        span,
448                                        int,
449                                        self.layout_of(ty),
450                                    );
451                                    template_str.push_str(&string);
452                                }
453
454                                ConstScalar::Ptr(ptr, _) => {
455                                    let (prov, offset) = ptr.prov_and_relative_offset();
456                                    let global_alloc = self.tcx.global_alloc(prov.alloc_id());
457                                    let llval =
458                                        self.alloc_to_backend(global_alloc, true, None).unwrap();
459
460                                    self.add_compiler_used_global(llval);
461                                    let symbol = llvm::build_string(|s| unsafe {
462                                        llvm::LLVMRustGetMangledName(llval, s);
463                                    })
464                                    .expect("symbol is not valid UTF-8");
465                                    template_str
466                                        .push_str(&escape_symbol_name(self.tcx, &symbol, span));
467
468                                    if offset != Size::ZERO {
469                                        let offset =
470                                            self.sign_extend_to_target_isize(offset.bytes());
471                                        template_str.write_fmt(format_args!("{0:+}", offset))write!(template_str, "{offset:+}").unwrap();
472                                    }
473                                }
474                            }
475                        }
476                        GlobalAsmOperandRef::SymThreadLocalStatic { def_id } => {
477                            let llval = self
478                                .renamed_statics
479                                .borrow()
480                                .get(&def_id)
481                                .copied()
482                                .unwrap_or_else(|| self.get_static(def_id));
483                            self.add_compiler_used_global(llval);
484                            let symbol = llvm::build_string(|s| unsafe {
485                                llvm::LLVMRustGetMangledName(llval, s);
486                            })
487                            .expect("symbol is not valid UTF-8");
488                            template_str.push_str(&escape_symbol_name(self.tcx, &symbol, span));
489                        }
490                    }
491                }
492            }
493        }
494
495        // Just to play it safe, if intel was used, reset the assembly syntax to att.
496        if #[allow(non_exhaustive_omitted_patterns)] match asm_arch {
    InlineAsmArch::X86 | InlineAsmArch::X86_64 => true,
    _ => false,
}matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
497            && !options.contains(InlineAsmOptions::ATT_SYNTAX)
498        {
499            template_str.push_str("\n.att_syntax\n");
500        }
501
502        llvm::append_module_inline_asm(self.llmod, template_str.as_bytes());
503    }
504
505    fn mangled_name(&self, instance: Instance<'tcx>) -> String {
506        let llval = self.get_fn(instance);
507        llvm::build_string(|s| unsafe {
508            llvm::LLVMRustGetMangledName(llval, s);
509        })
510        .expect("symbol is not valid UTF-8")
511    }
512}
513
514pub(crate) fn inline_asm_call<'ll>(
515    bx: &mut Builder<'_, 'll, '_>,
516    asm: &str,
517    cons: &str,
518    inputs: &[&'ll Value],
519    output: &'ll llvm::Type,
520    labels: &[&'ll llvm::BasicBlock],
521    volatile: bool,
522    alignstack: bool,
523    dia: llvm::AsmDialect,
524    line_spans: &[Span],
525    unwind: bool,
526    dest: Option<&'ll llvm::BasicBlock>,
527    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
528) -> Option<&'ll Value> {
529    let argtys = inputs
530        .iter()
531        .map(|v| {
532            {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:532",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(532u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Asm Input Type: {0:?}",
                                                    *v) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("Asm Input Type: {:?}", *v);
533            bx.cx.val_ty(*v)
534        })
535        .collect::<Vec<_>>();
536
537    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:537",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(537u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Asm Output Type: {0:?}",
                                                    output) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("Asm Output Type: {:?}", output);
538    let fty = bx.cx.type_func(&argtys, output);
539
540    // Ask LLVM to verify that the constraints are well-formed.
541    let constraints_ok = unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr(), cons.len()) };
542    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:542",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(542u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("constraint verification result: {0:?}",
                                                    constraints_ok) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("constraint verification result: {:?}", constraints_ok);
543    if !constraints_ok {
544        // LLVM has detected an issue with our constraints, so bail out.
545        return None;
546    }
547
548    let v = unsafe {
549        llvm::LLVMGetInlineAsm(
550            fty,
551            asm.as_ptr(),
552            asm.len(),
553            cons.as_ptr(),
554            cons.len(),
555            volatile.to_llvm_bool(),
556            alignstack.to_llvm_bool(),
557            dia,
558            unwind.to_llvm_bool(),
559        )
560    };
561
562    let call = if !labels.is_empty() {
563        if !catch_funclet.is_none() {
    ::core::panicking::panic("assertion failed: catch_funclet.is_none()")
};assert!(catch_funclet.is_none());
564        bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None)
565    } else if let Some((catch, funclet)) = catch_funclet {
566        bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None)
567    } else {
568        bx.call(fty, None, None, v, inputs, None, None)
569    };
570
571    // Store mark in a metadata node so we can map LLVM errors
572    // back to source locations. See #17552.
573    let key = "srcloc";
574    let kind = bx.get_md_kind_id(key);
575
576    // `srcloc` contains one 64-bit integer for each line of assembly code,
577    // where the lower 32 bits hold the lo byte position and the upper 32 bits
578    // hold the hi byte position.
579    let mut srcloc = ::alloc::vec::Vec::new()vec![];
580    if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
581        // LLVM inserts an extra line to add the ".intel_syntax", so add
582        // a dummy srcloc entry for it.
583        //
584        // Don't do this if we only have 1 line span since that may be
585        // due to the asm template string coming from a macro. LLVM will
586        // default to the first srcloc for lines that don't have an
587        // associated srcloc.
588        srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0)));
589    }
590    srcloc.extend(line_spans.iter().map(|span| {
591        llvm::LLVMValueAsMetadata(
592            bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
593        )
594    }));
595    bx.cx.set_metadata_node(call, kind, &srcloc);
596
597    Some(call)
598}
599
600/// If the register is an xmm/ymm/zmm register then return its index.
601fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
602    use X86InlineAsmReg::*;
603    match reg {
604        InlineAsmReg::X86(reg) if reg as u32 >= xmm0 as u32 && reg as u32 <= xmm15 as u32 => {
605            Some(reg as u32 - xmm0 as u32)
606        }
607        InlineAsmReg::X86(reg) if reg as u32 >= ymm0 as u32 && reg as u32 <= ymm15 as u32 => {
608            Some(reg as u32 - ymm0 as u32)
609        }
610        InlineAsmReg::X86(reg) if reg as u32 >= zmm0 as u32 && reg as u32 <= zmm31 as u32 => {
611            Some(reg as u32 - zmm0 as u32)
612        }
613        _ => None,
614    }
615}
616
617/// If the register is an AArch64 integer register then return its index.
618fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
619    match reg {
620        InlineAsmReg::AArch64(r) => r.reg_index(),
621        _ => None,
622    }
623}
624
625/// If the register is an AArch64 vector register then return its index.
626fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
627    match reg {
628        InlineAsmReg::AArch64(reg) => reg.vreg_index(),
629        _ => None,
630    }
631}
632
633/// If the register is a Hexagon register pair then return its LLVM double register index.
634/// LLVM uses `d0`, `d1`, ... for Hexagon double registers in inline asm constraints,
635/// not the assembly-printed `r1:0`, `r3:2`, ... format.
636fn hexagon_reg_pair_index(reg: InlineAsmReg) -> Option<u32> {
637    match reg {
638        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r1_0) => Some(0),
639        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r3_2) => Some(1),
640        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r5_4) => Some(2),
641        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r7_6) => Some(3),
642        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r9_8) => Some(4),
643        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r11_10) => Some(5),
644        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r13_12) => Some(6),
645        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r15_14) => Some(7),
646        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r17_16) => Some(8),
647        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r21_20) => Some(10),
648        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r23_22) => Some(11),
649        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r25_24) => Some(12),
650        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r27_26) => Some(13),
651        _ => None,
652    }
653}
654
655/// If the register is a Hexagon HVX vector pair then return its LLVM W-register index.
656/// LLVM uses `w0`, `w1`, ... for Hexagon vector pair registers in inline asm constraints.
657fn hexagon_vreg_pair_index(reg: InlineAsmReg) -> Option<u32> {
658    match reg {
659        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v1_0) => Some(0),
660        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v3_2) => Some(1),
661        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v5_4) => Some(2),
662        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v7_6) => Some(3),
663        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v9_8) => Some(4),
664        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v11_10) => Some(5),
665        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v13_12) => Some(6),
666        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v15_14) => Some(7),
667        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v17_16) => Some(8),
668        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v19_18) => Some(9),
669        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v21_20) => Some(10),
670        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v23_22) => Some(11),
671        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v25_24) => Some(12),
672        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v27_26) => Some(13),
673        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v29_28) => Some(14),
674        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v31_30) => Some(15),
675        _ => None,
676    }
677}
678
679/// Converts a register class to an LLVM constraint code.
680fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
681    use InlineAsmRegClass::*;
682    match reg {
683        // For vector registers LLVM wants the register name to match the type size.
684        InlineAsmRegOrRegClass::Reg(reg) => {
685            if let Some(idx) = xmm_reg_index(reg) {
686                let class = if let Some(layout) = layout {
687                    match layout.size.bytes() {
688                        64 => 'z',
689                        32 => 'y',
690                        _ => 'x',
691                    }
692                } else {
693                    // We use f32 as the type for discarded outputs
694                    'x'
695                };
696                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}mm{1}}}", class, idx))
    })format!("{{{}mm{}}}", class, idx)
697            } else if let Some(idx) = a64_reg_index(reg) {
698                let class = if let Some(layout) = layout {
699                    match layout.size.bytes() {
700                        8 => 'x',
701                        _ => 'w',
702                    }
703                } else {
704                    // We use i32 as the type for discarded outputs
705                    'w'
706                };
707                if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
708                    // LLVM doesn't recognize x30. use lr instead.
709                    "{lr}".to_string()
710                } else {
711                    ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}{1}}}", class, idx))
    })format!("{{{}{}}}", class, idx)
712                }
713            } else if let Some(idx) = a64_vreg_index(reg) {
714                let class = if let Some(layout) = layout {
715                    match layout.size.bytes() {
716                        16 => 'q',
717                        8 => 'd',
718                        4 => 's',
719                        2 => 'h',
720                        1 => 'd', // We fixup i8 to i8x8
721                        _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
722                    }
723                } else {
724                    // We use i64x2 as the type for discarded outputs
725                    'q'
726                };
727                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}{1}}}", class, idx))
    })format!("{{{}{}}}", class, idx)
728            } else if let Some(idx) = hexagon_reg_pair_index(reg) {
729                // LLVM uses `dN` for Hexagon double registers, not the `rN+1:N` asm syntax.
730                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{d{0}}}", idx))
    })format!("{{d{}}}", idx)
731            } else if let Some(idx) = hexagon_vreg_pair_index(reg) {
732                // LLVM uses `wN` for Hexagon HVX vector pair registers.
733                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{w{0}}}", idx))
    })format!("{{w{}}}", idx)
734            } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
735                // LLVM doesn't recognize r14
736                "{lr}".to_string()
737            } else {
738                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}}}", reg.name()))
    })format!("{{{}}}", reg.name())
739            }
740        }
741        // The constraints can be retrieved from
742        // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
743        InlineAsmRegOrRegClass::RegClass(reg) => match reg {
744            AArch64(AArch64InlineAsmRegClass::reg) => "r",
745            AArch64(AArch64InlineAsmRegClass::vreg) => "w",
746            AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
747            AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
748            Arm(ArmInlineAsmRegClass::reg) => "r",
749            Arm(ArmInlineAsmRegClass::sreg)
750            | Arm(ArmInlineAsmRegClass::dreg_low16)
751            | Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
752            Arm(ArmInlineAsmRegClass::sreg_low16)
753            | Arm(ArmInlineAsmRegClass::dreg_low8)
754            | Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
755            Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w",
756            Amdgpu(AmdgpuInlineAsmRegClass::Sgpr(_)) => "s",
757            Amdgpu(AmdgpuInlineAsmRegClass::Vgpr(_)) => "v",
758            Hexagon(HexagonInlineAsmRegClass::reg) => "r",
759            Hexagon(HexagonInlineAsmRegClass::reg_pair) => "r",
760            Hexagon(HexagonInlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
761            Hexagon(HexagonInlineAsmRegClass::vreg) => "v",
762            Hexagon(HexagonInlineAsmRegClass::vreg_pair) => "v",
763            Hexagon(HexagonInlineAsmRegClass::qreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
764            LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
765            LoongArch(LoongArchInlineAsmRegClass::freg)
766            | LoongArch(LoongArchInlineAsmRegClass::vreg)
767            | LoongArch(LoongArchInlineAsmRegClass::xreg) => "f",
768            Mips(MipsInlineAsmRegClass::reg) => "r",
769            Mips(MipsInlineAsmRegClass::freg) => "f",
770            Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
771            Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
772            Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
773            PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
774            PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
775            PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
776            PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
777            PowerPC(PowerPCInlineAsmRegClass::vsreg) => "^wa",
778            PowerPC(
779                PowerPCInlineAsmRegClass::cr
780                | PowerPCInlineAsmRegClass::ctr
781                | PowerPCInlineAsmRegClass::lr
782                | PowerPCInlineAsmRegClass::xer
783                | PowerPCInlineAsmRegClass::spe_acc,
784            ) => {
785                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
786            }
787            RiscV(RiscVInlineAsmRegClass::reg) => "r",
788            RiscV(RiscVInlineAsmRegClass::freg) => "f",
789            RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
790            X86(X86InlineAsmRegClass::reg) => "r",
791            X86(X86InlineAsmRegClass::reg_abcd) => "Q",
792            X86(X86InlineAsmRegClass::reg_byte) => "q",
793            X86(X86InlineAsmRegClass::xmm_reg) | X86(X86InlineAsmRegClass::ymm_reg) => "x",
794            X86(X86InlineAsmRegClass::zmm_reg) => "v",
795            X86(X86InlineAsmRegClass::kreg) => "^Yk",
796            X86(
797                X86InlineAsmRegClass::x87_reg
798                | X86InlineAsmRegClass::mmx_reg
799                | X86InlineAsmRegClass::kreg0
800                | X86InlineAsmRegClass::tmm_reg,
801            ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
802            Xtensa(XtensaInlineAsmRegClass::freg) => "f",
803            Xtensa(XtensaInlineAsmRegClass::reg) => "r",
804            Xtensa(XtensaInlineAsmRegClass::sreg | XtensaInlineAsmRegClass::breg) => {
805                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
806            }
807            Wasm(WasmInlineAsmRegClass::local) => "r",
808            Bpf(BpfInlineAsmRegClass::reg) => "r",
809            Bpf(BpfInlineAsmRegClass::wreg) => "w",
810            Avr(AvrInlineAsmRegClass::reg) => "r",
811            Avr(AvrInlineAsmRegClass::reg_upper) => "d",
812            Avr(AvrInlineAsmRegClass::reg_pair) => "r",
813            Avr(AvrInlineAsmRegClass::reg_iw) => "w",
814            Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
815            S390x(S390xInlineAsmRegClass::reg) => "r",
816            S390x(S390xInlineAsmRegClass::reg_addr) => "a",
817            S390x(S390xInlineAsmRegClass::freg) => "f",
818            S390x(S390xInlineAsmRegClass::vreg) => "v",
819            S390x(S390xInlineAsmRegClass::areg) => {
820                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
821            }
822            Sparc(SparcInlineAsmRegClass::reg) => "r",
823            Sparc(SparcInlineAsmRegClass::yreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
824            Msp430(Msp430InlineAsmRegClass::reg) => "r",
825            M68k(M68kInlineAsmRegClass::reg) => "r",
826            M68k(M68kInlineAsmRegClass::reg_addr) => "a",
827            M68k(M68kInlineAsmRegClass::reg_data) => "d",
828            CSKY(CSKYInlineAsmRegClass::reg) => "r",
829            CSKY(CSKYInlineAsmRegClass::freg) => "f",
830            SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
831            Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
832        }
833        .to_string(),
834    }
835}
836
837/// Converts a modifier into LLVM's equivalent modifier.
838fn modifier_to_llvm(
839    arch: InlineAsmArch,
840    reg: InlineAsmRegClass,
841    modifier: Option<char>,
842) -> Option<char> {
843    use InlineAsmRegClass::*;
844    // The modifiers can be retrieved from
845    // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
846    match reg {
847        AArch64(AArch64InlineAsmRegClass::reg) => modifier,
848        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
849            if modifier == Some('v') {
850                None
851            } else {
852                modifier
853            }
854        }
855        AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
856        Arm(ArmInlineAsmRegClass::reg) => None,
857        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
858        Arm(ArmInlineAsmRegClass::dreg)
859        | Arm(ArmInlineAsmRegClass::dreg_low16)
860        | Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
861        Arm(ArmInlineAsmRegClass::qreg)
862        | Arm(ArmInlineAsmRegClass::qreg_low8)
863        | Arm(ArmInlineAsmRegClass::qreg_low4) => {
864            if modifier.is_none() {
865                Some('q')
866            } else {
867                modifier
868            }
869        }
870        Amdgpu(_) => None,
871        Hexagon(_) => None,
872        LoongArch(LoongArchInlineAsmRegClass::reg) => None,
873        LoongArch(LoongArchInlineAsmRegClass::freg) => modifier,
874        LoongArch(LoongArchInlineAsmRegClass::vreg) => {
875            if modifier.is_none() {
876                Some('w')
877            } else {
878                modifier
879            }
880        }
881        LoongArch(LoongArchInlineAsmRegClass::xreg) => {
882            if modifier.is_none() {
883                Some('u')
884            } else {
885                modifier
886            }
887        }
888        Mips(_) => None,
889        Nvptx(_) => None,
890        PowerPC(PowerPCInlineAsmRegClass::vsreg) => {
891            // The documentation for the 'x' modifier is missing for llvm, and the gcc
892            // documentation is simply "use this for any vsx argument". It is needed
893            // to ensure the correct vsx register number is used.
894            if modifier.is_none() { Some('x') } else { modifier }
895        }
896        PowerPC(_) => None,
897        RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
898        RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
899        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
900            None if arch == InlineAsmArch::X86_64 => Some('q'),
901            None => Some('k'),
902            Some('l') => Some('b'),
903            Some('h') => Some('h'),
904            Some('x') => Some('w'),
905            Some('e') => Some('k'),
906            Some('r') => Some('q'),
907            _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
908        },
909        X86(X86InlineAsmRegClass::reg_byte) => None,
910        X86(reg @ X86InlineAsmRegClass::xmm_reg)
911        | X86(reg @ X86InlineAsmRegClass::ymm_reg)
912        | X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
913            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
914            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
915            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
916            (_, Some('x')) => Some('x'),
917            (_, Some('y')) => Some('t'),
918            (_, Some('z')) => Some('g'),
919            _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
920        },
921        X86(X86InlineAsmRegClass::kreg) => None,
922        X86(
923            X86InlineAsmRegClass::x87_reg
924            | X86InlineAsmRegClass::mmx_reg
925            | X86InlineAsmRegClass::kreg0
926            | X86InlineAsmRegClass::tmm_reg,
927        ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
928        Xtensa(_) => None,
929        Wasm(WasmInlineAsmRegClass::local) => None,
930        Bpf(_) => None,
931        Avr(AvrInlineAsmRegClass::reg_pair)
932        | Avr(AvrInlineAsmRegClass::reg_iw)
933        | Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
934            Some('h') => Some('B'),
935            Some('l') => Some('A'),
936            _ => None,
937        },
938        Avr(_) => None,
939        S390x(_) => None,
940        Sparc(_) => None,
941        Msp430(_) => None,
942        SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
943        M68k(_) => None,
944        CSKY(_) => None,
945        Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
946    }
947}
948
949/// Type to use for outputs that are discarded. It doesn't really matter what
950/// the type is, as long as it is valid for the constraint code.
951fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
952    use InlineAsmRegClass::*;
953    match reg {
954        AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
955        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
956            cx.type_vector(cx.type_i64(), 2)
957        }
958        AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
959        Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
960        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
961        Arm(ArmInlineAsmRegClass::dreg)
962        | Arm(ArmInlineAsmRegClass::dreg_low16)
963        | Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
964        Arm(ArmInlineAsmRegClass::qreg)
965        | Arm(ArmInlineAsmRegClass::qreg_low8)
966        | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2),
967        Amdgpu(_) => cx.type_i32(),
968        Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
969        Hexagon(HexagonInlineAsmRegClass::reg_pair) => cx.type_i64(),
970        Hexagon(HexagonInlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
971        Hexagon(HexagonInlineAsmRegClass::vreg) => {
972            // HVX vector register size depends on the HVX mode.
973            // LLVM's "v" constraint requires the exact vector width.
974            if cx.tcx.sess.internal_target_features.contains(&sym::hvx_length128b) {
975                cx.type_vector(cx.type_i32(), 32) // 1024-bit for 128B mode
976            } else {
977                cx.type_vector(cx.type_i32(), 16) // 512-bit for 64B mode
978            }
979        }
980        Hexagon(HexagonInlineAsmRegClass::vreg_pair) => {
981            if cx.tcx.sess.internal_target_features.contains(&sym::hvx_length128b) {
982                cx.type_vector(cx.type_i32(), 64) // 2048-bit for 128B mode
983            } else {
984                cx.type_vector(cx.type_i32(), 32) // 1024-bit for 64B mode
985            }
986        }
987        Hexagon(HexagonInlineAsmRegClass::qreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
988        LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
989        LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
990        LoongArch(LoongArchInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
991        LoongArch(LoongArchInlineAsmRegClass::xreg) => cx.type_vector(cx.type_i32(), 8),
992        Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
993        Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
994        Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
995        Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
996        Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
997        PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
998        PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
999        PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
1000        PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
1001        PowerPC(PowerPCInlineAsmRegClass::vsreg) => cx.type_vector(cx.type_i32(), 4),
1002        PowerPC(
1003            PowerPCInlineAsmRegClass::cr
1004            | PowerPCInlineAsmRegClass::ctr
1005            | PowerPCInlineAsmRegClass::lr
1006            | PowerPCInlineAsmRegClass::xer
1007            | PowerPCInlineAsmRegClass::spe_acc,
1008        ) => {
1009            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
1010        }
1011        RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
1012        RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
1013        RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
1014        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
1015        X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
1016        X86(X86InlineAsmRegClass::xmm_reg)
1017        | X86(X86InlineAsmRegClass::ymm_reg)
1018        | X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
1019        X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
1020        X86(
1021            X86InlineAsmRegClass::x87_reg
1022            | X86InlineAsmRegClass::mmx_reg
1023            | X86InlineAsmRegClass::kreg0
1024            | X86InlineAsmRegClass::tmm_reg,
1025        ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
1026        Xtensa(XtensaInlineAsmRegClass::reg) => cx.type_i32(),
1027        Xtensa(XtensaInlineAsmRegClass::freg) => cx.type_f32(),
1028        Xtensa(XtensaInlineAsmRegClass::sreg | XtensaInlineAsmRegClass::breg) => {
1029            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
1030        }
1031        Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
1032        Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
1033        Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
1034        Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
1035        Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
1036        Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
1037        Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
1038        Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
1039        S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
1040        S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
1041        S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
1042        S390x(S390xInlineAsmRegClass::areg) => {
1043            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
1044        }
1045        Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),
1046        Sparc(SparcInlineAsmRegClass::yreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
1047        Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
1048        M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
1049        M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
1050        M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
1051        CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
1052        CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
1053        SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
1054        Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
1055    }
1056}
1057
1058/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
1059/// the equivalent integer type.
1060fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
1061    let dl = &cx.tcx.data_layout;
1062    match scalar.primitive() {
1063        Primitive::Int(Integer::I8, _) => cx.type_i8(),
1064        Primitive::Int(Integer::I16, _) => cx.type_i16(),
1065        Primitive::Int(Integer::I32, _) => cx.type_i32(),
1066        Primitive::Int(Integer::I64, _) => cx.type_i64(),
1067        Primitive::Float(Float::F16) => cx.type_f16(),
1068        Primitive::Float(Float::F32) => cx.type_f32(),
1069        Primitive::Float(Float::F64) => cx.type_f64(),
1070        Primitive::Float(Float::F128) => cx.type_f128(),
1071        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
1072        Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
1073        _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
1074    }
1075}
1076
1077fn any_target_feature_enabled(
1078    cx: &CodegenCx<'_, '_>,
1079    instance: Instance<'_>,
1080    features: &[Symbol],
1081) -> bool {
1082    let enabled = cx.tcx.asm_target_features(instance.def_id());
1083    features.iter().any(|feat| enabled.contains(feat))
1084}
1085
1086/// Fix up an input value to work around LLVM bugs.
1087fn llvm_fixup_input<'ll, 'tcx>(
1088    bx: &mut Builder<'_, 'll, 'tcx>,
1089    mut value: &'ll Value,
1090    reg: InlineAsmRegClass,
1091    layout: &TyAndLayout<'tcx>,
1092    instance: Instance<'_>,
1093) -> &'ll Value {
1094    use InlineAsmRegClass::*;
1095    let dl = &bx.tcx.data_layout;
1096    match (reg, layout.backend_repr) {
1097        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1098            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1099                let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
1100                bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
1101            } else {
1102                value
1103            }
1104        }
1105        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1106            if s.primitive() != Primitive::Float(Float::F128) =>
1107        {
1108            let elem_ty = llvm_asm_scalar_type(bx.cx, s);
1109            let count = 16 / layout.size.bytes();
1110            let vec_ty = bx.cx.type_vector(elem_ty, count);
1111            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
1112            if let Primitive::Pointer(_) = s.primitive() {
1113                let t = bx.type_from_integer(dl.ptr_sized_integer());
1114                value = bx.ptrtoint(value, t);
1115            }
1116            bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
1117        }
1118        (
1119            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1120            BackendRepr::SimdVector { element, count },
1121        ) if layout.size.bytes() == 8 => {
1122            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1123            let count = count.as_u32();
1124            let vec_ty = bx.cx.type_vector(elem_ty, u64::from(count));
1125            let indices: Vec<_> = (0..count * 2).map(|x| bx.const_u32(x)).collect();
1126            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1127        }
1128        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1129            if s.primitive() == Primitive::Float(Float::F64) =>
1130        {
1131            bx.bitcast(value, bx.cx.type_i64())
1132        }
1133        (
1134            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1135            BackendRepr::SimdVector { .. },
1136        ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
1137        (
1138            X86(
1139                X86InlineAsmRegClass::xmm_reg
1140                | X86InlineAsmRegClass::ymm_reg
1141                | X86InlineAsmRegClass::zmm_reg,
1142            ),
1143            BackendRepr::Scalar(s),
1144        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1145            && s.primitive() == Primitive::Float(Float::F128) =>
1146        {
1147            bx.bitcast(value, bx.type_vector(bx.type_i32(), 4))
1148        }
1149        (
1150            X86(
1151                X86InlineAsmRegClass::xmm_reg
1152                | X86InlineAsmRegClass::ymm_reg
1153                | X86InlineAsmRegClass::zmm_reg,
1154            ),
1155            BackendRepr::Scalar(s),
1156        ) if s.primitive() == Primitive::Float(Float::F16) => {
1157            let value = bx.insert_element(
1158                bx.const_undef(bx.type_vector(bx.type_f16(), 8)),
1159                value,
1160                bx.const_usize(0),
1161            );
1162            bx.bitcast(value, bx.type_vector(bx.type_i16(), 8))
1163        }
1164        (
1165            X86(
1166                X86InlineAsmRegClass::xmm_reg
1167                | X86InlineAsmRegClass::ymm_reg
1168                | X86InlineAsmRegClass::zmm_reg,
1169            ),
1170            BackendRepr::SimdVector { element, count },
1171        ) if let count = count.as_u64()
1172            && let 8 | 16 = count
1173            && element.primitive() == Primitive::Float(Float::F16) =>
1174        {
1175            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1176        }
1177        (
1178            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1179            BackendRepr::Scalar(s),
1180        ) => {
1181            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1182                bx.bitcast(value, bx.cx.type_f32())
1183            } else {
1184                value
1185            }
1186        }
1187        (
1188            Arm(
1189                ArmInlineAsmRegClass::dreg
1190                | ArmInlineAsmRegClass::dreg_low8
1191                | ArmInlineAsmRegClass::dreg_low16,
1192            ),
1193            BackendRepr::Scalar(s),
1194        ) => {
1195            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1196                bx.bitcast(value, bx.cx.type_f64())
1197            } else {
1198                value
1199            }
1200        }
1201        (
1202            Arm(
1203                ArmInlineAsmRegClass::dreg
1204                | ArmInlineAsmRegClass::dreg_low8
1205                | ArmInlineAsmRegClass::dreg_low16
1206                | ArmInlineAsmRegClass::qreg
1207                | ArmInlineAsmRegClass::qreg_low4
1208                | ArmInlineAsmRegClass::qreg_low8,
1209            ),
1210            BackendRepr::SimdVector { element, count },
1211        ) if let count = count.as_u64()
1212            && let 4 | 8 = count
1213            && element.primitive() == Primitive::Float(Float::F16) =>
1214        {
1215            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1216        }
1217        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1218            if s.primitive() == Primitive::Float(Float::F16) =>
1219        {
1220            // Smaller floats are always "NaN-boxed" inside larger floats on LoongArch.
1221            let value = bx.bitcast(value, bx.type_i16());
1222            let value = bx.zext(value, bx.type_i32());
1223            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1224            bx.bitcast(value, bx.type_f32())
1225        }
1226        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1227            match s.primitive() {
1228                // MIPS only supports register-length arithmetics.
1229                Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
1230                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_i32()),
1231                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_i64()),
1232                _ => value,
1233            }
1234        }
1235        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1236            if s.primitive() == Primitive::Float(Float::F16)
1237                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1238        {
1239            // Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.
1240            let value = bx.bitcast(value, bx.type_i16());
1241            let value = bx.zext(value, bx.type_i32());
1242            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1243            bx.bitcast(value, bx.type_f32())
1244        }
1245        (
1246            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1247            BackendRepr::Scalar(s),
1248        ) if let Primitive::Float(float @ (Float::F32 | Float::F64)) = s.primitive() => {
1249            let num_lanes = 16 / float.size().bytes();
1250            bx.insert_element(
1251                bx.const_undef(bx.type_vector(bx.type_from_float(float), num_lanes)),
1252                value,
1253                bx.const_usize(match bx.target_spec().endian {
1254                    Endian::Little => num_lanes - 1,
1255                    Endian::Big => 0,
1256                }),
1257            )
1258        }
1259        _ => value,
1260    }
1261}
1262
1263/// Fix up an output value to work around LLVM bugs.
1264fn llvm_fixup_output<'ll, 'tcx>(
1265    bx: &mut Builder<'_, 'll, 'tcx>,
1266    mut value: &'ll Value,
1267    reg: InlineAsmRegClass,
1268    layout: &TyAndLayout<'tcx>,
1269    instance: Instance<'_>,
1270) -> &'ll Value {
1271    use InlineAsmRegClass::*;
1272    match (reg, layout.backend_repr) {
1273        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1274            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1275                bx.extract_element(value, bx.const_i32(0))
1276            } else {
1277                value
1278            }
1279        }
1280        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1281            if s.primitive() != Primitive::Float(Float::F128) =>
1282        {
1283            value = bx.extract_element(value, bx.const_i32(0));
1284            if let Primitive::Pointer(_) = s.primitive() {
1285                value = bx.inttoptr(value, layout.llvm_type(bx.cx));
1286            }
1287            value
1288        }
1289        (
1290            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1291            BackendRepr::SimdVector { element, count },
1292        ) if layout.size.bytes() == 8 => {
1293            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1294            let count = count.as_u64();
1295            let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
1296            let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
1297            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1298        }
1299        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1300            if s.primitive() == Primitive::Float(Float::F64) =>
1301        {
1302            bx.bitcast(value, bx.cx.type_f64())
1303        }
1304        (
1305            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1306            BackendRepr::SimdVector { .. },
1307        ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
1308        (
1309            X86(
1310                X86InlineAsmRegClass::xmm_reg
1311                | X86InlineAsmRegClass::ymm_reg
1312                | X86InlineAsmRegClass::zmm_reg,
1313            ),
1314            BackendRepr::Scalar(s),
1315        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1316            && s.primitive() == Primitive::Float(Float::F128) =>
1317        {
1318            bx.bitcast(value, bx.type_f128())
1319        }
1320        (
1321            X86(
1322                X86InlineAsmRegClass::xmm_reg
1323                | X86InlineAsmRegClass::ymm_reg
1324                | X86InlineAsmRegClass::zmm_reg,
1325            ),
1326            BackendRepr::Scalar(s),
1327        ) if s.primitive() == Primitive::Float(Float::F16) => {
1328            let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8));
1329            bx.extract_element(value, bx.const_usize(0))
1330        }
1331        (
1332            X86(
1333                X86InlineAsmRegClass::xmm_reg
1334                | X86InlineAsmRegClass::ymm_reg
1335                | X86InlineAsmRegClass::zmm_reg,
1336            ),
1337            BackendRepr::SimdVector { element, count },
1338        ) if let count = count.as_u64()
1339            && let 8 | 16 = count
1340            && element.primitive() == Primitive::Float(Float::F16) =>
1341        {
1342            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1343        }
1344        (
1345            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1346            BackendRepr::Scalar(s),
1347        ) => {
1348            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1349                bx.bitcast(value, bx.cx.type_i32())
1350            } else {
1351                value
1352            }
1353        }
1354        (
1355            Arm(
1356                ArmInlineAsmRegClass::dreg
1357                | ArmInlineAsmRegClass::dreg_low8
1358                | ArmInlineAsmRegClass::dreg_low16,
1359            ),
1360            BackendRepr::Scalar(s),
1361        ) => {
1362            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1363                bx.bitcast(value, bx.cx.type_i64())
1364            } else {
1365                value
1366            }
1367        }
1368        (
1369            Arm(
1370                ArmInlineAsmRegClass::dreg
1371                | ArmInlineAsmRegClass::dreg_low8
1372                | ArmInlineAsmRegClass::dreg_low16
1373                | ArmInlineAsmRegClass::qreg
1374                | ArmInlineAsmRegClass::qreg_low4
1375                | ArmInlineAsmRegClass::qreg_low8,
1376            ),
1377            BackendRepr::SimdVector { element, count },
1378        ) if let count = count.as_u64()
1379            && let 4 | 8 = count
1380            && element.primitive() == Primitive::Float(Float::F16) =>
1381        {
1382            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1383        }
1384        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1385            if s.primitive() == Primitive::Float(Float::F16) =>
1386        {
1387            let value = bx.bitcast(value, bx.type_i32());
1388            let value = bx.trunc(value, bx.type_i16());
1389            bx.bitcast(value, bx.type_f16())
1390        }
1391        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1392            match s.primitive() {
1393                // MIPS only supports register-length arithmetics.
1394                Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
1395                Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
1396                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_f32()),
1397                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_f64()),
1398                _ => value,
1399            }
1400        }
1401        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1402            if s.primitive() == Primitive::Float(Float::F16)
1403                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1404        {
1405            let value = bx.bitcast(value, bx.type_i32());
1406            let value = bx.trunc(value, bx.type_i16());
1407            bx.bitcast(value, bx.type_f16())
1408        }
1409        (
1410            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1411            BackendRepr::Scalar(s),
1412        ) if let Primitive::Float(float @ (Float::F32 | Float::F64)) = s.primitive() => {
1413            let num_lanes = 16 / float.size().bytes();
1414            bx.extract_element(
1415                value,
1416                bx.const_usize(match bx.target_spec().endian {
1417                    Endian::Little => num_lanes - 1,
1418                    Endian::Big => 0,
1419                }),
1420            )
1421        }
1422        _ => value,
1423    }
1424}
1425
1426/// Output type to use for llvm_fixup_output.
1427fn llvm_fixup_output_type<'ll, 'tcx>(
1428    cx: &CodegenCx<'ll, 'tcx>,
1429    reg: InlineAsmRegClass,
1430    layout: &TyAndLayout<'tcx>,
1431    instance: Instance<'_>,
1432) -> &'ll Type {
1433    use InlineAsmRegClass::*;
1434    match (reg, layout.backend_repr) {
1435        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1436            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1437                cx.type_vector(cx.type_i8(), 8)
1438            } else {
1439                layout.llvm_type(cx)
1440            }
1441        }
1442        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1443            if s.primitive() != Primitive::Float(Float::F128) =>
1444        {
1445            let elem_ty = llvm_asm_scalar_type(cx, s);
1446            let count = 16 / layout.size.bytes();
1447            cx.type_vector(elem_ty, count)
1448        }
1449        (
1450            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1451            BackendRepr::SimdVector { element, count },
1452        ) if layout.size.bytes() == 8 => {
1453            let elem_ty = llvm_asm_scalar_type(cx, element);
1454            cx.type_vector(elem_ty, count.as_u64() * 2)
1455        }
1456        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1457            if s.primitive() == Primitive::Float(Float::F64) =>
1458        {
1459            cx.type_i64()
1460        }
1461        (
1462            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1463            BackendRepr::SimdVector { .. },
1464        ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
1465        (
1466            X86(
1467                X86InlineAsmRegClass::xmm_reg
1468                | X86InlineAsmRegClass::ymm_reg
1469                | X86InlineAsmRegClass::zmm_reg,
1470            ),
1471            BackendRepr::Scalar(s),
1472        ) if cx.sess().asm_arch == Some(InlineAsmArch::X86)
1473            && s.primitive() == Primitive::Float(Float::F128) =>
1474        {
1475            cx.type_vector(cx.type_i32(), 4)
1476        }
1477        (
1478            X86(
1479                X86InlineAsmRegClass::xmm_reg
1480                | X86InlineAsmRegClass::ymm_reg
1481                | X86InlineAsmRegClass::zmm_reg,
1482            ),
1483            BackendRepr::Scalar(s),
1484        ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8),
1485        (
1486            X86(
1487                X86InlineAsmRegClass::xmm_reg
1488                | X86InlineAsmRegClass::ymm_reg
1489                | X86InlineAsmRegClass::zmm_reg,
1490            ),
1491            BackendRepr::SimdVector { element, count },
1492        ) if let count = count.as_u64()
1493            && let 8 | 16 = count
1494            && element.primitive() == Primitive::Float(Float::F16) =>
1495        {
1496            cx.type_vector(cx.type_i16(), count)
1497        }
1498        (
1499            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1500            BackendRepr::Scalar(s),
1501        ) => {
1502            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1503                cx.type_f32()
1504            } else {
1505                layout.llvm_type(cx)
1506            }
1507        }
1508        (
1509            Arm(
1510                ArmInlineAsmRegClass::dreg
1511                | ArmInlineAsmRegClass::dreg_low8
1512                | ArmInlineAsmRegClass::dreg_low16,
1513            ),
1514            BackendRepr::Scalar(s),
1515        ) => {
1516            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1517                cx.type_f64()
1518            } else {
1519                layout.llvm_type(cx)
1520            }
1521        }
1522        (
1523            Arm(
1524                ArmInlineAsmRegClass::dreg
1525                | ArmInlineAsmRegClass::dreg_low8
1526                | ArmInlineAsmRegClass::dreg_low16
1527                | ArmInlineAsmRegClass::qreg
1528                | ArmInlineAsmRegClass::qreg_low4
1529                | ArmInlineAsmRegClass::qreg_low8,
1530            ),
1531            BackendRepr::SimdVector { element, count },
1532        ) if let count = count.as_u64()
1533            && let 4 | 8 = count
1534            && element.primitive() == Primitive::Float(Float::F16) =>
1535        {
1536            cx.type_vector(cx.type_i16(), count)
1537        }
1538        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1539            if s.primitive() == Primitive::Float(Float::F16) =>
1540        {
1541            cx.type_f32()
1542        }
1543        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1544            match s.primitive() {
1545                // MIPS only supports register-length arithmetics.
1546                Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1547                Primitive::Float(Float::F32) => cx.type_i32(),
1548                Primitive::Float(Float::F64) => cx.type_i64(),
1549                _ => layout.llvm_type(cx),
1550            }
1551        }
1552        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1553            if s.primitive() == Primitive::Float(Float::F16)
1554                && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
1555        {
1556            cx.type_f32()
1557        }
1558        (
1559            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1560            BackendRepr::Scalar(s),
1561        ) if let Primitive::Float(float @ (Float::F32 | Float::F64)) = s.primitive() => {
1562            cx.type_vector(cx.type_from_float(float), 16 / float.size().bytes())
1563        }
1564        _ => layout.llvm_type(cx),
1565    }
1566}
rustc_codegen_llvm/asm.rs

rustc_codegen_llvm/
asm.rs
