rustc_codegen_llvm/
asm.rs

1use std::assert_matches;
2use std::fmt::Write;
3
4use rustc_abi::{BackendRepr, Endian, Float, Integer, Primitive, Scalar, Size};
5use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
6use rustc_codegen_ssa::mir::operand::OperandValue;
7use rustc_codegen_ssa::traits::*;
8use rustc_data_structures::fx::FxHashMap;
9use rustc_middle::mir::interpret::{PointerArithmetic, Scalar as ConstScalar};
10use rustc_middle::ty::Instance;
11use rustc_middle::ty::layout::TyAndLayout;
12use rustc_middle::{bug, span_bug};
13use rustc_span::{Pos, Span, Symbol, sym};
14use rustc_target::asm::*;
15use rustc_target::spec::HasTargetSpec;
16use smallvec::SmallVec;
17use tracing::debug;
18
19use crate::builder::Builder;
20use crate::common::Funclet;
21use crate::context::CodegenCx;
22use crate::llvm::{self, ToLlvmBool, Type, Value};
23use crate::type_of::LayoutLlvmExt;
24use crate::{attributes, llvm_util};
25
26impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
27    fn codegen_inline_asm(
28        &mut self,
29        template: &[InlineAsmTemplatePiece],
30        operands: &[InlineAsmOperandRef<'tcx, Self>],
31        options: InlineAsmOptions,
32        line_spans: &[Span],
33        instance: Instance<'_>,
34        dest: Option<Self::BasicBlock>,
35        catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
36    ) {
37        let asm_arch = self.tcx.sess.asm_arch.unwrap();
38
39        // Collect the types of output operands
40        let mut constraints = ::alloc::vec::Vec::new()vec![];
41        let mut clobbers = ::alloc::vec::Vec::new()vec![];
42        let mut output_types = ::alloc::vec::Vec::new()vec![];
43        let mut op_idx = FxHashMap::default();
44        let mut clobbered_x87 = false;
45        for (idx, op) in operands.iter().enumerate() {
46            match *op {
47                InlineAsmOperandRef::Out { reg, late, place } => {
48                    let is_target_supported = |reg_class: InlineAsmRegClass| {
49                        for &(_, feature) in reg_class.supported_types(asm_arch, true).as_ref() {
50                            if let Some(feature) = feature {
51                                if self
52                                    .tcx
53                                    .asm_target_features(instance.def_id())
54                                    .contains(&feature)
55                                {
56                                    return true;
57                                }
58                            } else {
59                                // Register class is unconditionally supported
60                                return true;
61                            }
62                        }
63                        false
64                    };
65
66                    let mut layout = None;
67                    let ty = if let Some(ref place) = place {
68                        layout = Some(&place.layout);
69                        llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)
70                    } else if #[allow(non_exhaustive_omitted_patterns)] match reg.reg_class() {
    InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg |
        X86InlineAsmRegClass::x87_reg) => true,
    _ => false,
}matches!(
71                        reg.reg_class(),
72                        InlineAsmRegClass::X86(
73                            X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg
74                        )
75                    ) {
76                        // Special handling for x87/mmx registers: we always
77                        // clobber the whole set if one register is marked as
78                        // clobbered. This is due to the way LLVM handles the
79                        // FP stack in inline assembly.
80                        if !clobbered_x87 {
81                            clobbered_x87 = true;
82                            clobbers.push("~{st}".to_string());
83                            for i in 1..=7 {
84                                clobbers.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("~{{st({0})}}", i))
    })format!("~{{st({})}}", i));
85                            }
86                        }
87                        continue;
88                    } else if !is_target_supported(reg.reg_class())
89                        || reg.reg_class().is_clobber_only(asm_arch, true)
90                    {
91                        // We turn discarded outputs into clobber constraints
92                        // if the target feature needed by the register class is
93                        // disabled. This is necessary otherwise LLVM will try
94                        // to actually allocate a register for the dummy output.
95                        {
    match reg {
        InlineAsmRegOrRegClass::Reg(_) => {}
        ref left_val => {
            ::core::panicking::assert_matches_failed(left_val,
                "InlineAsmRegOrRegClass::Reg(_)",
                ::core::option::Option::None);
        }
    }
};assert_matches!(reg, InlineAsmRegOrRegClass::Reg(_));
96                        clobbers.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("~{0}", reg_to_llvm(reg, None)))
    })format!("~{}", reg_to_llvm(reg, None)));
97                        continue;
98                    } else {
99                        // If the output is discarded, we don't really care what
100                        // type is used. We're just using this to tell LLVM to
101                        // reserve the register.
102                        dummy_output_type(self.cx, reg.reg_class())
103                    };
104                    output_types.push(ty);
105                    op_idx.insert(idx, constraints.len());
106                    let prefix = if late { "=" } else { "=&" };
107                    constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", prefix,
                reg_to_llvm(reg, layout)))
    })format!("{}{}", prefix, reg_to_llvm(reg, layout)));
108                }
109                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
110                    let layout = if let Some(ref out_place) = out_place {
111                        &out_place.layout
112                    } else {
113                        // LLVM required tied operands to have the same type,
114                        // so we just use the type of the input.
115                        &in_value.layout
116                    };
117                    let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);
118                    output_types.push(ty);
119                    op_idx.insert(idx, constraints.len());
120                    let prefix = if late { "=" } else { "=&" };
121                    constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}{1}", prefix,
                reg_to_llvm(reg, Some(layout))))
    })format!("{}{}", prefix, reg_to_llvm(reg, Some(layout))));
122                }
123                _ => {}
124            }
125        }
126
127        // Collect input operands
128        let mut inputs = ::alloc::vec::Vec::new()vec![];
129        for (idx, op) in operands.iter().enumerate() {
130            match *op {
131                InlineAsmOperandRef::In { reg, value } => {
132                    let llval = llvm_fixup_input(
133                        self,
134                        value.immediate(),
135                        reg.reg_class(),
136                        &value.layout,
137                        instance,
138                    );
139                    inputs.push(llval);
140                    op_idx.insert(idx, constraints.len());
141                    constraints.push(reg_to_llvm(reg, Some(&value.layout)));
142                }
143                InlineAsmOperandRef::InOut { reg, late, in_value, out_place: _ } => {
144                    let value = llvm_fixup_input(
145                        self,
146                        in_value.immediate(),
147                        reg.reg_class(),
148                        &in_value.layout,
149                        instance,
150                    );
151                    inputs.push(value);
152
153                    // In the case of fixed registers, we have the choice of
154                    // either using a tied operand or duplicating the constraint.
155                    // We prefer the latter because it matches the behavior of
156                    // Clang.
157                    if late && #[allow(non_exhaustive_omitted_patterns)] match reg {
    InlineAsmRegOrRegClass::Reg(_) => true,
    _ => false,
}matches!(reg, InlineAsmRegOrRegClass::Reg(_)) {
158                        constraints.push(reg_to_llvm(reg, Some(&in_value.layout)));
159                    } else {
160                        constraints.push(::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}", op_idx[&idx]))
    })format!("{}", op_idx[&idx]));
161                    }
162                }
163                InlineAsmOperandRef::Const { value, ty: _ } => match value {
164                    ConstScalar::Int(_) => (),
165                    ConstScalar::Ptr(ptr, _) => {
166                        let (prov, _) = ptr.prov_and_relative_offset();
167                        let global_alloc = self.tcx.global_alloc(prov.alloc_id());
168                        let value = self.cx.alloc_to_backend(global_alloc, false, None).unwrap();
169                        inputs.push(value);
170                        op_idx.insert(idx, constraints.len());
171                        constraints.push("s".to_string());
172                    }
173                },
174                InlineAsmOperandRef::SymThreadLocalStatic { def_id } => {
175                    inputs.push(self.cx.get_static(def_id));
176                    op_idx.insert(idx, constraints.len());
177                    constraints.push("s".to_string());
178                }
179                _ => {}
180            }
181        }
182
183        // Build the template string
184        let mut labels = ::alloc::vec::Vec::new()vec![];
185        let mut template_str = String::new();
186        for piece in template {
187            match *piece {
188                InlineAsmTemplatePiece::String(ref s) => {
189                    if s.contains('$') {
190                        for c in s.chars() {
191                            if c == '$' {
192                                template_str.push_str("$$");
193                            } else {
194                                template_str.push(c);
195                            }
196                        }
197                    } else {
198                        template_str.push_str(s)
199                    }
200                }
201                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span } => {
202                    match operands[operand_idx] {
203                        InlineAsmOperandRef::In { reg, .. }
204                        | InlineAsmOperandRef::Out { reg, .. }
205                        | InlineAsmOperandRef::InOut { reg, .. } => {
206                            let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier);
207                            if let Some(modifier) = modifier {
208                                template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:{1}}}",
                op_idx[&operand_idx], modifier))
    })format!(
209                                    "${{{}:{}}}",
210                                    op_idx[&operand_idx], modifier
211                                ));
212                            } else {
213                                template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}}}", op_idx[&operand_idx]))
    })format!("${{{}}}", op_idx[&operand_idx]));
214                            }
215                        }
216                        InlineAsmOperandRef::Const { value, ty } => {
217                            match value {
218                                ConstScalar::Int(int) => {
219                                    // Const operands get injected directly into the template
220                                    let string = rustc_codegen_ssa::common::asm_const_to_str(
221                                        self.tcx,
222                                        span,
223                                        int,
224                                        self.layout_of(ty),
225                                    );
226                                    template_str.push_str(&string);
227                                }
228                                ConstScalar::Ptr(ptr, _) => {
229                                    let (_, offset) = ptr.prov_and_relative_offset();
230
231                                    // Only emit the raw symbol name
232                                    template_str
233                                        .push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:c}}", op_idx[&operand_idx]))
    })format!("${{{}:c}}", op_idx[&operand_idx]));
234
235                                    if offset != Size::ZERO {
236                                        let offset =
237                                            self.sign_extend_to_target_isize(offset.bytes());
238                                        template_str.write_fmt(format_args!("{0:+}", offset))write!(template_str, "{offset:+}").unwrap();
239                                    }
240                                }
241                            }
242                        }
243                        InlineAsmOperandRef::SymThreadLocalStatic { .. } => {
244                            // Only emit the raw symbol name
245                            template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:c}}", op_idx[&operand_idx]))
    })format!("${{{}:c}}", op_idx[&operand_idx]));
246                        }
247                        InlineAsmOperandRef::Label { label } => {
248                            template_str.push_str(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("${{{0}:l}}", constraints.len()))
    })format!("${{{}:l}}", constraints.len()));
249                            constraints.push("!i".to_owned());
250                            labels.push(label);
251                        }
252                    }
253                }
254            }
255        }
256
257        constraints.append(&mut clobbers);
258        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
259            match asm_arch {
260                InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC | InlineAsmArch::Arm => {
261                    constraints.push("~{cc}".to_string());
262                }
263                InlineAsmArch::Amdgpu => {}
264                InlineAsmArch::X86 | InlineAsmArch::X86_64 => {
265                    constraints.extend_from_slice(&[
266                        "~{dirflag}".to_string(),
267                        "~{fpsr}".to_string(),
268                        "~{flags}".to_string(),
269                    ]);
270                }
271                InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => {
272                    constraints.extend_from_slice(&[
273                        "~{fflags}".to_string(),
274                        "~{vtype}".to_string(),
275                        "~{vl}".to_string(),
276                        "~{vxsat}".to_string(),
277                        "~{vxrm}".to_string(),
278                    ]);
279                }
280                InlineAsmArch::Avr => {
281                    constraints.push("~{sreg}".to_string());
282                }
283                InlineAsmArch::Nvptx64 => {}
284                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
285                InlineAsmArch::Hexagon => {}
286                InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
287                    constraints.extend_from_slice(&[
288                        "~{$fcc0}".to_string(),
289                        "~{$fcc1}".to_string(),
290                        "~{$fcc2}".to_string(),
291                        "~{$fcc3}".to_string(),
292                        "~{$fcc4}".to_string(),
293                        "~{$fcc5}".to_string(),
294                        "~{$fcc6}".to_string(),
295                        "~{$fcc7}".to_string(),
296                    ]);
297                }
298                InlineAsmArch::Mips | InlineAsmArch::Mips64 => {}
299                InlineAsmArch::S390x => {
300                    constraints.push("~{cc}".to_string());
301                }
302                InlineAsmArch::Sparc | InlineAsmArch::Sparc64 => {
303                    // In LLVM, ~{icc} represents icc and xcc in 64-bit code.
304                    // https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/Sparc/SparcRegisterInfo.td#L64
305                    constraints.push("~{icc}".to_string());
306                    constraints.push("~{fcc0}".to_string());
307                    constraints.push("~{fcc1}".to_string());
308                    constraints.push("~{fcc2}".to_string());
309                    constraints.push("~{fcc3}".to_string());
310                }
311                InlineAsmArch::SpirV => {}
312                InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {}
313                InlineAsmArch::Xtensa => {}
314                InlineAsmArch::Bpf => {}
315                InlineAsmArch::Msp430 => {
316                    constraints.push("~{sr}".to_string());
317                }
318                InlineAsmArch::M68k => {
319                    constraints.push("~{ccr}".to_string());
320                }
321                InlineAsmArch::CSKY => {
322                    constraints.push("~{psr}".to_string());
323                }
324            }
325        }
326        if !options.contains(InlineAsmOptions::NOMEM) {
327            // This is actually ignored by LLVM, but it's probably best to keep
328            // it just in case. LLVM instead uses the ReadOnly/ReadNone
329            // attributes on the call instruction to optimize.
330            constraints.push("~{memory}".to_string());
331        }
332        let volatile = !options.contains(InlineAsmOptions::PURE);
333        let alignstack = !options.contains(InlineAsmOptions::NOSTACK);
334        let output_type = match &output_types[..] {
335            [] => self.type_void(),
336            [ty] => ty,
337            tys => self.type_struct(tys, false),
338        };
339        let dialect = match asm_arch {
340            InlineAsmArch::X86 | InlineAsmArch::X86_64
341                if !options.contains(InlineAsmOptions::ATT_SYNTAX) =>
342            {
343                llvm::AsmDialect::Intel
344            }
345            _ => llvm::AsmDialect::Att,
346        };
347        let result = inline_asm_call(
348            self,
349            &template_str,
350            &constraints.join(","),
351            &inputs,
352            output_type,
353            &labels,
354            volatile,
355            alignstack,
356            dialect,
357            line_spans,
358            options.contains(InlineAsmOptions::MAY_UNWIND),
359            dest,
360            catch_funclet,
361        )
362        .unwrap_or_else(|| ::rustc_middle::util::bug::span_bug_fmt(line_spans[0],
    format_args!("LLVM asm constraint validation failed"))span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
363
364        let mut attrs = SmallVec::<[_; 2]>::new();
365        if options.contains(InlineAsmOptions::PURE) {
366            if options.contains(InlineAsmOptions::NOMEM) {
367                attrs.push(llvm::MemoryEffects::None.create_attr(self.cx.llcx));
368            } else if options.contains(InlineAsmOptions::READONLY) {
369                attrs.push(llvm::MemoryEffects::ReadOnly.create_attr(self.cx.llcx));
370            }
371            attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
372        } else if options.contains(InlineAsmOptions::NOMEM) {
373            attrs.push(llvm::MemoryEffects::InaccessibleMemOnly.create_attr(self.cx.llcx));
374        } else if options.contains(InlineAsmOptions::READONLY) {
375            attrs.push(llvm::MemoryEffects::ReadOnlyNotPure.create_attr(self.cx.llcx));
376        }
377        attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
378
379        // Write results to outputs. We need to do this for all possible control flow.
380        //
381        // Note that `dest` maybe populated with unreachable_block when asm goto with outputs
382        // is used (because we need to codegen callbr which always needs a destination), so
383        // here we use the NORETURN option to determine if `dest` should be used.
384        for block in (if options.contains(InlineAsmOptions::NORETURN) { None } else { Some(dest) })
385            .into_iter()
386            .chain(labels.iter().copied().map(Some))
387        {
388            if let Some(block) = block {
389                self.switch_to_block(block);
390            }
391
392            for (idx, op) in operands.iter().enumerate() {
393                if let InlineAsmOperandRef::Out { reg, place: Some(place), .. }
394                | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op
395                {
396                    let value = if output_types.len() == 1 {
397                        result
398                    } else {
399                        self.extract_value(result, op_idx[&idx] as u64)
400                    };
401                    let value =
402                        llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);
403                    OperandValue::Immediate(value).store(self, place);
404                }
405            }
406        }
407    }
408}
409
410impl<'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
411    fn codegen_global_asm(
412        &mut self,
413        template: &[InlineAsmTemplatePiece],
414        operands: &[GlobalAsmOperandRef<'tcx>],
415        options: InlineAsmOptions,
416        _line_spans: &[Span],
417    ) {
418        let asm_arch = self.tcx.sess.asm_arch.unwrap();
419
420        // Build the template string
421        let mut template_str = String::new();
422
423        // On X86 platforms there are two assembly syntaxes. Rust uses intel by default,
424        // but AT&T can be specified explicitly.
425        if #[allow(non_exhaustive_omitted_patterns)] match asm_arch {
    InlineAsmArch::X86 | InlineAsmArch::X86_64 => true,
    _ => false,
}matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) {
426            if options.contains(InlineAsmOptions::ATT_SYNTAX) {
427                template_str.push_str(".att_syntax\n")
428            } else {
429                template_str.push_str(".intel_syntax\n")
430            }
431        }
432
433        for piece in template {
434            match *piece {
435                InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s),
436                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span } => {
437                    use rustc_codegen_ssa::back::symbol_export::escape_symbol_name;
438                    match operands[operand_idx] {
439                        GlobalAsmOperandRef::Const { value, ty } => {
440                            match value {
441                                ConstScalar::Int(int) => {
442                                    // Const operands get injected directly into the
443                                    // template. Note that we don't need to escape $
444                                    // here unlike normal inline assembly.
445                                    let string = rustc_codegen_ssa::common::asm_const_to_str(
446                                        self.tcx,
447                                        span,
448                                        int,
449                                        self.layout_of(ty),
450                                    );
451                                    template_str.push_str(&string);
452                                }
453
454                                ConstScalar::Ptr(ptr, _) => {
455                                    let (prov, offset) = ptr.prov_and_relative_offset();
456                                    let global_alloc = self.tcx.global_alloc(prov.alloc_id());
457                                    let llval =
458                                        self.alloc_to_backend(global_alloc, true, None).unwrap();
459
460                                    self.add_compiler_used_global(llval);
461                                    let symbol = llvm::build_string(|s| unsafe {
462                                        llvm::LLVMRustGetMangledName(llval, s);
463                                    })
464                                    .expect("symbol is not valid UTF-8");
465                                    template_str
466                                        .push_str(&escape_symbol_name(self.tcx, &symbol, span));
467
468                                    if offset != Size::ZERO {
469                                        let offset =
470                                            self.sign_extend_to_target_isize(offset.bytes());
471                                        template_str.write_fmt(format_args!("{0:+}", offset))write!(template_str, "{offset:+}").unwrap();
472                                    }
473                                }
474                            }
475                        }
476                        GlobalAsmOperandRef::SymThreadLocalStatic { def_id } => {
477                            let llval = self
478                                .renamed_statics
479                                .borrow()
480                                .get(&def_id)
481                                .copied()
482                                .unwrap_or_else(|| self.get_static(def_id));
483                            self.add_compiler_used_global(llval);
484                            let symbol = llvm::build_string(|s| unsafe {
485                                llvm::LLVMRustGetMangledName(llval, s);
486                            })
487                            .expect("symbol is not valid UTF-8");
488                            template_str.push_str(&escape_symbol_name(self.tcx, &symbol, span));
489                        }
490                    }
491                }
492            }
493        }
494
495        // Just to play it safe, if intel was used, reset the assembly syntax to att.
496        if #[allow(non_exhaustive_omitted_patterns)] match asm_arch {
    InlineAsmArch::X86 | InlineAsmArch::X86_64 => true,
    _ => false,
}matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
497            && !options.contains(InlineAsmOptions::ATT_SYNTAX)
498        {
499            template_str.push_str("\n.att_syntax\n");
500        }
501
502        let target_features = self.tcx.global_backend_features(()).join(",");
503        let target_cpu = llvm_util::target_cpu(self.tcx.sess);
504
505        llvm::append_module_inline_asm(
506            self.llmod,
507            template_str.as_bytes(),
508            &target_features,
509            target_cpu,
510        );
511    }
512
513    fn mangled_name(&self, instance: Instance<'tcx>) -> String {
514        let llval = self.get_fn(instance);
515        llvm::build_string(|s| unsafe {
516            llvm::LLVMRustGetMangledName(llval, s);
517        })
518        .expect("symbol is not valid UTF-8")
519    }
520}
521
522pub(crate) fn inline_asm_call<'ll>(
523    bx: &mut Builder<'_, 'll, '_>,
524    asm: &str,
525    cons: &str,
526    inputs: &[&'ll Value],
527    output: &'ll llvm::Type,
528    labels: &[&'ll llvm::BasicBlock],
529    volatile: bool,
530    alignstack: bool,
531    dia: llvm::AsmDialect,
532    line_spans: &[Span],
533    unwind: bool,
534    dest: Option<&'ll llvm::BasicBlock>,
535    catch_funclet: Option<(&'ll llvm::BasicBlock, Option<&Funclet<'ll>>)>,
536) -> Option<&'ll Value> {
537    let argtys = inputs
538        .iter()
539        .map(|v| {
540            {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:540",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(540u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Asm Input Type: {0:?}",
                                                    *v) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("Asm Input Type: {:?}", *v);
541            bx.cx.val_ty(*v)
542        })
543        .collect::<Vec<_>>();
544
545    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:545",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(545u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Asm Output Type: {0:?}",
                                                    output) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("Asm Output Type: {:?}", output);
546    let fty = bx.cx.type_func(&argtys, output);
547
548    // Ask LLVM to verify that the constraints are well-formed.
549    let constraints_ok = unsafe { llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr(), cons.len()) };
550    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event compiler/rustc_codegen_llvm/src/asm.rs:550",
                        "rustc_codegen_llvm::asm", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("compiler/rustc_codegen_llvm/src/asm.rs"),
                        ::tracing_core::__macro_support::Option::Some(550u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::asm"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("constraint verification result: {0:?}",
                                                    constraints_ok) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("constraint verification result: {:?}", constraints_ok);
551    if !constraints_ok {
552        // LLVM has detected an issue with our constraints, so bail out.
553        return None;
554    }
555
556    let v = unsafe {
557        llvm::LLVMGetInlineAsm(
558            fty,
559            asm.as_ptr(),
560            asm.len(),
561            cons.as_ptr(),
562            cons.len(),
563            volatile.to_llvm_bool(),
564            alignstack.to_llvm_bool(),
565            dia,
566            unwind.to_llvm_bool(),
567        )
568    };
569
570    let call = if !labels.is_empty() {
571        if !catch_funclet.is_none() {
    ::core::panicking::panic("assertion failed: catch_funclet.is_none()")
};assert!(catch_funclet.is_none());
572        bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None)
573    } else if let Some((catch, funclet)) = catch_funclet {
574        bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None)
575    } else {
576        bx.call(fty, None, None, v, inputs, None, None)
577    };
578
579    // Store mark in a metadata node so we can map LLVM errors
580    // back to source locations. See #17552.
581    let key = "srcloc";
582    let kind = bx.get_md_kind_id(key);
583
584    // `srcloc` contains one 64-bit integer for each line of assembly code,
585    // where the lower 32 bits hold the lo byte position and the upper 32 bits
586    // hold the hi byte position.
587    let mut srcloc = ::alloc::vec::Vec::new()vec![];
588    if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 {
589        // LLVM inserts an extra line to add the ".intel_syntax", so add
590        // a dummy srcloc entry for it.
591        //
592        // Don't do this if we only have 1 line span since that may be
593        // due to the asm template string coming from a macro. LLVM will
594        // default to the first srcloc for lines that don't have an
595        // associated srcloc.
596        srcloc.push(llvm::LLVMValueAsMetadata(bx.const_u64(0)));
597    }
598    srcloc.extend(line_spans.iter().map(|span| {
599        llvm::LLVMValueAsMetadata(
600            bx.const_u64(u64::from(span.lo().to_u32()) | (u64::from(span.hi().to_u32()) << 32)),
601        )
602    }));
603    bx.cx.set_metadata_node(call, kind, &srcloc);
604
605    Some(call)
606}
607
608/// If the register is an xmm/ymm/zmm register then return its index.
609fn xmm_reg_index(reg: InlineAsmReg) -> Option<u32> {
610    use X86InlineAsmReg::*;
611    match reg {
612        InlineAsmReg::X86(reg) if reg as u32 >= xmm0 as u32 && reg as u32 <= xmm15 as u32 => {
613            Some(reg as u32 - xmm0 as u32)
614        }
615        InlineAsmReg::X86(reg) if reg as u32 >= ymm0 as u32 && reg as u32 <= ymm15 as u32 => {
616            Some(reg as u32 - ymm0 as u32)
617        }
618        InlineAsmReg::X86(reg) if reg as u32 >= zmm0 as u32 && reg as u32 <= zmm31 as u32 => {
619            Some(reg as u32 - zmm0 as u32)
620        }
621        _ => None,
622    }
623}
624
625/// If the register is an AArch64 integer register then return its index.
626fn a64_reg_index(reg: InlineAsmReg) -> Option<u32> {
627    match reg {
628        InlineAsmReg::AArch64(r) => r.reg_index(),
629        _ => None,
630    }
631}
632
633/// If the register is an AArch64 vector register then return its index.
634fn a64_vreg_index(reg: InlineAsmReg) -> Option<u32> {
635    match reg {
636        InlineAsmReg::AArch64(reg) => reg.vreg_index(),
637        _ => None,
638    }
639}
640
641/// If the register is a Hexagon register pair then return its LLVM double register index.
642/// LLVM uses `d0`, `d1`, ... for Hexagon double registers in inline asm constraints,
643/// not the assembly-printed `r1:0`, `r3:2`, ... format.
644fn hexagon_reg_pair_index(reg: InlineAsmReg) -> Option<u32> {
645    match reg {
646        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r1_0) => Some(0),
647        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r3_2) => Some(1),
648        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r5_4) => Some(2),
649        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r7_6) => Some(3),
650        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r9_8) => Some(4),
651        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r11_10) => Some(5),
652        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r13_12) => Some(6),
653        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r15_14) => Some(7),
654        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r17_16) => Some(8),
655        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r21_20) => Some(10),
656        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r23_22) => Some(11),
657        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r25_24) => Some(12),
658        InlineAsmReg::Hexagon(HexagonInlineAsmReg::r27_26) => Some(13),
659        _ => None,
660    }
661}
662
663/// If the register is a Hexagon HVX vector pair then return its LLVM W-register index.
664/// LLVM uses `w0`, `w1`, ... for Hexagon vector pair registers in inline asm constraints.
665fn hexagon_vreg_pair_index(reg: InlineAsmReg) -> Option<u32> {
666    match reg {
667        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v1_0) => Some(0),
668        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v3_2) => Some(1),
669        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v5_4) => Some(2),
670        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v7_6) => Some(3),
671        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v9_8) => Some(4),
672        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v11_10) => Some(5),
673        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v13_12) => Some(6),
674        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v15_14) => Some(7),
675        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v17_16) => Some(8),
676        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v19_18) => Some(9),
677        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v21_20) => Some(10),
678        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v23_22) => Some(11),
679        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v25_24) => Some(12),
680        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v27_26) => Some(13),
681        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v29_28) => Some(14),
682        InlineAsmReg::Hexagon(HexagonInlineAsmReg::v31_30) => Some(15),
683        _ => None,
684    }
685}
686
687/// Converts a register class to an LLVM constraint code.
688fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String {
689    use InlineAsmRegClass::*;
690    match reg {
691        // For vector registers LLVM wants the register name to match the type size.
692        InlineAsmRegOrRegClass::Reg(reg) => {
693            if let Some(idx) = xmm_reg_index(reg) {
694                let class = if let Some(layout) = layout {
695                    match layout.size.bytes() {
696                        64 => 'z',
697                        32 => 'y',
698                        _ => 'x',
699                    }
700                } else {
701                    // We use f32 as the type for discarded outputs
702                    'x'
703                };
704                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}mm{1}}}", class, idx))
    })format!("{{{}mm{}}}", class, idx)
705            } else if let Some(idx) = a64_reg_index(reg) {
706                let class = if let Some(layout) = layout {
707                    match layout.size.bytes() {
708                        8 => 'x',
709                        _ => 'w',
710                    }
711                } else {
712                    // We use i32 as the type for discarded outputs
713                    'w'
714                };
715                if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) {
716                    // LLVM doesn't recognize x30. use lr instead.
717                    "{lr}".to_string()
718                } else {
719                    ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}{1}}}", class, idx))
    })format!("{{{}{}}}", class, idx)
720                }
721            } else if let Some(idx) = a64_vreg_index(reg) {
722                let class = if let Some(layout) = layout {
723                    match layout.size.bytes() {
724                        16 => 'q',
725                        8 => 'd',
726                        4 => 's',
727                        2 => 'h',
728                        1 => 'd', // We fixup i8 to i8x8
729                        _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
730                    }
731                } else {
732                    // We use i64x2 as the type for discarded outputs
733                    'q'
734                };
735                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}{1}}}", class, idx))
    })format!("{{{}{}}}", class, idx)
736            } else if let Some(idx) = hexagon_reg_pair_index(reg) {
737                // LLVM uses `dN` for Hexagon double registers, not the `rN+1:N` asm syntax.
738                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{d{0}}}", idx))
    })format!("{{d{}}}", idx)
739            } else if let Some(idx) = hexagon_vreg_pair_index(reg) {
740                // LLVM uses `wN` for Hexagon HVX vector pair registers.
741                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{w{0}}}", idx))
    })format!("{{w{}}}", idx)
742            } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) {
743                // LLVM doesn't recognize r14
744                "{lr}".to_string()
745            } else {
746                ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{{{0}}}", reg.name()))
    })format!("{{{}}}", reg.name())
747            }
748        }
749        // The constraints can be retrieved from
750        // https://llvm.org/docs/LangRef.html#supported-constraint-code-list
751        InlineAsmRegOrRegClass::RegClass(reg) => match reg {
752            AArch64(AArch64InlineAsmRegClass::reg) => "r",
753            AArch64(AArch64InlineAsmRegClass::vreg) => "w",
754            AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
755            AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
756            Arm(ArmInlineAsmRegClass::reg) => "r",
757            Arm(ArmInlineAsmRegClass::sreg)
758            | Arm(ArmInlineAsmRegClass::dreg_low16)
759            | Arm(ArmInlineAsmRegClass::qreg_low8) => "t",
760            Arm(ArmInlineAsmRegClass::sreg_low16)
761            | Arm(ArmInlineAsmRegClass::dreg_low8)
762            | Arm(ArmInlineAsmRegClass::qreg_low4) => "x",
763            Arm(ArmInlineAsmRegClass::dreg) | Arm(ArmInlineAsmRegClass::qreg) => "w",
764            Amdgpu(AmdgpuInlineAsmRegClass::Sgpr(_)) => "s",
765            Amdgpu(AmdgpuInlineAsmRegClass::Vgpr(_)) => "v",
766            Hexagon(HexagonInlineAsmRegClass::reg) => "r",
767            Hexagon(HexagonInlineAsmRegClass::reg_pair) => "r",
768            Hexagon(HexagonInlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
769            Hexagon(HexagonInlineAsmRegClass::vreg) => "v",
770            Hexagon(HexagonInlineAsmRegClass::vreg_pair) => "v",
771            Hexagon(HexagonInlineAsmRegClass::qreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
772            LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
773            LoongArch(LoongArchInlineAsmRegClass::freg)
774            | LoongArch(LoongArchInlineAsmRegClass::vreg)
775            | LoongArch(LoongArchInlineAsmRegClass::xreg) => "f",
776            Mips(MipsInlineAsmRegClass::reg) => "r",
777            Mips(MipsInlineAsmRegClass::freg) => "f",
778            Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
779            Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
780            Nvptx(NvptxInlineAsmRegClass::reg64) => "l",
781            PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
782            PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
783            PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
784            PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
785            PowerPC(PowerPCInlineAsmRegClass::vsreg) => "^wa",
786            PowerPC(
787                PowerPCInlineAsmRegClass::cr
788                | PowerPCInlineAsmRegClass::ctr
789                | PowerPCInlineAsmRegClass::lr
790                | PowerPCInlineAsmRegClass::xer
791                | PowerPCInlineAsmRegClass::spe_acc,
792            ) => {
793                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
794            }
795            RiscV(RiscVInlineAsmRegClass::reg) => "r",
796            RiscV(RiscVInlineAsmRegClass::freg) => "f",
797            RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
798            X86(X86InlineAsmRegClass::reg) => "r",
799            X86(X86InlineAsmRegClass::reg_abcd) => "Q",
800            X86(X86InlineAsmRegClass::reg_byte) => "q",
801            X86(X86InlineAsmRegClass::xmm_reg) | X86(X86InlineAsmRegClass::ymm_reg) => "x",
802            X86(X86InlineAsmRegClass::zmm_reg) => "v",
803            X86(X86InlineAsmRegClass::kreg) => "^Yk",
804            X86(
805                X86InlineAsmRegClass::x87_reg
806                | X86InlineAsmRegClass::mmx_reg
807                | X86InlineAsmRegClass::kreg0
808                | X86InlineAsmRegClass::tmm_reg,
809            ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
810            Xtensa(XtensaInlineAsmRegClass::freg) => "f",
811            Xtensa(XtensaInlineAsmRegClass::reg) => "r",
812            Xtensa(XtensaInlineAsmRegClass::sreg | XtensaInlineAsmRegClass::breg) => {
813                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
814            }
815            Wasm(WasmInlineAsmRegClass::local) => "r",
816            Bpf(BpfInlineAsmRegClass::reg) => "r",
817            Bpf(BpfInlineAsmRegClass::wreg) => "w",
818            Avr(AvrInlineAsmRegClass::reg) => "r",
819            Avr(AvrInlineAsmRegClass::reg_upper) => "d",
820            Avr(AvrInlineAsmRegClass::reg_pair) => "r",
821            Avr(AvrInlineAsmRegClass::reg_iw) => "w",
822            Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
823            S390x(S390xInlineAsmRegClass::reg) => "r",
824            S390x(S390xInlineAsmRegClass::reg_addr) => "a",
825            S390x(S390xInlineAsmRegClass::freg) => "f",
826            S390x(S390xInlineAsmRegClass::vreg) => "v",
827            S390x(S390xInlineAsmRegClass::areg) => {
828                {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
829            }
830            Sparc(SparcInlineAsmRegClass::reg) => "r",
831            Sparc(SparcInlineAsmRegClass::yreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
832            Msp430(Msp430InlineAsmRegClass::reg) => "r",
833            M68k(M68kInlineAsmRegClass::reg) => "r",
834            M68k(M68kInlineAsmRegClass::reg_addr) => "a",
835            M68k(M68kInlineAsmRegClass::reg_data) => "d",
836            CSKY(CSKYInlineAsmRegClass::reg) => "r",
837            CSKY(CSKYInlineAsmRegClass::freg) => "f",
838            SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
839            Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
840        }
841        .to_string(),
842    }
843}
844
845/// Converts a modifier into LLVM's equivalent modifier.
846fn modifier_to_llvm(
847    arch: InlineAsmArch,
848    reg: InlineAsmRegClass,
849    modifier: Option<char>,
850) -> Option<char> {
851    use InlineAsmRegClass::*;
852    // The modifiers can be retrieved from
853    // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers
854    match reg {
855        AArch64(AArch64InlineAsmRegClass::reg) => modifier,
856        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
857            if modifier == Some('v') {
858                None
859            } else {
860                modifier
861            }
862        }
863        AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
864        Arm(ArmInlineAsmRegClass::reg) => None,
865        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None,
866        Arm(ArmInlineAsmRegClass::dreg)
867        | Arm(ArmInlineAsmRegClass::dreg_low16)
868        | Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
869        Arm(ArmInlineAsmRegClass::qreg)
870        | Arm(ArmInlineAsmRegClass::qreg_low8)
871        | Arm(ArmInlineAsmRegClass::qreg_low4) => {
872            if modifier.is_none() {
873                Some('q')
874            } else {
875                modifier
876            }
877        }
878        Amdgpu(_) => None,
879        Hexagon(_) => None,
880        LoongArch(LoongArchInlineAsmRegClass::reg) => None,
881        LoongArch(LoongArchInlineAsmRegClass::freg) => modifier,
882        LoongArch(LoongArchInlineAsmRegClass::vreg) => {
883            if modifier.is_none() {
884                Some('w')
885            } else {
886                modifier
887            }
888        }
889        LoongArch(LoongArchInlineAsmRegClass::xreg) => {
890            if modifier.is_none() {
891                Some('u')
892            } else {
893                modifier
894            }
895        }
896        Mips(_) => None,
897        Nvptx(_) => None,
898        PowerPC(PowerPCInlineAsmRegClass::vsreg) => {
899            // The documentation for the 'x' modifier is missing for llvm, and the gcc
900            // documentation is simply "use this for any vsx argument". It is needed
901            // to ensure the correct vsx register number is used.
902            if modifier.is_none() { Some('x') } else { modifier }
903        }
904        PowerPC(_) => None,
905        RiscV(RiscVInlineAsmRegClass::reg) | RiscV(RiscVInlineAsmRegClass::freg) => None,
906        RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
907        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
908            None if arch == InlineAsmArch::X86_64 => Some('q'),
909            None => Some('k'),
910            Some('l') => Some('b'),
911            Some('h') => Some('h'),
912            Some('x') => Some('w'),
913            Some('e') => Some('k'),
914            Some('r') => Some('q'),
915            _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
916        },
917        X86(X86InlineAsmRegClass::reg_byte) => None,
918        X86(reg @ X86InlineAsmRegClass::xmm_reg)
919        | X86(reg @ X86InlineAsmRegClass::ymm_reg)
920        | X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
921            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
922            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
923            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
924            (_, Some('x')) => Some('x'),
925            (_, Some('y')) => Some('t'),
926            (_, Some('z')) => Some('g'),
927            _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
928        },
929        X86(X86InlineAsmRegClass::kreg) => None,
930        X86(
931            X86InlineAsmRegClass::x87_reg
932            | X86InlineAsmRegClass::mmx_reg
933            | X86InlineAsmRegClass::kreg0
934            | X86InlineAsmRegClass::tmm_reg,
935        ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
936        Xtensa(_) => None,
937        Wasm(WasmInlineAsmRegClass::local) => None,
938        Bpf(_) => None,
939        Avr(AvrInlineAsmRegClass::reg_pair)
940        | Avr(AvrInlineAsmRegClass::reg_iw)
941        | Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
942            Some('h') => Some('B'),
943            Some('l') => Some('A'),
944            _ => None,
945        },
946        Avr(_) => None,
947        S390x(_) => None,
948        Sparc(_) => None,
949        Msp430(_) => None,
950        SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
951        M68k(_) => None,
952        CSKY(_) => None,
953        Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
954    }
955}
956
957/// Type to use for outputs that are discarded. It doesn't really matter what
958/// the type is, as long as it is valid for the constraint code.
959fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type {
960    use InlineAsmRegClass::*;
961    match reg {
962        AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
963        AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
964            cx.type_vector(cx.type_i64(), 2)
965        }
966        AArch64(AArch64InlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
967        Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
968        Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
969        Arm(ArmInlineAsmRegClass::dreg)
970        | Arm(ArmInlineAsmRegClass::dreg_low16)
971        | Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
972        Arm(ArmInlineAsmRegClass::qreg)
973        | Arm(ArmInlineAsmRegClass::qreg_low8)
974        | Arm(ArmInlineAsmRegClass::qreg_low4) => cx.type_vector(cx.type_i64(), 2),
975        Amdgpu(_) => cx.type_i32(),
976        Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
977        Hexagon(HexagonInlineAsmRegClass::reg_pair) => cx.type_i64(),
978        Hexagon(HexagonInlineAsmRegClass::preg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
979        Hexagon(HexagonInlineAsmRegClass::vreg) => {
980            // HVX vector register size depends on the HVX mode.
981            // LLVM's "v" constraint requires the exact vector width.
982            if cx.tcx.sess.internal_target_features.contains(&sym::hvx_length128b) {
983                cx.type_vector(cx.type_i32(), 32) // 1024-bit for 128B mode
984            } else {
985                cx.type_vector(cx.type_i32(), 16) // 512-bit for 64B mode
986            }
987        }
988        Hexagon(HexagonInlineAsmRegClass::vreg_pair) => {
989            if cx.tcx.sess.internal_target_features.contains(&sym::hvx_length128b) {
990                cx.type_vector(cx.type_i32(), 64) // 2048-bit for 128B mode
991            } else {
992                cx.type_vector(cx.type_i32(), 32) // 1024-bit for 64B mode
993            }
994        }
995        Hexagon(HexagonInlineAsmRegClass::qreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
996        LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
997        LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
998        LoongArch(LoongArchInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
999        LoongArch(LoongArchInlineAsmRegClass::xreg) => cx.type_vector(cx.type_i32(), 8),
1000        Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
1001        Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
1002        Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
1003        Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
1004        Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
1005        PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
1006        PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
1007        PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
1008        PowerPC(PowerPCInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i32(), 4),
1009        PowerPC(PowerPCInlineAsmRegClass::vsreg) => cx.type_vector(cx.type_i32(), 4),
1010        PowerPC(
1011            PowerPCInlineAsmRegClass::cr
1012            | PowerPCInlineAsmRegClass::ctr
1013            | PowerPCInlineAsmRegClass::lr
1014            | PowerPCInlineAsmRegClass::xer
1015            | PowerPCInlineAsmRegClass::spe_acc,
1016        ) => {
1017            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
1018        }
1019        RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
1020        RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
1021        RiscV(RiscVInlineAsmRegClass::vreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
1022        X86(X86InlineAsmRegClass::reg) | X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
1023        X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
1024        X86(X86InlineAsmRegClass::xmm_reg)
1025        | X86(X86InlineAsmRegClass::ymm_reg)
1026        | X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
1027        X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
1028        X86(
1029            X86InlineAsmRegClass::x87_reg
1030            | X86InlineAsmRegClass::mmx_reg
1031            | X86InlineAsmRegClass::kreg0
1032            | X86InlineAsmRegClass::tmm_reg,
1033        ) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
1034        Xtensa(XtensaInlineAsmRegClass::reg) => cx.type_i32(),
1035        Xtensa(XtensaInlineAsmRegClass::freg) => cx.type_f32(),
1036        Xtensa(XtensaInlineAsmRegClass::sreg | XtensaInlineAsmRegClass::breg) => {
1037            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
1038        }
1039        Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
1040        Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
1041        Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
1042        Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
1043        Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
1044        Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
1045        Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
1046        Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
1047        S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
1048        S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
1049        S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
1050        S390x(S390xInlineAsmRegClass::areg) => {
1051            {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only")
1052        }
1053        Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),
1054        Sparc(SparcInlineAsmRegClass::yreg) => {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("clobber-only")));
}unreachable!("clobber-only"),
1055        Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
1056        M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
1057        M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
1058        M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
1059        CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
1060        CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
1061        SpirV(SpirVInlineAsmRegClass::reg) => ::rustc_middle::util::bug::bug_fmt(format_args!("LLVM backend does not support SPIR-V"))bug!("LLVM backend does not support SPIR-V"),
1062        Err => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
1063    }
1064}
1065
1066/// Helper function to get the LLVM type for a Scalar. Pointers are returned as
1067/// the equivalent integer type.
1068fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type {
1069    let dl = &cx.tcx.data_layout;
1070    match scalar.primitive() {
1071        Primitive::Int(Integer::I8, _) => cx.type_i8(),
1072        Primitive::Int(Integer::I16, _) => cx.type_i16(),
1073        Primitive::Int(Integer::I32, _) => cx.type_i32(),
1074        Primitive::Int(Integer::I64, _) => cx.type_i64(),
1075        Primitive::Float(Float::F16) => cx.type_f16(),
1076        Primitive::Float(Float::F32) => cx.type_f32(),
1077        Primitive::Float(Float::F64) => cx.type_f64(),
1078        Primitive::Float(Float::F128) => cx.type_f128(),
1079        // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
1080        Primitive::Pointer(_) => cx.type_from_integer(dl.ptr_sized_integer()),
1081        _ => ::core::panicking::panic("internal error: entered unreachable code")unreachable!(),
1082    }
1083}
1084
1085fn any_target_feature_enabled(
1086    cx: &CodegenCx<'_, '_>,
1087    instance: Instance<'_>,
1088    features: &[Symbol],
1089) -> bool {
1090    let enabled = cx.tcx.asm_target_features(instance.def_id());
1091    features.iter().any(|feat| enabled.contains(feat))
1092}
1093
1094/// Fix up an input value to work around LLVM bugs.
1095fn llvm_fixup_input<'ll, 'tcx>(
1096    bx: &mut Builder<'_, 'll, 'tcx>,
1097    mut value: &'ll Value,
1098    reg: InlineAsmRegClass,
1099    layout: &TyAndLayout<'tcx>,
1100    instance: Instance<'_>,
1101) -> &'ll Value {
1102    use InlineAsmRegClass::*;
1103    let dl = &bx.tcx.data_layout;
1104    match (reg, layout.backend_repr) {
1105        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1106            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1107                let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8);
1108                bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
1109            } else {
1110                value
1111            }
1112        }
1113        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1114            if s.primitive() != Primitive::Float(Float::F128) =>
1115        {
1116            let elem_ty = llvm_asm_scalar_type(bx.cx, s);
1117            let count = 16 / layout.size.bytes();
1118            let vec_ty = bx.cx.type_vector(elem_ty, count);
1119            // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
1120            if let Primitive::Pointer(_) = s.primitive() {
1121                let t = bx.type_from_integer(dl.ptr_sized_integer());
1122                value = bx.ptrtoint(value, t);
1123            }
1124            bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0))
1125        }
1126        (
1127            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1128            BackendRepr::SimdVector { element, count },
1129        ) if layout.size.bytes() == 8 => {
1130            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1131            let count = count.as_u32();
1132            let vec_ty = bx.cx.type_vector(elem_ty, u64::from(count));
1133            let indices: Vec<_> = (0..count * 2).map(|x| bx.const_u32(x)).collect();
1134            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1135        }
1136        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1137            if s.primitive() == Primitive::Float(Float::F64) =>
1138        {
1139            bx.bitcast(value, bx.cx.type_i64())
1140        }
1141        (
1142            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1143            BackendRepr::SimdVector { .. },
1144        ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)),
1145        (
1146            X86(
1147                X86InlineAsmRegClass::xmm_reg
1148                | X86InlineAsmRegClass::ymm_reg
1149                | X86InlineAsmRegClass::zmm_reg,
1150            ),
1151            BackendRepr::Scalar(s),
1152        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1153            && s.primitive() == Primitive::Float(Float::F128) =>
1154        {
1155            bx.bitcast(value, bx.type_vector(bx.type_i32(), 4))
1156        }
1157        (
1158            X86(
1159                X86InlineAsmRegClass::xmm_reg
1160                | X86InlineAsmRegClass::ymm_reg
1161                | X86InlineAsmRegClass::zmm_reg,
1162            ),
1163            BackendRepr::Scalar(s),
1164        ) if s.primitive() == Primitive::Float(Float::F16) => {
1165            let value = bx.insert_element(
1166                bx.const_undef(bx.type_vector(bx.type_f16(), 8)),
1167                value,
1168                bx.const_usize(0),
1169            );
1170            bx.bitcast(value, bx.type_vector(bx.type_i16(), 8))
1171        }
1172        (
1173            X86(
1174                X86InlineAsmRegClass::xmm_reg
1175                | X86InlineAsmRegClass::ymm_reg
1176                | X86InlineAsmRegClass::zmm_reg,
1177            ),
1178            BackendRepr::SimdVector { element, count },
1179        ) if let count = count.as_u64()
1180            && let 8 | 16 = count
1181            && element.primitive() == Primitive::Float(Float::F16) =>
1182        {
1183            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1184        }
1185        (
1186            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1187            BackendRepr::Scalar(s),
1188        ) => {
1189            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1190                bx.bitcast(value, bx.cx.type_f32())
1191            } else {
1192                value
1193            }
1194        }
1195        (
1196            Arm(
1197                ArmInlineAsmRegClass::dreg
1198                | ArmInlineAsmRegClass::dreg_low8
1199                | ArmInlineAsmRegClass::dreg_low16,
1200            ),
1201            BackendRepr::Scalar(s),
1202        ) => {
1203            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1204                bx.bitcast(value, bx.cx.type_f64())
1205            } else {
1206                value
1207            }
1208        }
1209        (
1210            Arm(
1211                ArmInlineAsmRegClass::dreg
1212                | ArmInlineAsmRegClass::dreg_low8
1213                | ArmInlineAsmRegClass::dreg_low16
1214                | ArmInlineAsmRegClass::qreg
1215                | ArmInlineAsmRegClass::qreg_low4
1216                | ArmInlineAsmRegClass::qreg_low8,
1217            ),
1218            BackendRepr::SimdVector { element, count },
1219        ) if let count = count.as_u64()
1220            && let 4 | 8 = count
1221            && element.primitive() == Primitive::Float(Float::F16) =>
1222        {
1223            bx.bitcast(value, bx.type_vector(bx.type_i16(), count))
1224        }
1225        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1226            if s.primitive() == Primitive::Float(Float::F16) =>
1227        {
1228            // Smaller floats are always "NaN-boxed" inside larger floats on LoongArch.
1229            let value = bx.bitcast(value, bx.type_i16());
1230            let value = bx.zext(value, bx.type_i32());
1231            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1232            bx.bitcast(value, bx.type_f32())
1233        }
1234        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1235            match s.primitive() {
1236                // MIPS only supports register-length arithmetics.
1237                Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()),
1238                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_i32()),
1239                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_i64()),
1240                _ => value,
1241            }
1242        }
1243        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1244            if s.primitive() == Primitive::Float(Float::F16)
1245                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1246        {
1247            // Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.
1248            let value = bx.bitcast(value, bx.type_i16());
1249            let value = bx.zext(value, bx.type_i32());
1250            let value = bx.or(value, bx.const_u32(0xFFFF_0000));
1251            bx.bitcast(value, bx.type_f32())
1252        }
1253        (
1254            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1255            BackendRepr::Scalar(s),
1256        ) if let Primitive::Float(float @ (Float::F32 | Float::F64)) = s.primitive() => {
1257            let num_lanes = 16 / float.size().bytes();
1258            bx.insert_element(
1259                bx.const_undef(bx.type_vector(bx.type_from_float(float), num_lanes)),
1260                value,
1261                bx.const_usize(match bx.target_spec().endian {
1262                    Endian::Little => num_lanes - 1,
1263                    Endian::Big => 0,
1264                }),
1265            )
1266        }
1267        _ => value,
1268    }
1269}
1270
1271/// Fix up an output value to work around LLVM bugs.
1272fn llvm_fixup_output<'ll, 'tcx>(
1273    bx: &mut Builder<'_, 'll, 'tcx>,
1274    mut value: &'ll Value,
1275    reg: InlineAsmRegClass,
1276    layout: &TyAndLayout<'tcx>,
1277    instance: Instance<'_>,
1278) -> &'ll Value {
1279    use InlineAsmRegClass::*;
1280    match (reg, layout.backend_repr) {
1281        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1282            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1283                bx.extract_element(value, bx.const_i32(0))
1284            } else {
1285                value
1286            }
1287        }
1288        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1289            if s.primitive() != Primitive::Float(Float::F128) =>
1290        {
1291            value = bx.extract_element(value, bx.const_i32(0));
1292            if let Primitive::Pointer(_) = s.primitive() {
1293                value = bx.inttoptr(value, layout.llvm_type(bx.cx));
1294            }
1295            value
1296        }
1297        (
1298            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1299            BackendRepr::SimdVector { element, count },
1300        ) if layout.size.bytes() == 8 => {
1301            let elem_ty = llvm_asm_scalar_type(bx.cx, element);
1302            let count = count.as_u64();
1303            let vec_ty = bx.cx.type_vector(elem_ty, count * 2);
1304            let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect();
1305            bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices))
1306        }
1307        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1308            if s.primitive() == Primitive::Float(Float::F64) =>
1309        {
1310            bx.bitcast(value, bx.cx.type_f64())
1311        }
1312        (
1313            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1314            BackendRepr::SimdVector { .. },
1315        ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)),
1316        (
1317            X86(
1318                X86InlineAsmRegClass::xmm_reg
1319                | X86InlineAsmRegClass::ymm_reg
1320                | X86InlineAsmRegClass::zmm_reg,
1321            ),
1322            BackendRepr::Scalar(s),
1323        ) if bx.sess().asm_arch == Some(InlineAsmArch::X86)
1324            && s.primitive() == Primitive::Float(Float::F128) =>
1325        {
1326            bx.bitcast(value, bx.type_f128())
1327        }
1328        (
1329            X86(
1330                X86InlineAsmRegClass::xmm_reg
1331                | X86InlineAsmRegClass::ymm_reg
1332                | X86InlineAsmRegClass::zmm_reg,
1333            ),
1334            BackendRepr::Scalar(s),
1335        ) if s.primitive() == Primitive::Float(Float::F16) => {
1336            let value = bx.bitcast(value, bx.type_vector(bx.type_f16(), 8));
1337            bx.extract_element(value, bx.const_usize(0))
1338        }
1339        (
1340            X86(
1341                X86InlineAsmRegClass::xmm_reg
1342                | X86InlineAsmRegClass::ymm_reg
1343                | X86InlineAsmRegClass::zmm_reg,
1344            ),
1345            BackendRepr::SimdVector { element, count },
1346        ) if let count = count.as_u64()
1347            && let 8 | 16 = count
1348            && element.primitive() == Primitive::Float(Float::F16) =>
1349        {
1350            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1351        }
1352        (
1353            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1354            BackendRepr::Scalar(s),
1355        ) => {
1356            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1357                bx.bitcast(value, bx.cx.type_i32())
1358            } else {
1359                value
1360            }
1361        }
1362        (
1363            Arm(
1364                ArmInlineAsmRegClass::dreg
1365                | ArmInlineAsmRegClass::dreg_low8
1366                | ArmInlineAsmRegClass::dreg_low16,
1367            ),
1368            BackendRepr::Scalar(s),
1369        ) => {
1370            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1371                bx.bitcast(value, bx.cx.type_i64())
1372            } else {
1373                value
1374            }
1375        }
1376        (
1377            Arm(
1378                ArmInlineAsmRegClass::dreg
1379                | ArmInlineAsmRegClass::dreg_low8
1380                | ArmInlineAsmRegClass::dreg_low16
1381                | ArmInlineAsmRegClass::qreg
1382                | ArmInlineAsmRegClass::qreg_low4
1383                | ArmInlineAsmRegClass::qreg_low8,
1384            ),
1385            BackendRepr::SimdVector { element, count },
1386        ) if let count = count.as_u64()
1387            && let 4 | 8 = count
1388            && element.primitive() == Primitive::Float(Float::F16) =>
1389        {
1390            bx.bitcast(value, bx.type_vector(bx.type_f16(), count))
1391        }
1392        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1393            if s.primitive() == Primitive::Float(Float::F16) =>
1394        {
1395            let value = bx.bitcast(value, bx.type_i32());
1396            let value = bx.trunc(value, bx.type_i16());
1397            bx.bitcast(value, bx.type_f16())
1398        }
1399        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1400            match s.primitive() {
1401                // MIPS only supports register-length arithmetics.
1402                Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()),
1403                Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()),
1404                Primitive::Float(Float::F32) => bx.bitcast(value, bx.cx.type_f32()),
1405                Primitive::Float(Float::F64) => bx.bitcast(value, bx.cx.type_f64()),
1406                _ => value,
1407            }
1408        }
1409        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1410            if s.primitive() == Primitive::Float(Float::F16)
1411                && !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>
1412        {
1413            let value = bx.bitcast(value, bx.type_i32());
1414            let value = bx.trunc(value, bx.type_i16());
1415            bx.bitcast(value, bx.type_f16())
1416        }
1417        (
1418            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1419            BackendRepr::Scalar(s),
1420        ) if let Primitive::Float(float @ (Float::F32 | Float::F64)) = s.primitive() => {
1421            let num_lanes = 16 / float.size().bytes();
1422            bx.extract_element(
1423                value,
1424                bx.const_usize(match bx.target_spec().endian {
1425                    Endian::Little => num_lanes - 1,
1426                    Endian::Big => 0,
1427                }),
1428            )
1429        }
1430        _ => value,
1431    }
1432}
1433
1434/// Output type to use for llvm_fixup_output.
1435fn llvm_fixup_output_type<'ll, 'tcx>(
1436    cx: &CodegenCx<'ll, 'tcx>,
1437    reg: InlineAsmRegClass,
1438    layout: &TyAndLayout<'tcx>,
1439    instance: Instance<'_>,
1440) -> &'ll Type {
1441    use InlineAsmRegClass::*;
1442    match (reg, layout.backend_repr) {
1443        (AArch64(AArch64InlineAsmRegClass::vreg), BackendRepr::Scalar(s)) => {
1444            if let Primitive::Int(Integer::I8, _) = s.primitive() {
1445                cx.type_vector(cx.type_i8(), 8)
1446            } else {
1447                layout.llvm_type(cx)
1448            }
1449        }
1450        (AArch64(AArch64InlineAsmRegClass::vreg_low16), BackendRepr::Scalar(s))
1451            if s.primitive() != Primitive::Float(Float::F128) =>
1452        {
1453            let elem_ty = llvm_asm_scalar_type(cx, s);
1454            let count = 16 / layout.size.bytes();
1455            cx.type_vector(elem_ty, count)
1456        }
1457        (
1458            AArch64(AArch64InlineAsmRegClass::vreg_low16),
1459            BackendRepr::SimdVector { element, count },
1460        ) if layout.size.bytes() == 8 => {
1461            let elem_ty = llvm_asm_scalar_type(cx, element);
1462            cx.type_vector(elem_ty, count.as_u64() * 2)
1463        }
1464        (X86(X86InlineAsmRegClass::reg_abcd), BackendRepr::Scalar(s))
1465            if s.primitive() == Primitive::Float(Float::F64) =>
1466        {
1467            cx.type_i64()
1468        }
1469        (
1470            X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg),
1471            BackendRepr::SimdVector { .. },
1472        ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8),
1473        (
1474            X86(
1475                X86InlineAsmRegClass::xmm_reg
1476                | X86InlineAsmRegClass::ymm_reg
1477                | X86InlineAsmRegClass::zmm_reg,
1478            ),
1479            BackendRepr::Scalar(s),
1480        ) if cx.sess().asm_arch == Some(InlineAsmArch::X86)
1481            && s.primitive() == Primitive::Float(Float::F128) =>
1482        {
1483            cx.type_vector(cx.type_i32(), 4)
1484        }
1485        (
1486            X86(
1487                X86InlineAsmRegClass::xmm_reg
1488                | X86InlineAsmRegClass::ymm_reg
1489                | X86InlineAsmRegClass::zmm_reg,
1490            ),
1491            BackendRepr::Scalar(s),
1492        ) if s.primitive() == Primitive::Float(Float::F16) => cx.type_vector(cx.type_i16(), 8),
1493        (
1494            X86(
1495                X86InlineAsmRegClass::xmm_reg
1496                | X86InlineAsmRegClass::ymm_reg
1497                | X86InlineAsmRegClass::zmm_reg,
1498            ),
1499            BackendRepr::SimdVector { element, count },
1500        ) if let count = count.as_u64()
1501            && let 8 | 16 = count
1502            && element.primitive() == Primitive::Float(Float::F16) =>
1503        {
1504            cx.type_vector(cx.type_i16(), count)
1505        }
1506        (
1507            Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16),
1508            BackendRepr::Scalar(s),
1509        ) => {
1510            if let Primitive::Int(Integer::I32, _) = s.primitive() {
1511                cx.type_f32()
1512            } else {
1513                layout.llvm_type(cx)
1514            }
1515        }
1516        (
1517            Arm(
1518                ArmInlineAsmRegClass::dreg
1519                | ArmInlineAsmRegClass::dreg_low8
1520                | ArmInlineAsmRegClass::dreg_low16,
1521            ),
1522            BackendRepr::Scalar(s),
1523        ) => {
1524            if let Primitive::Int(Integer::I64, _) = s.primitive() {
1525                cx.type_f64()
1526            } else {
1527                layout.llvm_type(cx)
1528            }
1529        }
1530        (
1531            Arm(
1532                ArmInlineAsmRegClass::dreg
1533                | ArmInlineAsmRegClass::dreg_low8
1534                | ArmInlineAsmRegClass::dreg_low16
1535                | ArmInlineAsmRegClass::qreg
1536                | ArmInlineAsmRegClass::qreg_low4
1537                | ArmInlineAsmRegClass::qreg_low8,
1538            ),
1539            BackendRepr::SimdVector { element, count },
1540        ) if let count = count.as_u64()
1541            && let 4 | 8 = count
1542            && element.primitive() == Primitive::Float(Float::F16) =>
1543        {
1544            cx.type_vector(cx.type_i16(), count)
1545        }
1546        (LoongArch(LoongArchInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1547            if s.primitive() == Primitive::Float(Float::F16) =>
1548        {
1549            cx.type_f32()
1550        }
1551        (Mips(MipsInlineAsmRegClass::reg), BackendRepr::Scalar(s)) => {
1552            match s.primitive() {
1553                // MIPS only supports register-length arithmetics.
1554                Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(),
1555                Primitive::Float(Float::F32) => cx.type_i32(),
1556                Primitive::Float(Float::F64) => cx.type_i64(),
1557                _ => layout.llvm_type(cx),
1558            }
1559        }
1560        (RiscV(RiscVInlineAsmRegClass::freg), BackendRepr::Scalar(s))
1561            if s.primitive() == Primitive::Float(Float::F16)
1562                && !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>
1563        {
1564            cx.type_f32()
1565        }
1566        (
1567            PowerPC(PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg),
1568            BackendRepr::Scalar(s),
1569        ) if let Primitive::Float(float @ (Float::F32 | Float::F64)) = s.primitive() => {
1570            cx.type_vector(cx.type_from_float(float), 16 / float.size().bytes())
1571        }
1572        _ => layout.llvm_type(cx),
1573    }
1574}
rustc_codegen_llvm/asm.rs

rustc_codegen_llvm/
asm.rs
