rustc_codegen_llvm/
builder.rs

Help
1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5pub(crate) mod autodiff;
6
7use libc::{c_char, c_uint, size_t};
8use rustc_abi as abi;
9use rustc_abi::{Align, Size, WrappingRange};
10use rustc_codegen_ssa::MemFlags;
11use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::PlaceRef;
14use rustc_codegen_ssa::traits::*;
15use rustc_data_structures::small_c_str::SmallCStr;
16use rustc_hir::def_id::DefId;
17use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
18use rustc_middle::ty::layout::{
19    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
20    TyAndLayout,
21};
22use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
23use rustc_sanitizers::{cfi, kcfi};
24use rustc_session::config::OptLevel;
25use rustc_span::Span;
26use rustc_target::callconv::FnAbi;
27use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
28use smallvec::SmallVec;
29use tracing::{debug, instrument};
30
31use crate::abi::FnAbiLlvmExt;
32use crate::common::Funclet;
33use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
34use crate::llvm::{
35    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
36};
37use crate::type_::Type;
38use crate::type_of::LayoutLlvmExt;
39use crate::value::Value;
40use crate::{attributes, llvm_util};
41
42#[must_use]
43pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
44    pub llbuilder: &'ll mut llvm::Builder<'ll>,
45    pub cx: &'a GenericCx<'ll, CX>,
46}
47
48pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
49pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
50
51impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
52    fn drop(&mut self) {
53        unsafe {
54            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
55        }
56    }
57}
58
59impl<'a, 'll> SBuilder<'a, 'll> {
60    pub(crate) fn call(
61        &mut self,
62        llty: &'ll Type,
63        llfn: &'ll Value,
64        args: &[&'ll Value],
65        funclet: Option<&Funclet<'ll>>,
66    ) -> &'ll Value {
67        debug!("call {:?} with args ({:?})", llfn, args);
68
69        let args = self.check_call("call", llty, llfn, args);
70        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
71        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
72        if let Some(funclet_bundle) = funclet_bundle {
73            bundles.push(funclet_bundle);
74        }
75
76        let call = unsafe {
77            llvm::LLVMBuildCallWithOperandBundles(
78                self.llbuilder,
79                llty,
80                llfn,
81                args.as_ptr() as *const &llvm::Value,
82                args.len() as c_uint,
83                bundles.as_ptr(),
84                bundles.len() as c_uint,
85                c"".as_ptr(),
86            )
87        };
88        call
89    }
90}
91
92impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
93    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
94        // Create a fresh builder from the simple context.
95        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
96        GenericBuilder { llbuilder, cx: scx }
97    }
98
99    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
100        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
101    }
102
103    pub(crate) fn ret_void(&mut self) {
104        llvm::LLVMBuildRetVoid(self.llbuilder);
105    }
106
107    pub(crate) fn ret(&mut self, v: &'ll Value) {
108        unsafe {
109            llvm::LLVMBuildRet(self.llbuilder, v);
110        }
111    }
112
113    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
114        let bx = Self::with_cx(cx);
115        unsafe {
116            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
117        }
118        bx
119    }
120}
121
122/// Empty string, to be used where LLVM expects an instruction name, indicating
123/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
124// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
125const UNNAMED: *const c_char = c"".as_ptr();
126
127impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
128    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
129    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
130    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
131    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
132    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
133    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
134
135    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
136    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
137    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
138}
139
140impl abi::HasDataLayout for Builder<'_, '_, '_> {
141    fn data_layout(&self) -> &abi::TargetDataLayout {
142        self.cx.data_layout()
143    }
144}
145
146impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
147    #[inline]
148    fn tcx(&self) -> TyCtxt<'tcx> {
149        self.cx.tcx
150    }
151}
152
153impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
154    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
155        self.cx.typing_env()
156    }
157}
158
159impl HasTargetSpec for Builder<'_, '_, '_> {
160    #[inline]
161    fn target_spec(&self) -> &Target {
162        self.cx.target_spec()
163    }
164}
165
166impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
167    #[inline]
168    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
169        self.cx.handle_layout_err(err, span, ty)
170    }
171}
172
173impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
174    #[inline]
175    fn handle_fn_abi_err(
176        &self,
177        err: FnAbiError<'tcx>,
178        span: Span,
179        fn_abi_request: FnAbiRequest<'tcx>,
180    ) -> ! {
181        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
182    }
183}
184
185impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
186    type Target = CodegenCx<'ll, 'tcx>;
187
188    #[inline]
189    fn deref(&self) -> &Self::Target {
190        self.cx
191    }
192}
193
194macro_rules! math_builder_methods {
195    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
196        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
197            unsafe {
198                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
199            }
200        })+
201    }
202}
203
204macro_rules! set_math_builder_methods {
205    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
206        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
207            unsafe {
208                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
209                llvm::$llvm_set_math(instr);
210                instr
211            }
212        })+
213    }
214}
215
216impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
217    type CodegenCx = CodegenCx<'ll, 'tcx>;
218
219    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
220        let bx = Builder::with_cx(cx);
221        unsafe {
222            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
223        }
224        bx
225    }
226
227    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
228        self.cx
229    }
230
231    fn llbb(&self) -> &'ll BasicBlock {
232        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
233    }
234
235    fn set_span(&mut self, _span: Span) {}
236
237    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
238        unsafe {
239            let name = SmallCStr::new(name);
240            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
241        }
242    }
243
244    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
245        Self::append_block(self.cx, self.llfn(), name)
246    }
247
248    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
249        *self = Self::build(self.cx, llbb)
250    }
251
252    fn ret_void(&mut self) {
253        llvm::LLVMBuildRetVoid(self.llbuilder);
254    }
255
256    fn ret(&mut self, v: &'ll Value) {
257        unsafe {
258            llvm::LLVMBuildRet(self.llbuilder, v);
259        }
260    }
261
262    fn br(&mut self, dest: &'ll BasicBlock) {
263        unsafe {
264            llvm::LLVMBuildBr(self.llbuilder, dest);
265        }
266    }
267
268    fn cond_br(
269        &mut self,
270        cond: &'ll Value,
271        then_llbb: &'ll BasicBlock,
272        else_llbb: &'ll BasicBlock,
273    ) {
274        unsafe {
275            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
276        }
277    }
278
279    fn switch(
280        &mut self,
281        v: &'ll Value,
282        else_llbb: &'ll BasicBlock,
283        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
284    ) {
285        let switch =
286            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
287        for (on_val, dest) in cases {
288            let on_val = self.const_uint_big(self.val_ty(v), on_val);
289            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
290        }
291    }
292
293    fn switch_with_weights(
294        &mut self,
295        v: Self::Value,
296        else_llbb: Self::BasicBlock,
297        else_is_cold: bool,
298        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
299    ) {
300        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
301            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
302            return;
303        }
304
305        let id_str = "branch_weights";
306        let id = unsafe {
307            llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len())
308        };
309
310        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
311        // This function handles switch instructions with more than 2 targets and it needs to
312        // emit branch weights metadata instead of using the intrinsic.
313        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
314        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
315        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
316        let weight =
317            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
318
319        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
320        md.push(id);
321        md.push(weight(else_is_cold));
322
323        let switch =
324            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
325        for (on_val, dest, is_cold) in cases {
326            let on_val = self.const_uint_big(self.val_ty(v), on_val);
327            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
328            md.push(weight(is_cold));
329        }
330
331        unsafe {
332            let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
333            self.cx.set_metadata(switch, llvm::MD_prof, md_node);
334        }
335    }
336
337    fn invoke(
338        &mut self,
339        llty: &'ll Type,
340        fn_attrs: Option<&CodegenFnAttrs>,
341        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
342        llfn: &'ll Value,
343        args: &[&'ll Value],
344        then: &'ll BasicBlock,
345        catch: &'ll BasicBlock,
346        funclet: Option<&Funclet<'ll>>,
347        instance: Option<Instance<'tcx>>,
348    ) -> &'ll Value {
349        debug!("invoke {:?} with args ({:?})", llfn, args);
350
351        let args = self.check_call("invoke", llty, llfn, args);
352        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
353        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
354        if let Some(funclet_bundle) = funclet_bundle {
355            bundles.push(funclet_bundle);
356        }
357
358        // Emit CFI pointer type membership test
359        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
360
361        // Emit KCFI operand bundle
362        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
363        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) {
364            bundles.push(kcfi_bundle);
365        }
366
367        let invoke = unsafe {
368            llvm::LLVMBuildInvokeWithOperandBundles(
369                self.llbuilder,
370                llty,
371                llfn,
372                args.as_ptr(),
373                args.len() as c_uint,
374                then,
375                catch,
376                bundles.as_ptr(),
377                bundles.len() as c_uint,
378                UNNAMED,
379            )
380        };
381        if let Some(fn_abi) = fn_abi {
382            fn_abi.apply_attrs_callsite(self, invoke);
383        }
384        invoke
385    }
386
387    fn unreachable(&mut self) {
388        unsafe {
389            llvm::LLVMBuildUnreachable(self.llbuilder);
390        }
391    }
392
393    math_builder_methods! {
394        add(a, b) => LLVMBuildAdd,
395        fadd(a, b) => LLVMBuildFAdd,
396        sub(a, b) => LLVMBuildSub,
397        fsub(a, b) => LLVMBuildFSub,
398        mul(a, b) => LLVMBuildMul,
399        fmul(a, b) => LLVMBuildFMul,
400        udiv(a, b) => LLVMBuildUDiv,
401        exactudiv(a, b) => LLVMBuildExactUDiv,
402        sdiv(a, b) => LLVMBuildSDiv,
403        exactsdiv(a, b) => LLVMBuildExactSDiv,
404        fdiv(a, b) => LLVMBuildFDiv,
405        urem(a, b) => LLVMBuildURem,
406        srem(a, b) => LLVMBuildSRem,
407        frem(a, b) => LLVMBuildFRem,
408        shl(a, b) => LLVMBuildShl,
409        lshr(a, b) => LLVMBuildLShr,
410        ashr(a, b) => LLVMBuildAShr,
411        and(a, b) => LLVMBuildAnd,
412        or(a, b) => LLVMBuildOr,
413        xor(a, b) => LLVMBuildXor,
414        neg(x) => LLVMBuildNeg,
415        fneg(x) => LLVMBuildFNeg,
416        not(x) => LLVMBuildNot,
417        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
418        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
419        unchecked_ssub(x, y) => LLVMBuildNSWSub,
420        unchecked_usub(x, y) => LLVMBuildNUWSub,
421        unchecked_smul(x, y) => LLVMBuildNSWMul,
422        unchecked_umul(x, y) => LLVMBuildNUWMul,
423    }
424
425    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
426        unsafe {
427            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
428            if llvm::LLVMIsAInstruction(add).is_some() {
429                llvm::LLVMSetNUW(add, True);
430                llvm::LLVMSetNSW(add, True);
431            }
432            add
433        }
434    }
435    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
436        unsafe {
437            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
438            if llvm::LLVMIsAInstruction(sub).is_some() {
439                llvm::LLVMSetNUW(sub, True);
440                llvm::LLVMSetNSW(sub, True);
441            }
442            sub
443        }
444    }
445    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
446        unsafe {
447            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
448            if llvm::LLVMIsAInstruction(mul).is_some() {
449                llvm::LLVMSetNUW(mul, True);
450                llvm::LLVMSetNSW(mul, True);
451            }
452            mul
453        }
454    }
455
456    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
457        unsafe {
458            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
459
460            // If a and b are both values, then `or` is a value, rather than
461            // an instruction, so we need to check before setting the flag.
462            // (See also `LLVMBuildNUWNeg` which also needs a check.)
463            if llvm::LLVMIsAInstruction(or).is_some() {
464                llvm::LLVMSetIsDisjoint(or, True);
465            }
466            or
467        }
468    }
469
470    set_math_builder_methods! {
471        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
472        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
473        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
474        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
475        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
476        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
477        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
478        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
479        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
480        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
481    }
482
483    fn checked_binop(
484        &mut self,
485        oop: OverflowOp,
486        ty: Ty<'_>,
487        lhs: Self::Value,
488        rhs: Self::Value,
489    ) -> (Self::Value, Self::Value) {
490        use rustc_middle::ty::IntTy::*;
491        use rustc_middle::ty::UintTy::*;
492        use rustc_middle::ty::{Int, Uint};
493
494        let new_kind = match ty.kind() {
495            Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)),
496            Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)),
497            t @ (Uint(_) | Int(_)) => *t,
498            _ => panic!("tried to get overflow intrinsic for op applied to non-int type"),
499        };
500
501        let name = match oop {
502            OverflowOp::Add => match new_kind {
503                Int(I8) => "llvm.sadd.with.overflow.i8",
504                Int(I16) => "llvm.sadd.with.overflow.i16",
505                Int(I32) => "llvm.sadd.with.overflow.i32",
506                Int(I64) => "llvm.sadd.with.overflow.i64",
507                Int(I128) => "llvm.sadd.with.overflow.i128",
508
509                Uint(U8) => "llvm.uadd.with.overflow.i8",
510                Uint(U16) => "llvm.uadd.with.overflow.i16",
511                Uint(U32) => "llvm.uadd.with.overflow.i32",
512                Uint(U64) => "llvm.uadd.with.overflow.i64",
513                Uint(U128) => "llvm.uadd.with.overflow.i128",
514
515                _ => unreachable!(),
516            },
517            OverflowOp::Sub => match new_kind {
518                Int(I8) => "llvm.ssub.with.overflow.i8",
519                Int(I16) => "llvm.ssub.with.overflow.i16",
520                Int(I32) => "llvm.ssub.with.overflow.i32",
521                Int(I64) => "llvm.ssub.with.overflow.i64",
522                Int(I128) => "llvm.ssub.with.overflow.i128",
523
524                Uint(_) => {
525                    // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
526                    // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
527                    // in the backend if profitable.
528                    let sub = self.sub(lhs, rhs);
529                    let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
530                    return (sub, cmp);
531                }
532
533                _ => unreachable!(),
534            },
535            OverflowOp::Mul => match new_kind {
536                Int(I8) => "llvm.smul.with.overflow.i8",
537                Int(I16) => "llvm.smul.with.overflow.i16",
538                Int(I32) => "llvm.smul.with.overflow.i32",
539                Int(I64) => "llvm.smul.with.overflow.i64",
540                Int(I128) => "llvm.smul.with.overflow.i128",
541
542                Uint(U8) => "llvm.umul.with.overflow.i8",
543                Uint(U16) => "llvm.umul.with.overflow.i16",
544                Uint(U32) => "llvm.umul.with.overflow.i32",
545                Uint(U64) => "llvm.umul.with.overflow.i64",
546                Uint(U128) => "llvm.umul.with.overflow.i128",
547
548                _ => unreachable!(),
549            },
550        };
551
552        let res = self.call_intrinsic(name, &[lhs, rhs]);
553        (self.extract_value(res, 0), self.extract_value(res, 1))
554    }
555
556    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
557        if self.cx().val_ty(val) == self.cx().type_i1() {
558            self.zext(val, self.cx().type_i8())
559        } else {
560            val
561        }
562    }
563
564    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
565        if scalar.is_bool() {
566            return self.unchecked_utrunc(val, self.cx().type_i1());
567        }
568        val
569    }
570
571    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
572        let mut bx = Builder::with_cx(self.cx);
573        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
574        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
575        unsafe {
576            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
577            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
578            // Cast to default addrspace if necessary
579            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
580        }
581    }
582
583    fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value {
584        unsafe {
585            let alloca =
586                llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED);
587            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
588            // Cast to default addrspace if necessary
589            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
590        }
591    }
592
593    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
594        unsafe {
595            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
596            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
597            load
598        }
599    }
600
601    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
602        unsafe {
603            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
604            llvm::LLVMSetVolatile(load, llvm::True);
605            load
606        }
607    }
608
609    fn atomic_load(
610        &mut self,
611        ty: &'ll Type,
612        ptr: &'ll Value,
613        order: rustc_codegen_ssa::common::AtomicOrdering,
614        size: Size,
615    ) -> &'ll Value {
616        unsafe {
617            let load = llvm::LLVMRustBuildAtomicLoad(
618                self.llbuilder,
619                ty,
620                ptr,
621                UNNAMED,
622                AtomicOrdering::from_generic(order),
623            );
624            // LLVM requires the alignment of atomic loads to be at least the size of the type.
625            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
626            load
627        }
628    }
629
630    #[instrument(level = "trace", skip(self))]
631    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
632        if place.layout.is_unsized() {
633            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
634            if matches!(tail.kind(), ty::Foreign(..)) {
635                // Unsized locals and, at least conceptually, even unsized arguments must be copied
636                // around, which requires dynamically determining their size. Therefore, we cannot
637                // allow `extern` types here. Consult t-opsem before removing this check.
638                panic!("unsized locals must not be `extern` types");
639            }
640        }
641        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
642
643        if place.layout.is_zst() {
644            return OperandRef::zero_sized(place.layout);
645        }
646
647        #[instrument(level = "trace", skip(bx))]
648        fn scalar_load_metadata<'a, 'll, 'tcx>(
649            bx: &mut Builder<'a, 'll, 'tcx>,
650            load: &'ll Value,
651            scalar: abi::Scalar,
652            layout: TyAndLayout<'tcx>,
653            offset: Size,
654        ) {
655            if bx.cx.sess().opts.optimize == OptLevel::No {
656                // Don't emit metadata we're not going to use
657                return;
658            }
659
660            if !scalar.is_uninit_valid() {
661                bx.noundef_metadata(load);
662            }
663
664            match scalar.primitive() {
665                abi::Primitive::Int(..) => {
666                    if !scalar.is_always_valid(bx) {
667                        bx.range_metadata(load, scalar.valid_range(bx));
668                    }
669                }
670                abi::Primitive::Pointer(_) => {
671                    if !scalar.valid_range(bx).contains(0) {
672                        bx.nonnull_metadata(load);
673                    }
674
675                    if let Some(pointee) = layout.pointee_info_at(bx, offset) {
676                        if let Some(_) = pointee.safe {
677                            bx.align_metadata(load, pointee.align);
678                        }
679                    }
680                }
681                abi::Primitive::Float(_) => {}
682            }
683        }
684
685        let val = if let Some(_) = place.val.llextra {
686            // FIXME: Merge with the `else` below?
687            OperandValue::Ref(place.val)
688        } else if place.layout.is_llvm_immediate() {
689            let mut const_llval = None;
690            let llty = place.layout.llvm_type(self);
691            unsafe {
692                if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
693                    if llvm::LLVMIsGlobalConstant(global) == llvm::True {
694                        if let Some(init) = llvm::LLVMGetInitializer(global) {
695                            if self.val_ty(init) == llty {
696                                const_llval = Some(init);
697                            }
698                        }
699                    }
700                }
701            }
702            let llval = const_llval.unwrap_or_else(|| {
703                let load = self.load(llty, place.val.llval, place.val.align);
704                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
705                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
706                    self.to_immediate_scalar(load, scalar)
707                } else {
708                    load
709                }
710            });
711            OperandValue::Immediate(llval)
712        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
713            let b_offset = a.size(self).align_to(b.align(self).abi);
714
715            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
716                let llptr = if i == 0 {
717                    place.val.llval
718                } else {
719                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
720                };
721                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
722                let load = self.load(llty, llptr, align);
723                scalar_load_metadata(self, load, scalar, layout, offset);
724                self.to_immediate_scalar(load, scalar)
725            };
726
727            OperandValue::Pair(
728                load(0, a, place.layout, place.val.align, Size::ZERO),
729                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
730            )
731        } else {
732            OperandValue::Ref(place.val)
733        };
734
735        OperandRef { val, layout: place.layout }
736    }
737
738    fn write_operand_repeatedly(
739        &mut self,
740        cg_elem: OperandRef<'tcx, &'ll Value>,
741        count: u64,
742        dest: PlaceRef<'tcx, &'ll Value>,
743    ) {
744        let zero = self.const_usize(0);
745        let count = self.const_usize(count);
746
747        let header_bb = self.append_sibling_block("repeat_loop_header");
748        let body_bb = self.append_sibling_block("repeat_loop_body");
749        let next_bb = self.append_sibling_block("repeat_loop_next");
750
751        self.br(header_bb);
752
753        let mut header_bx = Self::build(self.cx, header_bb);
754        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
755
756        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
757        header_bx.cond_br(keep_going, body_bb, next_bb);
758
759        let mut body_bx = Self::build(self.cx, body_bb);
760        let dest_elem = dest.project_index(&mut body_bx, i);
761        cg_elem.val.store(&mut body_bx, dest_elem);
762
763        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
764        body_bx.br(header_bb);
765        header_bx.add_incoming_to_phi(i, next, body_bb);
766
767        *self = Self::build(self.cx, next_bb);
768    }
769
770    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
771        if self.cx.sess().opts.optimize == OptLevel::No {
772            // Don't emit metadata we're not going to use
773            return;
774        }
775
776        unsafe {
777            let llty = self.cx.val_ty(load);
778            let md = [
779                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
780                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
781            ];
782            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
783            self.set_metadata(load, llvm::MD_range, md);
784        }
785    }
786
787    fn nonnull_metadata(&mut self, load: &'ll Value) {
788        unsafe {
789            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
790            self.set_metadata(load, llvm::MD_nonnull, md);
791        }
792    }
793
794    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
795        self.store_with_flags(val, ptr, align, MemFlags::empty())
796    }
797
798    fn store_with_flags(
799        &mut self,
800        val: &'ll Value,
801        ptr: &'ll Value,
802        align: Align,
803        flags: MemFlags,
804    ) -> &'ll Value {
805        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
806        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
807        unsafe {
808            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
809            let align =
810                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
811            llvm::LLVMSetAlignment(store, align);
812            if flags.contains(MemFlags::VOLATILE) {
813                llvm::LLVMSetVolatile(store, llvm::True);
814            }
815            if flags.contains(MemFlags::NONTEMPORAL) {
816                // Make sure that the current target architectures supports "sane" non-temporal
817                // stores, i.e., non-temporal stores that are equivalent to regular stores except
818                // for performance. LLVM doesn't seem to care about this, and will happily treat
819                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
820                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
821                // regular stores but require special fences. So we keep a list of architectures
822                // where `!nontemporal` is known to be truly just a hint, and use regular stores
823                // everywhere else. (In the future, we could alternatively ensure that an sfence
824                // gets emitted after a sequence of movnt before any kind of synchronizing
825                // operation. But it's not clear how to do that with LLVM.)
826                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
827                // <https://github.com/llvm/llvm-project/issues/64521>.
828                const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
829                    &["aarch64", "arm", "riscv32", "riscv64"];
830
831                let use_nontemporal =
832                    WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
833                if use_nontemporal {
834                    // According to LLVM [1] building a nontemporal store must
835                    // *always* point to a metadata value of the integer 1.
836                    //
837                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
838                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
839                    let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
840                    self.set_metadata(store, llvm::MD_nontemporal, md);
841                }
842            }
843            store
844        }
845    }
846
847    fn atomic_store(
848        &mut self,
849        val: &'ll Value,
850        ptr: &'ll Value,
851        order: rustc_codegen_ssa::common::AtomicOrdering,
852        size: Size,
853    ) {
854        debug!("Store {:?} -> {:?}", val, ptr);
855        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
856        unsafe {
857            let store = llvm::LLVMRustBuildAtomicStore(
858                self.llbuilder,
859                val,
860                ptr,
861                AtomicOrdering::from_generic(order),
862            );
863            // LLVM requires the alignment of atomic stores to be at least the size of the type.
864            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
865        }
866    }
867
868    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
869        unsafe {
870            llvm::LLVMBuildGEPWithNoWrapFlags(
871                self.llbuilder,
872                ty,
873                ptr,
874                indices.as_ptr(),
875                indices.len() as c_uint,
876                UNNAMED,
877                GEPNoWrapFlags::default(),
878            )
879        }
880    }
881
882    fn inbounds_gep(
883        &mut self,
884        ty: &'ll Type,
885        ptr: &'ll Value,
886        indices: &[&'ll Value],
887    ) -> &'ll Value {
888        unsafe {
889            llvm::LLVMBuildGEPWithNoWrapFlags(
890                self.llbuilder,
891                ty,
892                ptr,
893                indices.as_ptr(),
894                indices.len() as c_uint,
895                UNNAMED,
896                GEPNoWrapFlags::InBounds,
897            )
898        }
899    }
900
901    fn inbounds_nuw_gep(
902        &mut self,
903        ty: &'ll Type,
904        ptr: &'ll Value,
905        indices: &[&'ll Value],
906    ) -> &'ll Value {
907        unsafe {
908            llvm::LLVMBuildGEPWithNoWrapFlags(
909                self.llbuilder,
910                ty,
911                ptr,
912                indices.as_ptr(),
913                indices.len() as c_uint,
914                UNNAMED,
915                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
916            )
917        }
918    }
919
920    /* Casts */
921    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
922        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
923    }
924
925    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
926        debug_assert_ne!(self.val_ty(val), dest_ty);
927
928        let trunc = self.trunc(val, dest_ty);
929        if llvm_util::get_version() >= (19, 0, 0) {
930            unsafe {
931                if llvm::LLVMIsAInstruction(trunc).is_some() {
932                    llvm::LLVMSetNUW(trunc, True);
933                }
934            }
935        }
936        trunc
937    }
938
939    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
940        debug_assert_ne!(self.val_ty(val), dest_ty);
941
942        let trunc = self.trunc(val, dest_ty);
943        if llvm_util::get_version() >= (19, 0, 0) {
944            unsafe {
945                if llvm::LLVMIsAInstruction(trunc).is_some() {
946                    llvm::LLVMSetNSW(trunc, True);
947                }
948            }
949        }
950        trunc
951    }
952
953    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
954        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
955    }
956
957    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
958        self.fptoint_sat(false, val, dest_ty)
959    }
960
961    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
962        self.fptoint_sat(true, val, dest_ty)
963    }
964
965    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
966        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
967        // poor codegen. The reason for this is that the corresponding wasm
968        // instructions, `i32.trunc_f32_s` for example, will trap when the float
969        // is out-of-bounds, infinity, or nan. This means that LLVM
970        // automatically inserts control flow around `fptoui` and `fptosi`
971        // because the LLVM instruction `fptoui` is defined as producing a
972        // poison value, not having UB on out-of-bounds values.
973        //
974        // This method, however, is only used with non-saturating casts that
975        // have UB on out-of-bounds values. This means that it's ok if we use
976        // the raw wasm instruction since out-of-bounds values can do whatever
977        // we like. To ensure that LLVM picks the right instruction we choose
978        // the raw wasm intrinsic functions which avoid LLVM inserting all the
979        // other control flow automatically.
980        if self.sess().target.is_like_wasm {
981            let src_ty = self.cx.val_ty(val);
982            if self.cx.type_kind(src_ty) != TypeKind::Vector {
983                let float_width = self.cx.float_width(src_ty);
984                let int_width = self.cx.int_width(dest_ty);
985                let name = match (int_width, float_width) {
986                    (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"),
987                    (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"),
988                    (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"),
989                    (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"),
990                    _ => None,
991                };
992                if let Some(name) = name {
993                    return self.call_intrinsic(name, &[val]);
994                }
995            }
996        }
997        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
998    }
999
1000    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1001        // see `fptoui` above for why wasm is different here
1002        if self.sess().target.is_like_wasm {
1003            let src_ty = self.cx.val_ty(val);
1004            if self.cx.type_kind(src_ty) != TypeKind::Vector {
1005                let float_width = self.cx.float_width(src_ty);
1006                let int_width = self.cx.int_width(dest_ty);
1007                let name = match (int_width, float_width) {
1008                    (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"),
1009                    (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"),
1010                    (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"),
1011                    (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"),
1012                    _ => None,
1013                };
1014                if let Some(name) = name {
1015                    return self.call_intrinsic(name, &[val]);
1016                }
1017            }
1018        }
1019        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
1020    }
1021
1022    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1023        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1024    }
1025
1026    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1027        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1028    }
1029
1030    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1031        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
1032    }
1033
1034    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1035        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
1036    }
1037
1038    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1039        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
1040    }
1041
1042    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1043        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
1044    }
1045
1046    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1047        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
1048    }
1049
1050    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1051        unsafe {
1052            llvm::LLVMBuildIntCast2(
1053                self.llbuilder,
1054                val,
1055                dest_ty,
1056                if is_signed { True } else { False },
1057                UNNAMED,
1058            )
1059        }
1060    }
1061
1062    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1063        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1064    }
1065
1066    /* Comparisons */
1067    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1068        let op = llvm::IntPredicate::from_generic(op);
1069        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1070    }
1071
1072    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1073        let op = llvm::RealPredicate::from_generic(op);
1074        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1075    }
1076
1077    /* Miscellaneous instructions */
1078    fn memcpy(
1079        &mut self,
1080        dst: &'ll Value,
1081        dst_align: Align,
1082        src: &'ll Value,
1083        src_align: Align,
1084        size: &'ll Value,
1085        flags: MemFlags,
1086    ) {
1087        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1088        let size = self.intcast(size, self.type_isize(), false);
1089        let is_volatile = flags.contains(MemFlags::VOLATILE);
1090        unsafe {
1091            llvm::LLVMRustBuildMemCpy(
1092                self.llbuilder,
1093                dst,
1094                dst_align.bytes() as c_uint,
1095                src,
1096                src_align.bytes() as c_uint,
1097                size,
1098                is_volatile,
1099            );
1100        }
1101    }
1102
1103    fn memmove(
1104        &mut self,
1105        dst: &'ll Value,
1106        dst_align: Align,
1107        src: &'ll Value,
1108        src_align: Align,
1109        size: &'ll Value,
1110        flags: MemFlags,
1111    ) {
1112        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1113        let size = self.intcast(size, self.type_isize(), false);
1114        let is_volatile = flags.contains(MemFlags::VOLATILE);
1115        unsafe {
1116            llvm::LLVMRustBuildMemMove(
1117                self.llbuilder,
1118                dst,
1119                dst_align.bytes() as c_uint,
1120                src,
1121                src_align.bytes() as c_uint,
1122                size,
1123                is_volatile,
1124            );
1125        }
1126    }
1127
1128    fn memset(
1129        &mut self,
1130        ptr: &'ll Value,
1131        fill_byte: &'ll Value,
1132        size: &'ll Value,
1133        align: Align,
1134        flags: MemFlags,
1135    ) {
1136        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1137        let is_volatile = flags.contains(MemFlags::VOLATILE);
1138        unsafe {
1139            llvm::LLVMRustBuildMemSet(
1140                self.llbuilder,
1141                ptr,
1142                align.bytes() as c_uint,
1143                fill_byte,
1144                size,
1145                is_volatile,
1146            );
1147        }
1148    }
1149
1150    fn select(
1151        &mut self,
1152        cond: &'ll Value,
1153        then_val: &'ll Value,
1154        else_val: &'ll Value,
1155    ) -> &'ll Value {
1156        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1157    }
1158
1159    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1160        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1161    }
1162
1163    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1164        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1165    }
1166
1167    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1168        unsafe {
1169            let elt_ty = self.cx.val_ty(elt);
1170            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1171            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1172            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1173            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1174        }
1175    }
1176
1177    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1178        assert_eq!(idx as c_uint as u64, idx);
1179        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1180    }
1181
1182    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1183        assert_eq!(idx as c_uint as u64, idx);
1184        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1185    }
1186
1187    fn set_personality_fn(&mut self, personality: &'ll Value) {
1188        unsafe {
1189            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1190        }
1191    }
1192
1193    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1194        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1195        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1196        unsafe {
1197            llvm::LLVMSetCleanup(landing_pad, llvm::True);
1198        }
1199        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1200    }
1201
1202    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1203        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1204        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1205        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1206        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1207    }
1208
1209    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1210        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1211        let mut exn = self.const_poison(ty);
1212        exn = self.insert_value(exn, exn0, 0);
1213        exn = self.insert_value(exn, exn1, 1);
1214        unsafe {
1215            llvm::LLVMBuildResume(self.llbuilder, exn);
1216        }
1217    }
1218
1219    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1220        let ret = unsafe {
1221            llvm::LLVMBuildCleanupPad(
1222                self.llbuilder,
1223                parent,
1224                args.as_ptr(),
1225                args.len() as c_uint,
1226                c"cleanuppad".as_ptr(),
1227            )
1228        };
1229        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1230    }
1231
1232    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1233        unsafe {
1234            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1235                .expect("LLVM does not have support for cleanupret");
1236        }
1237    }
1238
1239    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1240        let ret = unsafe {
1241            llvm::LLVMBuildCatchPad(
1242                self.llbuilder,
1243                parent,
1244                args.as_ptr(),
1245                args.len() as c_uint,
1246                c"catchpad".as_ptr(),
1247            )
1248        };
1249        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1250    }
1251
1252    fn catch_switch(
1253        &mut self,
1254        parent: Option<&'ll Value>,
1255        unwind: Option<&'ll BasicBlock>,
1256        handlers: &[&'ll BasicBlock],
1257    ) -> &'ll Value {
1258        let ret = unsafe {
1259            llvm::LLVMBuildCatchSwitch(
1260                self.llbuilder,
1261                parent,
1262                unwind,
1263                handlers.len() as c_uint,
1264                c"catchswitch".as_ptr(),
1265            )
1266        };
1267        let ret = ret.expect("LLVM does not have support for catchswitch");
1268        for handler in handlers {
1269            unsafe {
1270                llvm::LLVMAddHandler(ret, handler);
1271            }
1272        }
1273        ret
1274    }
1275
1276    // Atomic Operations
1277    fn atomic_cmpxchg(
1278        &mut self,
1279        dst: &'ll Value,
1280        cmp: &'ll Value,
1281        src: &'ll Value,
1282        order: rustc_codegen_ssa::common::AtomicOrdering,
1283        failure_order: rustc_codegen_ssa::common::AtomicOrdering,
1284        weak: bool,
1285    ) -> (&'ll Value, &'ll Value) {
1286        let weak = if weak { llvm::True } else { llvm::False };
1287        unsafe {
1288            let value = llvm::LLVMBuildAtomicCmpXchg(
1289                self.llbuilder,
1290                dst,
1291                cmp,
1292                src,
1293                AtomicOrdering::from_generic(order),
1294                AtomicOrdering::from_generic(failure_order),
1295                llvm::False, // SingleThreaded
1296            );
1297            llvm::LLVMSetWeak(value, weak);
1298            let val = self.extract_value(value, 0);
1299            let success = self.extract_value(value, 1);
1300            (val, success)
1301        }
1302    }
1303
1304    fn atomic_rmw(
1305        &mut self,
1306        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1307        dst: &'ll Value,
1308        mut src: &'ll Value,
1309        order: rustc_codegen_ssa::common::AtomicOrdering,
1310    ) -> &'ll Value {
1311        // The only RMW operation that LLVM supports on pointers is compare-exchange.
1312        let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
1313            && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg;
1314        if requires_cast_to_int {
1315            src = self.ptrtoint(src, self.type_isize());
1316        }
1317        let mut res = unsafe {
1318            llvm::LLVMBuildAtomicRMW(
1319                self.llbuilder,
1320                AtomicRmwBinOp::from_generic(op),
1321                dst,
1322                src,
1323                AtomicOrdering::from_generic(order),
1324                llvm::False, // SingleThreaded
1325            )
1326        };
1327        if requires_cast_to_int {
1328            res = self.inttoptr(res, self.type_ptr());
1329        }
1330        res
1331    }
1332
1333    fn atomic_fence(
1334        &mut self,
1335        order: rustc_codegen_ssa::common::AtomicOrdering,
1336        scope: SynchronizationScope,
1337    ) {
1338        let single_threaded = match scope {
1339            SynchronizationScope::SingleThread => llvm::True,
1340            SynchronizationScope::CrossThread => llvm::False,
1341        };
1342        unsafe {
1343            llvm::LLVMBuildFence(
1344                self.llbuilder,
1345                AtomicOrdering::from_generic(order),
1346                single_threaded,
1347                UNNAMED,
1348            );
1349        }
1350    }
1351
1352    fn set_invariant_load(&mut self, load: &'ll Value) {
1353        unsafe {
1354            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1355            self.set_metadata(load, llvm::MD_invariant_load, md);
1356        }
1357    }
1358
1359    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1360        self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size);
1361    }
1362
1363    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1364        self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
1365    }
1366
1367    fn call(
1368        &mut self,
1369        llty: &'ll Type,
1370        fn_attrs: Option<&CodegenFnAttrs>,
1371        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1372        llfn: &'ll Value,
1373        args: &[&'ll Value],
1374        funclet: Option<&Funclet<'ll>>,
1375        instance: Option<Instance<'tcx>>,
1376    ) -> &'ll Value {
1377        debug!("call {:?} with args ({:?})", llfn, args);
1378
1379        let args = self.check_call("call", llty, llfn, args);
1380        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1381        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1382        if let Some(funclet_bundle) = funclet_bundle {
1383            bundles.push(funclet_bundle);
1384        }
1385
1386        // Emit CFI pointer type membership test
1387        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1388
1389        // Emit KCFI operand bundle
1390        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1391        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) {
1392            bundles.push(kcfi_bundle);
1393        }
1394
1395        let call = unsafe {
1396            llvm::LLVMBuildCallWithOperandBundles(
1397                self.llbuilder,
1398                llty,
1399                llfn,
1400                args.as_ptr() as *const &llvm::Value,
1401                args.len() as c_uint,
1402                bundles.as_ptr(),
1403                bundles.len() as c_uint,
1404                c"".as_ptr(),
1405            )
1406        };
1407        if let Some(fn_abi) = fn_abi {
1408            fn_abi.apply_attrs_callsite(self, call);
1409        }
1410        call
1411    }
1412
1413    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1414        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1415    }
1416
1417    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1418        // Cleanup is always the cold path.
1419        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1420        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1421    }
1422}
1423
1424impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1425    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1426        // Forward to the `get_static` method of `CodegenCx`
1427        let s = self.cx().get_static(def_id);
1428        // Cast to default address space if globals are in a different addrspace
1429        self.cx().const_pointercast(s, self.type_ptr())
1430    }
1431}
1432
1433impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1434    pub(crate) fn llfn(&self) -> &'ll Value {
1435        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1436    }
1437}
1438
1439impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1440    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1441        unsafe {
1442            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1443        }
1444    }
1445}
1446impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1447    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1448        unsafe {
1449            let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1450            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
1451            self.set_metadata(load, llvm::MD_align, md);
1452        }
1453    }
1454
1455    fn noundef_metadata(&mut self, load: &'ll Value) {
1456        unsafe {
1457            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1458            self.set_metadata(load, llvm::MD_noundef, md);
1459        }
1460    }
1461
1462    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1463        unsafe {
1464            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1465            self.set_metadata(inst, llvm::MD_unpredictable, md);
1466        }
1467    }
1468}
1469impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1470    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1471        unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1472    }
1473
1474    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1475        unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1476    }
1477
1478    pub(crate) fn insert_element(
1479        &mut self,
1480        vec: &'ll Value,
1481        elt: &'ll Value,
1482        idx: &'ll Value,
1483    ) -> &'ll Value {
1484        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1485    }
1486
1487    pub(crate) fn shuffle_vector(
1488        &mut self,
1489        v1: &'ll Value,
1490        v2: &'ll Value,
1491        mask: &'ll Value,
1492    ) -> &'ll Value {
1493        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1494    }
1495
1496    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1497        unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1498    }
1499    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1500        unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1501    }
1502    pub(crate) fn vector_reduce_fadd_reassoc(
1503        &mut self,
1504        acc: &'ll Value,
1505        src: &'ll Value,
1506    ) -> &'ll Value {
1507        unsafe {
1508            let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1509            llvm::LLVMRustSetAllowReassoc(instr);
1510            instr
1511        }
1512    }
1513    pub(crate) fn vector_reduce_fmul_reassoc(
1514        &mut self,
1515        acc: &'ll Value,
1516        src: &'ll Value,
1517    ) -> &'ll Value {
1518        unsafe {
1519            let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1520            llvm::LLVMRustSetAllowReassoc(instr);
1521            instr
1522        }
1523    }
1524    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1525        unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1526    }
1527    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1528        unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1529    }
1530    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1531        unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1532    }
1533    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1534        unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1535    }
1536    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1537        unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1538    }
1539    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1540        unsafe {
1541            llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
1542        }
1543    }
1544    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1545        unsafe {
1546            llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
1547        }
1548    }
1549    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1550        unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1551    }
1552    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1553        unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1554    }
1555
1556    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1557        unsafe {
1558            llvm::LLVMAddClause(landing_pad, clause);
1559        }
1560    }
1561
1562    pub(crate) fn catch_ret(
1563        &mut self,
1564        funclet: &Funclet<'ll>,
1565        unwind: &'ll BasicBlock,
1566    ) -> &'ll Value {
1567        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1568        ret.expect("LLVM does not have support for catchret")
1569    }
1570
1571    fn check_call<'b>(
1572        &mut self,
1573        typ: &str,
1574        fn_ty: &'ll Type,
1575        llfn: &'ll Value,
1576        args: &'b [&'ll Value],
1577    ) -> Cow<'b, [&'ll Value]> {
1578        assert!(
1579            self.cx.type_kind(fn_ty) == TypeKind::Function,
1580            "builder::{typ} not passed a function, but {fn_ty:?}"
1581        );
1582
1583        let param_tys = self.cx.func_params_types(fn_ty);
1584
1585        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1586            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1587
1588        if all_args_match {
1589            return Cow::Borrowed(args);
1590        }
1591
1592        let casted_args: Vec<_> = iter::zip(param_tys, args)
1593            .enumerate()
1594            .map(|(i, (expected_ty, &actual_val))| {
1595                let actual_ty = self.cx.val_ty(actual_val);
1596                if expected_ty != actual_ty {
1597                    debug!(
1598                        "type mismatch in function call of {:?}. \
1599                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1600                        llfn, expected_ty, i, actual_ty
1601                    );
1602                    self.bitcast(actual_val, expected_ty)
1603                } else {
1604                    actual_val
1605                }
1606            })
1607            .collect();
1608
1609        Cow::Owned(casted_args)
1610    }
1611
1612    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1613        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1614    }
1615}
1616
1617impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1618    pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value {
1619        let (ty, f) = self.cx.get_intrinsic(intrinsic);
1620        self.call(ty, None, None, f, args, None, None)
1621    }
1622
1623    fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
1624        let size = size.bytes();
1625        if size == 0 {
1626            return;
1627        }
1628
1629        if !self.cx().sess().emit_lifetime_markers() {
1630            return;
1631        }
1632
1633        self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]);
1634    }
1635}
1636impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1637    pub(crate) fn phi(
1638        &mut self,
1639        ty: &'ll Type,
1640        vals: &[&'ll Value],
1641        bbs: &[&'ll BasicBlock],
1642    ) -> &'ll Value {
1643        assert_eq!(vals.len(), bbs.len());
1644        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1645        unsafe {
1646            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1647            phi
1648        }
1649    }
1650
1651    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1652        unsafe {
1653            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1654        }
1655    }
1656}
1657impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1658    fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1659        let src_ty = self.cx.val_ty(val);
1660        let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector {
1661            assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty));
1662            (
1663                self.cx.element_type(src_ty),
1664                self.cx.element_type(dest_ty),
1665                Some(self.cx.vector_length(src_ty)),
1666            )
1667        } else {
1668            (src_ty, dest_ty, None)
1669        };
1670        let float_width = self.cx.float_width(float_ty);
1671        let int_width = self.cx.int_width(int_ty);
1672
1673        let instr = if signed { "fptosi" } else { "fptoui" };
1674        let name = if let Some(vector_length) = vector_length {
1675            format!("llvm.{instr}.sat.v{vector_length}i{int_width}.v{vector_length}f{float_width}")
1676        } else {
1677            format!("llvm.{instr}.sat.i{int_width}.f{float_width}")
1678        };
1679        let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty));
1680        self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None, None)
1681    }
1682
1683    pub(crate) fn landing_pad(
1684        &mut self,
1685        ty: &'ll Type,
1686        pers_fn: &'ll Value,
1687        num_clauses: usize,
1688    ) -> &'ll Value {
1689        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1690        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1691        // personality lives on the parent function anyway.
1692        self.set_personality_fn(pers_fn);
1693        unsafe {
1694            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1695        }
1696    }
1697
1698    pub(crate) fn callbr(
1699        &mut self,
1700        llty: &'ll Type,
1701        fn_attrs: Option<&CodegenFnAttrs>,
1702        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1703        llfn: &'ll Value,
1704        args: &[&'ll Value],
1705        default_dest: &'ll BasicBlock,
1706        indirect_dest: &[&'ll BasicBlock],
1707        funclet: Option<&Funclet<'ll>>,
1708        instance: Option<Instance<'tcx>>,
1709    ) -> &'ll Value {
1710        debug!("invoke {:?} with args ({:?})", llfn, args);
1711
1712        let args = self.check_call("callbr", llty, llfn, args);
1713        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1714        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1715        if let Some(funclet_bundle) = funclet_bundle {
1716            bundles.push(funclet_bundle);
1717        }
1718
1719        // Emit CFI pointer type membership test
1720        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1721
1722        // Emit KCFI operand bundle
1723        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1724        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.raw()) {
1725            bundles.push(kcfi_bundle);
1726        }
1727
1728        let callbr = unsafe {
1729            llvm::LLVMBuildCallBr(
1730                self.llbuilder,
1731                llty,
1732                llfn,
1733                default_dest,
1734                indirect_dest.as_ptr(),
1735                indirect_dest.len() as c_uint,
1736                args.as_ptr(),
1737                args.len() as c_uint,
1738                bundles.as_ptr(),
1739                bundles.len() as c_uint,
1740                UNNAMED,
1741            )
1742        };
1743        if let Some(fn_abi) = fn_abi {
1744            fn_abi.apply_attrs_callsite(self, callbr);
1745        }
1746        callbr
1747    }
1748
1749    // Emits CFI pointer type membership tests.
1750    fn cfi_type_test(
1751        &mut self,
1752        fn_attrs: Option<&CodegenFnAttrs>,
1753        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1754        instance: Option<Instance<'tcx>>,
1755        llfn: &'ll Value,
1756    ) {
1757        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1758        if self.tcx.sess.is_sanitizer_cfi_enabled()
1759            && let Some(fn_abi) = fn_abi
1760            && is_indirect_call
1761        {
1762            if let Some(fn_attrs) = fn_attrs
1763                && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1764            {
1765                return;
1766            }
1767
1768            let mut options = cfi::TypeIdOptions::empty();
1769            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1770                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1771            }
1772            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1773                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1774            }
1775
1776            let typeid = if let Some(instance) = instance {
1777                cfi::typeid_for_instance(self.tcx, instance, options)
1778            } else {
1779                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1780            };
1781            let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
1782            let dbg_loc = self.get_dbg_loc();
1783
1784            // Test whether the function pointer is associated with the type identifier.
1785            let cond = self.type_test(llfn, typeid_metadata);
1786            let bb_pass = self.append_sibling_block("type_test.pass");
1787            let bb_fail = self.append_sibling_block("type_test.fail");
1788            self.cond_br(cond, bb_pass, bb_fail);
1789
1790            self.switch_to_block(bb_fail);
1791            if let Some(dbg_loc) = dbg_loc {
1792                self.set_dbg_loc(dbg_loc);
1793            }
1794            self.abort();
1795            self.unreachable();
1796
1797            self.switch_to_block(bb_pass);
1798            if let Some(dbg_loc) = dbg_loc {
1799                self.set_dbg_loc(dbg_loc);
1800            }
1801        }
1802    }
1803
1804    // Emits KCFI operand bundles.
1805    fn kcfi_operand_bundle(
1806        &mut self,
1807        fn_attrs: Option<&CodegenFnAttrs>,
1808        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1809        instance: Option<Instance<'tcx>>,
1810        llfn: &'ll Value,
1811    ) -> Option<llvm::OperandBundleOwned<'ll>> {
1812        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1813        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1814            && let Some(fn_abi) = fn_abi
1815            && is_indirect_call
1816        {
1817            if let Some(fn_attrs) = fn_attrs
1818                && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1819            {
1820                return None;
1821            }
1822
1823            let mut options = kcfi::TypeIdOptions::empty();
1824            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1825                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1826            }
1827            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1828                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1829            }
1830
1831            let kcfi_typeid = if let Some(instance) = instance {
1832                kcfi::typeid_for_instance(self.tcx, instance, options)
1833            } else {
1834                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1835            };
1836
1837            Some(llvm::OperandBundleOwned::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1838        } else {
1839            None
1840        };
1841        kcfi_bundle
1842    }
1843
1844    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1845    #[instrument(level = "debug", skip(self))]
1846    pub(crate) fn instrprof_increment(
1847        &mut self,
1848        fn_name: &'ll Value,
1849        hash: &'ll Value,
1850        num_counters: &'ll Value,
1851        index: &'ll Value,
1852    ) {
1853        self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]);
1854    }
1855
1856    /// Emits a call to `llvm.instrprof.mcdc.parameters`.
1857    ///
1858    /// This doesn't produce any code directly, but is used as input by
1859    /// the LLVM pass that handles coverage instrumentation.
1860    ///
1861    /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.)
1862    ///
1863    /// [`CodeGenPGO::emitMCDCParameters`]:
1864    ///     https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124
1865    #[instrument(level = "debug", skip(self))]
1866    pub(crate) fn mcdc_parameters(
1867        &mut self,
1868        fn_name: &'ll Value,
1869        hash: &'ll Value,
1870        bitmap_bits: &'ll Value,
1871    ) {
1872        assert!(
1873            crate::llvm_util::get_version() >= (19, 0, 0),
1874            "MCDC intrinsics require LLVM 19 or later"
1875        );
1876        self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]);
1877    }
1878
1879    #[instrument(level = "debug", skip(self))]
1880    pub(crate) fn mcdc_tvbitmap_update(
1881        &mut self,
1882        fn_name: &'ll Value,
1883        hash: &'ll Value,
1884        bitmap_index: &'ll Value,
1885        mcdc_temp: &'ll Value,
1886    ) {
1887        assert!(
1888            crate::llvm_util::get_version() >= (19, 0, 0),
1889            "MCDC intrinsics require LLVM 19 or later"
1890        );
1891        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
1892        self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args);
1893    }
1894
1895    #[instrument(level = "debug", skip(self))]
1896    pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
1897        self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
1898    }
1899
1900    #[instrument(level = "debug", skip(self))]
1901    pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
1902        assert!(
1903            crate::llvm_util::get_version() >= (19, 0, 0),
1904            "MCDC intrinsics require LLVM 19 or later"
1905        );
1906        let align = self.tcx.data_layout.i32_align.abi;
1907        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
1908        let new_tv_index = self.add(current_tv_index, cond_index);
1909        self.store(new_tv_index, mcdc_temp, align);
1910    }
1911}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
