rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5pub(crate) mod autodiff;
6pub(crate) mod gpu_offload;
7
8use libc::{c_char, c_uint, size_t};
9use rustc_abi as abi;
10use rustc_abi::{Align, Size, WrappingRange};
11use rustc_codegen_ssa::MemFlags;
12use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
13use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
14use rustc_codegen_ssa::mir::place::PlaceRef;
15use rustc_codegen_ssa::traits::*;
16use rustc_data_structures::small_c_str::SmallCStr;
17use rustc_hir::def_id::DefId;
18use rustc_middle::bug;
19use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
20use rustc_middle::ty::layout::{
21    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
22    TyAndLayout,
23};
24use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
25use rustc_sanitizers::{cfi, kcfi};
26use rustc_session::config::OptLevel;
27use rustc_span::Span;
28use rustc_target::callconv::{FnAbi, PassMode};
29use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
30use smallvec::SmallVec;
31use tracing::{debug, instrument};
32
33use crate::abi::FnAbiLlvmExt;
34use crate::attributes;
35use crate::common::Funclet;
36use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
37use crate::llvm::{
38    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
39};
40use crate::type_::Type;
41use crate::type_of::LayoutLlvmExt;
42use crate::value::Value;
43
44#[must_use]
45pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
46    pub llbuilder: &'ll mut llvm::Builder<'ll>,
47    pub cx: &'a GenericCx<'ll, CX>,
48}
49
50pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
51pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
52
53impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
54    fn drop(&mut self) {
55        unsafe {
56            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
57        }
58    }
59}
60
61impl<'a, 'll> SBuilder<'a, 'll> {
62    pub(crate) fn call(
63        &mut self,
64        llty: &'ll Type,
65        llfn: &'ll Value,
66        args: &[&'ll Value],
67        funclet: Option<&Funclet<'ll>>,
68    ) -> &'ll Value {
69        debug!("call {:?} with args ({:?})", llfn, args);
70
71        let args = self.check_call("call", llty, llfn, args);
72        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
73        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
74        if let Some(funclet_bundle) = funclet_bundle {
75            bundles.push(funclet_bundle);
76        }
77
78        let call = unsafe {
79            llvm::LLVMBuildCallWithOperandBundles(
80                self.llbuilder,
81                llty,
82                llfn,
83                args.as_ptr() as *const &llvm::Value,
84                args.len() as c_uint,
85                bundles.as_ptr(),
86                bundles.len() as c_uint,
87                c"".as_ptr(),
88            )
89        };
90        call
91    }
92}
93
94impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
95    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
96        // Create a fresh builder from the simple context.
97        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
98        GenericBuilder { llbuilder, cx: scx }
99    }
100
101    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
102        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
103    }
104
105    pub(crate) fn ret_void(&mut self) {
106        llvm::LLVMBuildRetVoid(self.llbuilder);
107    }
108
109    pub(crate) fn ret(&mut self, v: &'ll Value) {
110        unsafe {
111            llvm::LLVMBuildRet(self.llbuilder, v);
112        }
113    }
114
115    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
116        let bx = Self::with_cx(cx);
117        unsafe {
118            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
119        }
120        bx
121    }
122
123    // The generic builder has less functionality and thus (unlike the other alloca) we can not
124    // easily jump to the beginning of the function to place our allocas there. We trust the user
125    // to manually do that. FIXME(offload): improve the genericCx and add more llvm wrappers to
126    // handle this.
127    pub(crate) fn direct_alloca(&mut self, ty: &'ll Type, align: Align, name: &str) -> &'ll Value {
128        let val = unsafe {
129            let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED);
130            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
131            // Cast to default addrspace if necessary
132            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx.type_ptr(), UNNAMED)
133        };
134        if name != "" {
135            let name = std::ffi::CString::new(name).unwrap();
136            llvm::set_value_name(val, &name.as_bytes());
137        }
138        val
139    }
140
141    pub(crate) fn inbounds_gep(
142        &mut self,
143        ty: &'ll Type,
144        ptr: &'ll Value,
145        indices: &[&'ll Value],
146    ) -> &'ll Value {
147        unsafe {
148            llvm::LLVMBuildGEPWithNoWrapFlags(
149                self.llbuilder,
150                ty,
151                ptr,
152                indices.as_ptr(),
153                indices.len() as c_uint,
154                UNNAMED,
155                GEPNoWrapFlags::InBounds,
156            )
157        }
158    }
159
160    pub(crate) fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
161        debug!("Store {:?} -> {:?}", val, ptr);
162        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
163        unsafe {
164            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
165            llvm::LLVMSetAlignment(store, align.bytes() as c_uint);
166            store
167        }
168    }
169
170    pub(crate) fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
171        unsafe {
172            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
173            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
174            load
175        }
176    }
177
178    fn memset(&mut self, ptr: &'ll Value, fill_byte: &'ll Value, size: &'ll Value, align: Align) {
179        unsafe {
180            llvm::LLVMRustBuildMemSet(
181                self.llbuilder,
182                ptr,
183                align.bytes() as c_uint,
184                fill_byte,
185                size,
186                false,
187            );
188        }
189    }
190}
191
192/// Empty string, to be used where LLVM expects an instruction name, indicating
193/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
194// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
195pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
196
197impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
198    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
199    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
200    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
201    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
202    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
203    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
204
205    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
206    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
207    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
208}
209
210impl abi::HasDataLayout for Builder<'_, '_, '_> {
211    fn data_layout(&self) -> &abi::TargetDataLayout {
212        self.cx.data_layout()
213    }
214}
215
216impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
217    #[inline]
218    fn tcx(&self) -> TyCtxt<'tcx> {
219        self.cx.tcx
220    }
221}
222
223impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
224    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
225        self.cx.typing_env()
226    }
227}
228
229impl HasTargetSpec for Builder<'_, '_, '_> {
230    #[inline]
231    fn target_spec(&self) -> &Target {
232        self.cx.target_spec()
233    }
234}
235
236impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
237    #[inline]
238    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
239        self.cx.handle_layout_err(err, span, ty)
240    }
241}
242
243impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
244    #[inline]
245    fn handle_fn_abi_err(
246        &self,
247        err: FnAbiError<'tcx>,
248        span: Span,
249        fn_abi_request: FnAbiRequest<'tcx>,
250    ) -> ! {
251        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
252    }
253}
254
255impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
256    type Target = CodegenCx<'ll, 'tcx>;
257
258    #[inline]
259    fn deref(&self) -> &Self::Target {
260        self.cx
261    }
262}
263
264macro_rules! math_builder_methods {
265    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
266        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
267            unsafe {
268                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
269            }
270        })+
271    }
272}
273
274macro_rules! set_math_builder_methods {
275    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
276        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
277            unsafe {
278                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
279                llvm::$llvm_set_math(instr);
280                instr
281            }
282        })+
283    }
284}
285
286impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
287    type CodegenCx = CodegenCx<'ll, 'tcx>;
288
289    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
290        let bx = Builder::with_cx(cx);
291        unsafe {
292            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
293        }
294        bx
295    }
296
297    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
298        self.cx
299    }
300
301    fn llbb(&self) -> &'ll BasicBlock {
302        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
303    }
304
305    fn set_span(&mut self, _span: Span) {}
306
307    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
308        unsafe {
309            let name = SmallCStr::new(name);
310            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
311        }
312    }
313
314    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
315        Self::append_block(self.cx, self.llfn(), name)
316    }
317
318    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
319        *self = Self::build(self.cx, llbb)
320    }
321
322    fn ret_void(&mut self) {
323        llvm::LLVMBuildRetVoid(self.llbuilder);
324    }
325
326    fn ret(&mut self, v: &'ll Value) {
327        unsafe {
328            llvm::LLVMBuildRet(self.llbuilder, v);
329        }
330    }
331
332    fn br(&mut self, dest: &'ll BasicBlock) {
333        unsafe {
334            llvm::LLVMBuildBr(self.llbuilder, dest);
335        }
336    }
337
338    fn cond_br(
339        &mut self,
340        cond: &'ll Value,
341        then_llbb: &'ll BasicBlock,
342        else_llbb: &'ll BasicBlock,
343    ) {
344        unsafe {
345            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
346        }
347    }
348
349    fn switch(
350        &mut self,
351        v: &'ll Value,
352        else_llbb: &'ll BasicBlock,
353        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
354    ) {
355        let switch =
356            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
357        for (on_val, dest) in cases {
358            let on_val = self.const_uint_big(self.val_ty(v), on_val);
359            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
360        }
361    }
362
363    fn switch_with_weights(
364        &mut self,
365        v: Self::Value,
366        else_llbb: Self::BasicBlock,
367        else_is_cold: bool,
368        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
369    ) {
370        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
371            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
372            return;
373        }
374
375        let id = self.cx.create_metadata(b"branch_weights");
376
377        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
378        // This function handles switch instructions with more than 2 targets and it needs to
379        // emit branch weights metadata instead of using the intrinsic.
380        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
381        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
382        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
383        let weight =
384            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
385
386        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
387        md.push(id);
388        md.push(weight(else_is_cold));
389
390        let switch =
391            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
392        for (on_val, dest, is_cold) in cases {
393            let on_val = self.const_uint_big(self.val_ty(v), on_val);
394            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
395            md.push(weight(is_cold));
396        }
397
398        unsafe {
399            let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
400            self.cx.set_metadata(switch, llvm::MD_prof, md_node);
401        }
402    }
403
404    fn invoke(
405        &mut self,
406        llty: &'ll Type,
407        fn_attrs: Option<&CodegenFnAttrs>,
408        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
409        llfn: &'ll Value,
410        args: &[&'ll Value],
411        then: &'ll BasicBlock,
412        catch: &'ll BasicBlock,
413        funclet: Option<&Funclet<'ll>>,
414        instance: Option<Instance<'tcx>>,
415    ) -> &'ll Value {
416        debug!("invoke {:?} with args ({:?})", llfn, args);
417
418        let args = self.check_call("invoke", llty, llfn, args);
419        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
420        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
421        if let Some(funclet_bundle) = funclet_bundle {
422            bundles.push(funclet_bundle);
423        }
424
425        // Emit CFI pointer type membership test
426        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
427
428        // Emit KCFI operand bundle
429        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
430        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
431            bundles.push(kcfi_bundle);
432        }
433
434        let invoke = unsafe {
435            llvm::LLVMBuildInvokeWithOperandBundles(
436                self.llbuilder,
437                llty,
438                llfn,
439                args.as_ptr(),
440                args.len() as c_uint,
441                then,
442                catch,
443                bundles.as_ptr(),
444                bundles.len() as c_uint,
445                UNNAMED,
446            )
447        };
448        if let Some(fn_abi) = fn_abi {
449            fn_abi.apply_attrs_callsite(self, invoke);
450        }
451        invoke
452    }
453
454    fn unreachable(&mut self) {
455        unsafe {
456            llvm::LLVMBuildUnreachable(self.llbuilder);
457        }
458    }
459
460    math_builder_methods! {
461        add(a, b) => LLVMBuildAdd,
462        fadd(a, b) => LLVMBuildFAdd,
463        sub(a, b) => LLVMBuildSub,
464        fsub(a, b) => LLVMBuildFSub,
465        mul(a, b) => LLVMBuildMul,
466        fmul(a, b) => LLVMBuildFMul,
467        udiv(a, b) => LLVMBuildUDiv,
468        exactudiv(a, b) => LLVMBuildExactUDiv,
469        sdiv(a, b) => LLVMBuildSDiv,
470        exactsdiv(a, b) => LLVMBuildExactSDiv,
471        fdiv(a, b) => LLVMBuildFDiv,
472        urem(a, b) => LLVMBuildURem,
473        srem(a, b) => LLVMBuildSRem,
474        frem(a, b) => LLVMBuildFRem,
475        shl(a, b) => LLVMBuildShl,
476        lshr(a, b) => LLVMBuildLShr,
477        ashr(a, b) => LLVMBuildAShr,
478        and(a, b) => LLVMBuildAnd,
479        or(a, b) => LLVMBuildOr,
480        xor(a, b) => LLVMBuildXor,
481        neg(x) => LLVMBuildNeg,
482        fneg(x) => LLVMBuildFNeg,
483        not(x) => LLVMBuildNot,
484        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
485        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
486        unchecked_ssub(x, y) => LLVMBuildNSWSub,
487        unchecked_usub(x, y) => LLVMBuildNUWSub,
488        unchecked_smul(x, y) => LLVMBuildNSWMul,
489        unchecked_umul(x, y) => LLVMBuildNUWMul,
490    }
491
492    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
493        unsafe {
494            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
495            if llvm::LLVMIsAInstruction(add).is_some() {
496                llvm::LLVMSetNUW(add, True);
497                llvm::LLVMSetNSW(add, True);
498            }
499            add
500        }
501    }
502    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
503        unsafe {
504            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
505            if llvm::LLVMIsAInstruction(sub).is_some() {
506                llvm::LLVMSetNUW(sub, True);
507                llvm::LLVMSetNSW(sub, True);
508            }
509            sub
510        }
511    }
512    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
513        unsafe {
514            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
515            if llvm::LLVMIsAInstruction(mul).is_some() {
516                llvm::LLVMSetNUW(mul, True);
517                llvm::LLVMSetNSW(mul, True);
518            }
519            mul
520        }
521    }
522
523    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
524        unsafe {
525            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
526
527            // If a and b are both values, then `or` is a value, rather than
528            // an instruction, so we need to check before setting the flag.
529            // (See also `LLVMBuildNUWNeg` which also needs a check.)
530            if llvm::LLVMIsAInstruction(or).is_some() {
531                llvm::LLVMSetIsDisjoint(or, True);
532            }
533            or
534        }
535    }
536
537    set_math_builder_methods! {
538        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
539        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
540        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
541        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
542        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
543        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
544        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
545        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
546        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
547        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
548    }
549
550    fn checked_binop(
551        &mut self,
552        oop: OverflowOp,
553        ty: Ty<'tcx>,
554        lhs: Self::Value,
555        rhs: Self::Value,
556    ) -> (Self::Value, Self::Value) {
557        let (size, signed) = ty.int_size_and_signed(self.tcx);
558        let width = size.bits();
559
560        if oop == OverflowOp::Sub && !signed {
561            // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
562            // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
563            // in the backend if profitable.
564            let sub = self.sub(lhs, rhs);
565            let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
566            return (sub, cmp);
567        }
568
569        let oop_str = match oop {
570            OverflowOp::Add => "add",
571            OverflowOp::Sub => "sub",
572            OverflowOp::Mul => "mul",
573        };
574
575        let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
576
577        let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
578        (self.extract_value(res, 0), self.extract_value(res, 1))
579    }
580
581    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
582        if self.cx().val_ty(val) == self.cx().type_i1() {
583            self.zext(val, self.cx().type_i8())
584        } else {
585            val
586        }
587    }
588
589    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
590        if scalar.is_bool() {
591            return self.unchecked_utrunc(val, self.cx().type_i1());
592        }
593        val
594    }
595
596    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
597        let mut bx = Builder::with_cx(self.cx);
598        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
599        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
600        unsafe {
601            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
602            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
603            // Cast to default addrspace if necessary
604            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
605        }
606    }
607
608    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
609        unsafe {
610            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
611            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
612            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
613            load
614        }
615    }
616
617    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
618        unsafe {
619            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
620            llvm::LLVMSetVolatile(load, llvm::True);
621            load
622        }
623    }
624
625    fn atomic_load(
626        &mut self,
627        ty: &'ll Type,
628        ptr: &'ll Value,
629        order: rustc_middle::ty::AtomicOrdering,
630        size: Size,
631    ) -> &'ll Value {
632        unsafe {
633            let load = llvm::LLVMRustBuildAtomicLoad(
634                self.llbuilder,
635                ty,
636                ptr,
637                UNNAMED,
638                AtomicOrdering::from_generic(order),
639            );
640            // LLVM requires the alignment of atomic loads to be at least the size of the type.
641            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
642            load
643        }
644    }
645
646    #[instrument(level = "trace", skip(self))]
647    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
648        if place.layout.is_unsized() {
649            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
650            if matches!(tail.kind(), ty::Foreign(..)) {
651                // Unsized locals and, at least conceptually, even unsized arguments must be copied
652                // around, which requires dynamically determining their size. Therefore, we cannot
653                // allow `extern` types here. Consult t-opsem before removing this check.
654                panic!("unsized locals must not be `extern` types");
655            }
656        }
657        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
658
659        if place.layout.is_zst() {
660            return OperandRef::zero_sized(place.layout);
661        }
662
663        #[instrument(level = "trace", skip(bx))]
664        fn scalar_load_metadata<'a, 'll, 'tcx>(
665            bx: &mut Builder<'a, 'll, 'tcx>,
666            load: &'ll Value,
667            scalar: abi::Scalar,
668            layout: TyAndLayout<'tcx>,
669            offset: Size,
670        ) {
671            if bx.cx.sess().opts.optimize == OptLevel::No {
672                // Don't emit metadata we're not going to use
673                return;
674            }
675
676            if !scalar.is_uninit_valid() {
677                bx.noundef_metadata(load);
678            }
679
680            match scalar.primitive() {
681                abi::Primitive::Int(..) => {
682                    if !scalar.is_always_valid(bx) {
683                        bx.range_metadata(load, scalar.valid_range(bx));
684                    }
685                }
686                abi::Primitive::Pointer(_) => {
687                    if !scalar.valid_range(bx).contains(0) {
688                        bx.nonnull_metadata(load);
689                    }
690
691                    if let Some(pointee) = layout.pointee_info_at(bx, offset)
692                        && let Some(_) = pointee.safe
693                    {
694                        bx.align_metadata(load, pointee.align);
695                    }
696                }
697                abi::Primitive::Float(_) => {}
698            }
699        }
700
701        let val = if let Some(_) = place.val.llextra {
702            // FIXME: Merge with the `else` below?
703            OperandValue::Ref(place.val)
704        } else if place.layout.is_llvm_immediate() {
705            let mut const_llval = None;
706            let llty = place.layout.llvm_type(self);
707            if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
708                if llvm::LLVMIsGlobalConstant(global) == llvm::True {
709                    if let Some(init) = llvm::LLVMGetInitializer(global) {
710                        if self.val_ty(init) == llty {
711                            const_llval = Some(init);
712                        }
713                    }
714                }
715            }
716
717            let llval = const_llval.unwrap_or_else(|| {
718                let load = self.load(llty, place.val.llval, place.val.align);
719                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
720                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
721                    self.to_immediate_scalar(load, scalar)
722                } else {
723                    load
724                }
725            });
726            OperandValue::Immediate(llval)
727        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
728            let b_offset = a.size(self).align_to(b.align(self).abi);
729
730            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
731                let llptr = if i == 0 {
732                    place.val.llval
733                } else {
734                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
735                };
736                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
737                let load = self.load(llty, llptr, align);
738                scalar_load_metadata(self, load, scalar, layout, offset);
739                self.to_immediate_scalar(load, scalar)
740            };
741
742            OperandValue::Pair(
743                load(0, a, place.layout, place.val.align, Size::ZERO),
744                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
745            )
746        } else {
747            OperandValue::Ref(place.val)
748        };
749
750        OperandRef { val, layout: place.layout }
751    }
752
753    fn write_operand_repeatedly(
754        &mut self,
755        cg_elem: OperandRef<'tcx, &'ll Value>,
756        count: u64,
757        dest: PlaceRef<'tcx, &'ll Value>,
758    ) {
759        let zero = self.const_usize(0);
760        let count = self.const_usize(count);
761
762        let header_bb = self.append_sibling_block("repeat_loop_header");
763        let body_bb = self.append_sibling_block("repeat_loop_body");
764        let next_bb = self.append_sibling_block("repeat_loop_next");
765
766        self.br(header_bb);
767
768        let mut header_bx = Self::build(self.cx, header_bb);
769        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
770
771        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
772        header_bx.cond_br(keep_going, body_bb, next_bb);
773
774        let mut body_bx = Self::build(self.cx, body_bb);
775        let dest_elem = dest.project_index(&mut body_bx, i);
776        cg_elem.val.store(&mut body_bx, dest_elem);
777
778        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
779        body_bx.br(header_bb);
780        header_bx.add_incoming_to_phi(i, next, body_bb);
781
782        *self = Self::build(self.cx, next_bb);
783    }
784
785    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
786        if self.cx.sess().opts.optimize == OptLevel::No {
787            // Don't emit metadata we're not going to use
788            return;
789        }
790
791        unsafe {
792            let llty = self.cx.val_ty(load);
793            let md = [
794                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
795                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
796            ];
797            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
798            self.set_metadata(load, llvm::MD_range, md);
799        }
800    }
801
802    fn nonnull_metadata(&mut self, load: &'ll Value) {
803        unsafe {
804            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
805            self.set_metadata(load, llvm::MD_nonnull, md);
806        }
807    }
808
809    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
810        self.store_with_flags(val, ptr, align, MemFlags::empty())
811    }
812
813    fn store_with_flags(
814        &mut self,
815        val: &'ll Value,
816        ptr: &'ll Value,
817        align: Align,
818        flags: MemFlags,
819    ) -> &'ll Value {
820        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
821        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
822        unsafe {
823            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
824            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
825            let align =
826                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
827            llvm::LLVMSetAlignment(store, align);
828            if flags.contains(MemFlags::VOLATILE) {
829                llvm::LLVMSetVolatile(store, llvm::True);
830            }
831            if flags.contains(MemFlags::NONTEMPORAL) {
832                // Make sure that the current target architectures supports "sane" non-temporal
833                // stores, i.e., non-temporal stores that are equivalent to regular stores except
834                // for performance. LLVM doesn't seem to care about this, and will happily treat
835                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
836                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
837                // regular stores but require special fences. So we keep a list of architectures
838                // where `!nontemporal` is known to be truly just a hint, and use regular stores
839                // everywhere else. (In the future, we could alternatively ensure that an sfence
840                // gets emitted after a sequence of movnt before any kind of synchronizing
841                // operation. But it's not clear how to do that with LLVM.)
842                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
843                // <https://github.com/llvm/llvm-project/issues/64521>.
844                const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
845                    &["aarch64", "arm", "riscv32", "riscv64"];
846
847                let use_nontemporal =
848                    WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
849                if use_nontemporal {
850                    // According to LLVM [1] building a nontemporal store must
851                    // *always* point to a metadata value of the integer 1.
852                    //
853                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
854                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
855                    let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
856                    self.set_metadata(store, llvm::MD_nontemporal, md);
857                }
858            }
859            store
860        }
861    }
862
863    fn atomic_store(
864        &mut self,
865        val: &'ll Value,
866        ptr: &'ll Value,
867        order: rustc_middle::ty::AtomicOrdering,
868        size: Size,
869    ) {
870        debug!("Store {:?} -> {:?}", val, ptr);
871        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
872        unsafe {
873            let store = llvm::LLVMRustBuildAtomicStore(
874                self.llbuilder,
875                val,
876                ptr,
877                AtomicOrdering::from_generic(order),
878            );
879            // LLVM requires the alignment of atomic stores to be at least the size of the type.
880            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
881        }
882    }
883
884    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
885        unsafe {
886            llvm::LLVMBuildGEPWithNoWrapFlags(
887                self.llbuilder,
888                ty,
889                ptr,
890                indices.as_ptr(),
891                indices.len() as c_uint,
892                UNNAMED,
893                GEPNoWrapFlags::default(),
894            )
895        }
896    }
897
898    fn inbounds_gep(
899        &mut self,
900        ty: &'ll Type,
901        ptr: &'ll Value,
902        indices: &[&'ll Value],
903    ) -> &'ll Value {
904        unsafe {
905            llvm::LLVMBuildGEPWithNoWrapFlags(
906                self.llbuilder,
907                ty,
908                ptr,
909                indices.as_ptr(),
910                indices.len() as c_uint,
911                UNNAMED,
912                GEPNoWrapFlags::InBounds,
913            )
914        }
915    }
916
917    fn inbounds_nuw_gep(
918        &mut self,
919        ty: &'ll Type,
920        ptr: &'ll Value,
921        indices: &[&'ll Value],
922    ) -> &'ll Value {
923        unsafe {
924            llvm::LLVMBuildGEPWithNoWrapFlags(
925                self.llbuilder,
926                ty,
927                ptr,
928                indices.as_ptr(),
929                indices.len() as c_uint,
930                UNNAMED,
931                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
932            )
933        }
934    }
935
936    /* Casts */
937    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
938        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
939    }
940
941    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
942        debug_assert_ne!(self.val_ty(val), dest_ty);
943
944        let trunc = self.trunc(val, dest_ty);
945        unsafe {
946            if llvm::LLVMIsAInstruction(trunc).is_some() {
947                llvm::LLVMSetNUW(trunc, True);
948            }
949        }
950        trunc
951    }
952
953    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
954        debug_assert_ne!(self.val_ty(val), dest_ty);
955
956        let trunc = self.trunc(val, dest_ty);
957        unsafe {
958            if llvm::LLVMIsAInstruction(trunc).is_some() {
959                llvm::LLVMSetNSW(trunc, True);
960            }
961        }
962        trunc
963    }
964
965    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
966        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
967    }
968
969    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
970        self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
971    }
972
973    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
974        self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
975    }
976
977    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
978        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
979        // poor codegen. The reason for this is that the corresponding wasm
980        // instructions, `i32.trunc_f32_s` for example, will trap when the float
981        // is out-of-bounds, infinity, or nan. This means that LLVM
982        // automatically inserts control flow around `fptoui` and `fptosi`
983        // because the LLVM instruction `fptoui` is defined as producing a
984        // poison value, not having UB on out-of-bounds values.
985        //
986        // This method, however, is only used with non-saturating casts that
987        // have UB on out-of-bounds values. This means that it's ok if we use
988        // the raw wasm instruction since out-of-bounds values can do whatever
989        // we like. To ensure that LLVM picks the right instruction we choose
990        // the raw wasm intrinsic functions which avoid LLVM inserting all the
991        // other control flow automatically.
992        if self.sess().target.is_like_wasm {
993            let src_ty = self.cx.val_ty(val);
994            if self.cx.type_kind(src_ty) != TypeKind::Vector {
995                let float_width = self.cx.float_width(src_ty);
996                let int_width = self.cx.int_width(dest_ty);
997                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
998                    return self.call_intrinsic(
999                        "llvm.wasm.trunc.unsigned",
1000                        &[dest_ty, src_ty],
1001                        &[val],
1002                    );
1003                }
1004            }
1005        }
1006        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
1007    }
1008
1009    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1010        // see `fptoui` above for why wasm is different here
1011        if self.sess().target.is_like_wasm {
1012            let src_ty = self.cx.val_ty(val);
1013            if self.cx.type_kind(src_ty) != TypeKind::Vector {
1014                let float_width = self.cx.float_width(src_ty);
1015                let int_width = self.cx.int_width(dest_ty);
1016                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
1017                    return self.call_intrinsic(
1018                        "llvm.wasm.trunc.signed",
1019                        &[dest_ty, src_ty],
1020                        &[val],
1021                    );
1022                }
1023            }
1024        }
1025        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
1026    }
1027
1028    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1029        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1030    }
1031
1032    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1033        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1034    }
1035
1036    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1037        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
1038    }
1039
1040    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1041        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
1042    }
1043
1044    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1045        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
1046    }
1047
1048    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1049        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
1050    }
1051
1052    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1053        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
1054    }
1055
1056    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1057        unsafe {
1058            llvm::LLVMBuildIntCast2(
1059                self.llbuilder,
1060                val,
1061                dest_ty,
1062                if is_signed { True } else { False },
1063                UNNAMED,
1064            )
1065        }
1066    }
1067
1068    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1069        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1070    }
1071
1072    /* Comparisons */
1073    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1074        let op = llvm::IntPredicate::from_generic(op);
1075        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1076    }
1077
1078    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1079        let op = llvm::RealPredicate::from_generic(op);
1080        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1081    }
1082
1083    fn three_way_compare(
1084        &mut self,
1085        ty: Ty<'tcx>,
1086        lhs: Self::Value,
1087        rhs: Self::Value,
1088    ) -> Option<Self::Value> {
1089        // FIXME: See comment on the definition of `three_way_compare`.
1090        if crate::llvm_util::get_version() < (20, 0, 0) {
1091            return None;
1092        }
1093
1094        let size = ty.primitive_size(self.tcx);
1095        let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1096
1097        Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
1098    }
1099
1100    /* Miscellaneous instructions */
1101    fn memcpy(
1102        &mut self,
1103        dst: &'ll Value,
1104        dst_align: Align,
1105        src: &'ll Value,
1106        src_align: Align,
1107        size: &'ll Value,
1108        flags: MemFlags,
1109    ) {
1110        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1111        let size = self.intcast(size, self.type_isize(), false);
1112        let is_volatile = flags.contains(MemFlags::VOLATILE);
1113        unsafe {
1114            llvm::LLVMRustBuildMemCpy(
1115                self.llbuilder,
1116                dst,
1117                dst_align.bytes() as c_uint,
1118                src,
1119                src_align.bytes() as c_uint,
1120                size,
1121                is_volatile,
1122            );
1123        }
1124    }
1125
1126    fn memmove(
1127        &mut self,
1128        dst: &'ll Value,
1129        dst_align: Align,
1130        src: &'ll Value,
1131        src_align: Align,
1132        size: &'ll Value,
1133        flags: MemFlags,
1134    ) {
1135        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1136        let size = self.intcast(size, self.type_isize(), false);
1137        let is_volatile = flags.contains(MemFlags::VOLATILE);
1138        unsafe {
1139            llvm::LLVMRustBuildMemMove(
1140                self.llbuilder,
1141                dst,
1142                dst_align.bytes() as c_uint,
1143                src,
1144                src_align.bytes() as c_uint,
1145                size,
1146                is_volatile,
1147            );
1148        }
1149    }
1150
1151    fn memset(
1152        &mut self,
1153        ptr: &'ll Value,
1154        fill_byte: &'ll Value,
1155        size: &'ll Value,
1156        align: Align,
1157        flags: MemFlags,
1158    ) {
1159        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1160        let is_volatile = flags.contains(MemFlags::VOLATILE);
1161        unsafe {
1162            llvm::LLVMRustBuildMemSet(
1163                self.llbuilder,
1164                ptr,
1165                align.bytes() as c_uint,
1166                fill_byte,
1167                size,
1168                is_volatile,
1169            );
1170        }
1171    }
1172
1173    fn select(
1174        &mut self,
1175        cond: &'ll Value,
1176        then_val: &'ll Value,
1177        else_val: &'ll Value,
1178    ) -> &'ll Value {
1179        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1180    }
1181
1182    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1183        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1184    }
1185
1186    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1187        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1188    }
1189
1190    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1191        unsafe {
1192            let elt_ty = self.cx.val_ty(elt);
1193            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1194            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1195            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1196            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1197        }
1198    }
1199
1200    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1201        assert_eq!(idx as c_uint as u64, idx);
1202        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1203    }
1204
1205    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1206        assert_eq!(idx as c_uint as u64, idx);
1207        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1208    }
1209
1210    fn set_personality_fn(&mut self, personality: &'ll Value) {
1211        unsafe {
1212            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1213        }
1214    }
1215
1216    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1217        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1218        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1219        unsafe {
1220            llvm::LLVMSetCleanup(landing_pad, llvm::True);
1221        }
1222        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1223    }
1224
1225    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) {
1226        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1227        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1228        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1229    }
1230
1231    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1232        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1233        let mut exn = self.const_poison(ty);
1234        exn = self.insert_value(exn, exn0, 0);
1235        exn = self.insert_value(exn, exn1, 1);
1236        unsafe {
1237            llvm::LLVMBuildResume(self.llbuilder, exn);
1238        }
1239    }
1240
1241    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1242        let ret = unsafe {
1243            llvm::LLVMBuildCleanupPad(
1244                self.llbuilder,
1245                parent,
1246                args.as_ptr(),
1247                args.len() as c_uint,
1248                c"cleanuppad".as_ptr(),
1249            )
1250        };
1251        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1252    }
1253
1254    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1255        unsafe {
1256            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1257                .expect("LLVM does not have support for cleanupret");
1258        }
1259    }
1260
1261    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1262        let ret = unsafe {
1263            llvm::LLVMBuildCatchPad(
1264                self.llbuilder,
1265                parent,
1266                args.as_ptr(),
1267                args.len() as c_uint,
1268                c"catchpad".as_ptr(),
1269            )
1270        };
1271        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1272    }
1273
1274    fn catch_switch(
1275        &mut self,
1276        parent: Option<&'ll Value>,
1277        unwind: Option<&'ll BasicBlock>,
1278        handlers: &[&'ll BasicBlock],
1279    ) -> &'ll Value {
1280        let ret = unsafe {
1281            llvm::LLVMBuildCatchSwitch(
1282                self.llbuilder,
1283                parent,
1284                unwind,
1285                handlers.len() as c_uint,
1286                c"catchswitch".as_ptr(),
1287            )
1288        };
1289        let ret = ret.expect("LLVM does not have support for catchswitch");
1290        for handler in handlers {
1291            unsafe {
1292                llvm::LLVMAddHandler(ret, handler);
1293            }
1294        }
1295        ret
1296    }
1297
1298    // Atomic Operations
1299    fn atomic_cmpxchg(
1300        &mut self,
1301        dst: &'ll Value,
1302        cmp: &'ll Value,
1303        src: &'ll Value,
1304        order: rustc_middle::ty::AtomicOrdering,
1305        failure_order: rustc_middle::ty::AtomicOrdering,
1306        weak: bool,
1307    ) -> (&'ll Value, &'ll Value) {
1308        let weak = if weak { llvm::True } else { llvm::False };
1309        unsafe {
1310            let value = llvm::LLVMBuildAtomicCmpXchg(
1311                self.llbuilder,
1312                dst,
1313                cmp,
1314                src,
1315                AtomicOrdering::from_generic(order),
1316                AtomicOrdering::from_generic(failure_order),
1317                llvm::False, // SingleThreaded
1318            );
1319            llvm::LLVMSetWeak(value, weak);
1320            let val = self.extract_value(value, 0);
1321            let success = self.extract_value(value, 1);
1322            (val, success)
1323        }
1324    }
1325
1326    fn atomic_rmw(
1327        &mut self,
1328        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1329        dst: &'ll Value,
1330        mut src: &'ll Value,
1331        order: rustc_middle::ty::AtomicOrdering,
1332    ) -> &'ll Value {
1333        // The only RMW operation that LLVM supports on pointers is compare-exchange.
1334        let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
1335            && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg;
1336        if requires_cast_to_int {
1337            src = self.ptrtoint(src, self.type_isize());
1338        }
1339        let mut res = unsafe {
1340            llvm::LLVMBuildAtomicRMW(
1341                self.llbuilder,
1342                AtomicRmwBinOp::from_generic(op),
1343                dst,
1344                src,
1345                AtomicOrdering::from_generic(order),
1346                llvm::False, // SingleThreaded
1347            )
1348        };
1349        if requires_cast_to_int {
1350            res = self.inttoptr(res, self.type_ptr());
1351        }
1352        res
1353    }
1354
1355    fn atomic_fence(
1356        &mut self,
1357        order: rustc_middle::ty::AtomicOrdering,
1358        scope: SynchronizationScope,
1359    ) {
1360        let single_threaded = match scope {
1361            SynchronizationScope::SingleThread => llvm::True,
1362            SynchronizationScope::CrossThread => llvm::False,
1363        };
1364        unsafe {
1365            llvm::LLVMBuildFence(
1366                self.llbuilder,
1367                AtomicOrdering::from_generic(order),
1368                single_threaded,
1369                UNNAMED,
1370            );
1371        }
1372    }
1373
1374    fn set_invariant_load(&mut self, load: &'ll Value) {
1375        unsafe {
1376            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1377            self.set_metadata(load, llvm::MD_invariant_load, md);
1378        }
1379    }
1380
1381    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1382        self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1383    }
1384
1385    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1386        self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1387    }
1388
1389    fn call(
1390        &mut self,
1391        llty: &'ll Type,
1392        fn_attrs: Option<&CodegenFnAttrs>,
1393        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1394        llfn: &'ll Value,
1395        args: &[&'ll Value],
1396        funclet: Option<&Funclet<'ll>>,
1397        instance: Option<Instance<'tcx>>,
1398    ) -> &'ll Value {
1399        debug!("call {:?} with args ({:?})", llfn, args);
1400
1401        let args = self.check_call("call", llty, llfn, args);
1402        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1403        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1404        if let Some(funclet_bundle) = funclet_bundle {
1405            bundles.push(funclet_bundle);
1406        }
1407
1408        // Emit CFI pointer type membership test
1409        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1410
1411        // Emit KCFI operand bundle
1412        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1413        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1414            bundles.push(kcfi_bundle);
1415        }
1416
1417        let call = unsafe {
1418            llvm::LLVMBuildCallWithOperandBundles(
1419                self.llbuilder,
1420                llty,
1421                llfn,
1422                args.as_ptr() as *const &llvm::Value,
1423                args.len() as c_uint,
1424                bundles.as_ptr(),
1425                bundles.len() as c_uint,
1426                c"".as_ptr(),
1427            )
1428        };
1429        if let Some(fn_abi) = fn_abi {
1430            fn_abi.apply_attrs_callsite(self, call);
1431        }
1432        call
1433    }
1434
1435    fn tail_call(
1436        &mut self,
1437        llty: Self::Type,
1438        fn_attrs: Option<&CodegenFnAttrs>,
1439        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
1440        llfn: Self::Value,
1441        args: &[Self::Value],
1442        funclet: Option<&Self::Funclet>,
1443        instance: Option<Instance<'tcx>>,
1444    ) {
1445        let call = self.call(llty, fn_attrs, Some(fn_abi), llfn, args, funclet, instance);
1446        llvm::LLVMRustSetTailCallKind(call, llvm::TailCallKind::MustTail);
1447
1448        match &fn_abi.ret.mode {
1449            PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(),
1450            PassMode::Direct(_) | PassMode::Pair { .. } => self.ret(call),
1451            mode @ PassMode::Cast { .. } => {
1452                bug!("Encountered `PassMode::{mode:?}` during codegen")
1453            }
1454        }
1455    }
1456
1457    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1458        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1459    }
1460
1461    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1462        // Cleanup is always the cold path.
1463        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1464        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1465    }
1466}
1467
1468impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1469    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1470        // Forward to the `get_static` method of `CodegenCx`
1471        let global = self.cx().get_static(def_id);
1472        if self.cx().tcx.is_thread_local_static(def_id) {
1473            let pointer =
1474                self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1475            // Cast to default address space if globals are in a different addrspace
1476            self.pointercast(pointer, self.type_ptr())
1477        } else {
1478            // Cast to default address space if globals are in a different addrspace
1479            self.cx().const_pointercast(global, self.type_ptr())
1480        }
1481    }
1482}
1483
1484impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1485    pub(crate) fn llfn(&self) -> &'ll Value {
1486        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1487    }
1488}
1489
1490impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1491    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1492        unsafe {
1493            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1494        }
1495    }
1496}
1497impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1498    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1499        unsafe {
1500            let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1501            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
1502            self.set_metadata(load, llvm::MD_align, md);
1503        }
1504    }
1505
1506    fn noundef_metadata(&mut self, load: &'ll Value) {
1507        unsafe {
1508            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1509            self.set_metadata(load, llvm::MD_noundef, md);
1510        }
1511    }
1512
1513    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1514        unsafe {
1515            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1516            self.set_metadata(inst, llvm::MD_unpredictable, md);
1517        }
1518    }
1519}
1520impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1521    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1522        unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1523    }
1524
1525    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1526        unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1527    }
1528
1529    pub(crate) fn insert_element(
1530        &mut self,
1531        vec: &'ll Value,
1532        elt: &'ll Value,
1533        idx: &'ll Value,
1534    ) -> &'ll Value {
1535        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1536    }
1537
1538    pub(crate) fn shuffle_vector(
1539        &mut self,
1540        v1: &'ll Value,
1541        v2: &'ll Value,
1542        mask: &'ll Value,
1543    ) -> &'ll Value {
1544        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1545    }
1546
1547    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1548        unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1549    }
1550    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1551        unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1552    }
1553    pub(crate) fn vector_reduce_fadd_reassoc(
1554        &mut self,
1555        acc: &'ll Value,
1556        src: &'ll Value,
1557    ) -> &'ll Value {
1558        unsafe {
1559            let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1560            llvm::LLVMRustSetAllowReassoc(instr);
1561            instr
1562        }
1563    }
1564    pub(crate) fn vector_reduce_fmul_reassoc(
1565        &mut self,
1566        acc: &'ll Value,
1567        src: &'ll Value,
1568    ) -> &'ll Value {
1569        unsafe {
1570            let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1571            llvm::LLVMRustSetAllowReassoc(instr);
1572            instr
1573        }
1574    }
1575    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1576        unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1577    }
1578    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1579        unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1580    }
1581    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1582        unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1583    }
1584    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1585        unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1586    }
1587    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1588        unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1589    }
1590    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1591        unsafe {
1592            llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
1593        }
1594    }
1595    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1596        unsafe {
1597            llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
1598        }
1599    }
1600    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1601        unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1602    }
1603    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1604        unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1605    }
1606
1607    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1608        unsafe {
1609            llvm::LLVMAddClause(landing_pad, clause);
1610        }
1611    }
1612
1613    pub(crate) fn catch_ret(
1614        &mut self,
1615        funclet: &Funclet<'ll>,
1616        unwind: &'ll BasicBlock,
1617    ) -> &'ll Value {
1618        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1619        ret.expect("LLVM does not have support for catchret")
1620    }
1621
1622    fn check_call<'b>(
1623        &mut self,
1624        typ: &str,
1625        fn_ty: &'ll Type,
1626        llfn: &'ll Value,
1627        args: &'b [&'ll Value],
1628    ) -> Cow<'b, [&'ll Value]> {
1629        assert!(
1630            self.cx.type_kind(fn_ty) == TypeKind::Function,
1631            "builder::{typ} not passed a function, but {fn_ty:?}"
1632        );
1633
1634        let param_tys = self.cx.func_params_types(fn_ty);
1635
1636        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1637            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1638
1639        if all_args_match {
1640            return Cow::Borrowed(args);
1641        }
1642
1643        let casted_args: Vec<_> = iter::zip(param_tys, args)
1644            .enumerate()
1645            .map(|(i, (expected_ty, &actual_val))| {
1646                let actual_ty = self.cx.val_ty(actual_val);
1647                if expected_ty != actual_ty {
1648                    debug!(
1649                        "type mismatch in function call of {:?}. \
1650                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1651                        llfn, expected_ty, i, actual_ty
1652                    );
1653                    self.bitcast(actual_val, expected_ty)
1654                } else {
1655                    actual_val
1656                }
1657            })
1658            .collect();
1659
1660        Cow::Owned(casted_args)
1661    }
1662
1663    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1664        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1665    }
1666}
1667
1668impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1669    pub(crate) fn call_intrinsic(
1670        &mut self,
1671        base_name: impl Into<Cow<'static, str>>,
1672        type_params: &[&'ll Type],
1673        args: &[&'ll Value],
1674    ) -> &'ll Value {
1675        let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1676        self.call(ty, None, None, f, args, None, None)
1677    }
1678
1679    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1680        let size = size.bytes();
1681        if size == 0 {
1682            return;
1683        }
1684
1685        if !self.cx().sess().emit_lifetime_markers() {
1686            return;
1687        }
1688
1689        self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
1690    }
1691}
1692impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1693    pub(crate) fn phi(
1694        &mut self,
1695        ty: &'ll Type,
1696        vals: &[&'ll Value],
1697        bbs: &[&'ll BasicBlock],
1698    ) -> &'ll Value {
1699        assert_eq!(vals.len(), bbs.len());
1700        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1701        unsafe {
1702            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1703            phi
1704        }
1705    }
1706
1707    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1708        unsafe {
1709            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1710        }
1711    }
1712}
1713impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1714    pub(crate) fn landing_pad(
1715        &mut self,
1716        ty: &'ll Type,
1717        pers_fn: &'ll Value,
1718        num_clauses: usize,
1719    ) -> &'ll Value {
1720        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1721        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1722        // personality lives on the parent function anyway.
1723        self.set_personality_fn(pers_fn);
1724        unsafe {
1725            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1726        }
1727    }
1728
1729    pub(crate) fn callbr(
1730        &mut self,
1731        llty: &'ll Type,
1732        fn_attrs: Option<&CodegenFnAttrs>,
1733        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1734        llfn: &'ll Value,
1735        args: &[&'ll Value],
1736        default_dest: &'ll BasicBlock,
1737        indirect_dest: &[&'ll BasicBlock],
1738        funclet: Option<&Funclet<'ll>>,
1739        instance: Option<Instance<'tcx>>,
1740    ) -> &'ll Value {
1741        debug!("invoke {:?} with args ({:?})", llfn, args);
1742
1743        let args = self.check_call("callbr", llty, llfn, args);
1744        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1745        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1746        if let Some(funclet_bundle) = funclet_bundle {
1747            bundles.push(funclet_bundle);
1748        }
1749
1750        // Emit CFI pointer type membership test
1751        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1752
1753        // Emit KCFI operand bundle
1754        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1755        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1756            bundles.push(kcfi_bundle);
1757        }
1758
1759        let callbr = unsafe {
1760            llvm::LLVMBuildCallBr(
1761                self.llbuilder,
1762                llty,
1763                llfn,
1764                default_dest,
1765                indirect_dest.as_ptr(),
1766                indirect_dest.len() as c_uint,
1767                args.as_ptr(),
1768                args.len() as c_uint,
1769                bundles.as_ptr(),
1770                bundles.len() as c_uint,
1771                UNNAMED,
1772            )
1773        };
1774        if let Some(fn_abi) = fn_abi {
1775            fn_abi.apply_attrs_callsite(self, callbr);
1776        }
1777        callbr
1778    }
1779
1780    // Emits CFI pointer type membership tests.
1781    fn cfi_type_test(
1782        &mut self,
1783        fn_attrs: Option<&CodegenFnAttrs>,
1784        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1785        instance: Option<Instance<'tcx>>,
1786        llfn: &'ll Value,
1787    ) {
1788        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1789        if self.tcx.sess.is_sanitizer_cfi_enabled()
1790            && let Some(fn_abi) = fn_abi
1791            && is_indirect_call
1792        {
1793            if let Some(fn_attrs) = fn_attrs
1794                && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1795            {
1796                return;
1797            }
1798
1799            let mut options = cfi::TypeIdOptions::empty();
1800            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1801                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1802            }
1803            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1804                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1805            }
1806
1807            let typeid = if let Some(instance) = instance {
1808                cfi::typeid_for_instance(self.tcx, instance, options)
1809            } else {
1810                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1811            };
1812            let typeid_metadata = self.cx.create_metadata(typeid.as_bytes());
1813            let dbg_loc = self.get_dbg_loc();
1814
1815            // Test whether the function pointer is associated with the type identifier using the
1816            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
1817            // calls to this intrinsic with code to test type membership.
1818            let typeid = self.get_metadata_value(typeid_metadata);
1819            let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
1820            let bb_pass = self.append_sibling_block("type_test.pass");
1821            let bb_fail = self.append_sibling_block("type_test.fail");
1822            self.cond_br(cond, bb_pass, bb_fail);
1823
1824            self.switch_to_block(bb_fail);
1825            if let Some(dbg_loc) = dbg_loc {
1826                self.set_dbg_loc(dbg_loc);
1827            }
1828            self.abort();
1829            self.unreachable();
1830
1831            self.switch_to_block(bb_pass);
1832            if let Some(dbg_loc) = dbg_loc {
1833                self.set_dbg_loc(dbg_loc);
1834            }
1835        }
1836    }
1837
1838    // Emits KCFI operand bundles.
1839    fn kcfi_operand_bundle(
1840        &mut self,
1841        fn_attrs: Option<&CodegenFnAttrs>,
1842        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1843        instance: Option<Instance<'tcx>>,
1844        llfn: &'ll Value,
1845    ) -> Option<llvm::OperandBundleBox<'ll>> {
1846        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1847        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1848            && let Some(fn_abi) = fn_abi
1849            && is_indirect_call
1850        {
1851            if let Some(fn_attrs) = fn_attrs
1852                && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1853            {
1854                return None;
1855            }
1856
1857            let mut options = kcfi::TypeIdOptions::empty();
1858            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1859                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1860            }
1861            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1862                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1863            }
1864
1865            let kcfi_typeid = if let Some(instance) = instance {
1866                kcfi::typeid_for_instance(self.tcx, instance, options)
1867            } else {
1868                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1869            };
1870
1871            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1872        } else {
1873            None
1874        };
1875        kcfi_bundle
1876    }
1877
1878    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1879    #[instrument(level = "debug", skip(self))]
1880    pub(crate) fn instrprof_increment(
1881        &mut self,
1882        fn_name: &'ll Value,
1883        hash: &'ll Value,
1884        num_counters: &'ll Value,
1885        index: &'ll Value,
1886    ) {
1887        self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
1888    }
1889
1890    /// Emits a call to `llvm.instrprof.mcdc.parameters`.
1891    ///
1892    /// This doesn't produce any code directly, but is used as input by
1893    /// the LLVM pass that handles coverage instrumentation.
1894    ///
1895    /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.)
1896    ///
1897    /// [`CodeGenPGO::emitMCDCParameters`]:
1898    ///     https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124
1899    #[instrument(level = "debug", skip(self))]
1900    pub(crate) fn mcdc_parameters(
1901        &mut self,
1902        fn_name: &'ll Value,
1903        hash: &'ll Value,
1904        bitmap_bits: &'ll Value,
1905    ) {
1906        self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[], &[fn_name, hash, bitmap_bits]);
1907    }
1908
1909    #[instrument(level = "debug", skip(self))]
1910    pub(crate) fn mcdc_tvbitmap_update(
1911        &mut self,
1912        fn_name: &'ll Value,
1913        hash: &'ll Value,
1914        bitmap_index: &'ll Value,
1915        mcdc_temp: &'ll Value,
1916    ) {
1917        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
1918        self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", &[], args);
1919    }
1920
1921    #[instrument(level = "debug", skip(self))]
1922    pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
1923        self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
1924    }
1925
1926    #[instrument(level = "debug", skip(self))]
1927    pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
1928        let align = self.tcx.data_layout.i32_align.abi;
1929        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
1930        let new_tv_index = self.add(current_tv_index, cond_index);
1931        self.store(new_tv_index, mcdc_temp, align);
1932    }
1933}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
