rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::iter;
3use std::ops::Deref;
4
5use rustc_ast::expand::typetree::FncTree;
6pub(crate) mod autodiff;
7pub(crate) mod gpu_offload;
8
9use libc::{c_char, c_uint};
10use rustc_abi as abi;
11use rustc_abi::{Align, Size, WrappingRange};
12use rustc_codegen_ssa::MemFlags;
13use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
14use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
15use rustc_codegen_ssa::mir::place::PlaceRef;
16use rustc_codegen_ssa::traits::*;
17use rustc_data_structures::small_c_str::SmallCStr;
18use rustc_hir::def_id::DefId;
19use rustc_middle::bug;
20use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
21use rustc_middle::ty::layout::{
22    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
23    TyAndLayout,
24};
25use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
26use rustc_sanitizers::{cfi, kcfi};
27use rustc_session::config::OptLevel;
28use rustc_span::Span;
29use rustc_target::callconv::{FnAbi, PassMode};
30use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
31use smallvec::SmallVec;
32use tracing::{debug, instrument};
33
34use crate::abi::FnAbiLlvmExt;
35use crate::attributes;
36use crate::common::Funclet;
37use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
38use crate::llvm::{
39    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE,
40    ToLlvmBool, Type, Value,
41};
42use crate::type_of::LayoutLlvmExt;
43
44#[must_use]
45pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
46    pub llbuilder: &'ll mut llvm::Builder<'ll>,
47    pub cx: &'a GenericCx<'ll, CX>,
48}
49
50pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
51pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
52
53impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
54    fn drop(&mut self) {
55        unsafe {
56            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
57        }
58    }
59}
60
61impl<'a, 'll> SBuilder<'a, 'll> {
62    pub(crate) fn call(
63        &mut self,
64        llty: &'ll Type,
65        llfn: &'ll Value,
66        args: &[&'ll Value],
67        funclet: Option<&Funclet<'ll>>,
68    ) -> &'ll Value {
69        debug!("call {:?} with args ({:?})", llfn, args);
70
71        let args = self.check_call("call", llty, llfn, args);
72        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
73        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
74        if let Some(funclet_bundle) = funclet_bundle {
75            bundles.push(funclet_bundle);
76        }
77
78        let call = unsafe {
79            llvm::LLVMBuildCallWithOperandBundles(
80                self.llbuilder,
81                llty,
82                llfn,
83                args.as_ptr() as *const &llvm::Value,
84                args.len() as c_uint,
85                bundles.as_ptr(),
86                bundles.len() as c_uint,
87                c"".as_ptr(),
88            )
89        };
90        call
91    }
92}
93
94impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
95    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
96        // Create a fresh builder from the simple context.
97        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
98        GenericBuilder { llbuilder, cx: scx }
99    }
100
101    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
102        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
103    }
104
105    pub(crate) fn ret_void(&mut self) {
106        llvm::LLVMBuildRetVoid(self.llbuilder);
107    }
108
109    pub(crate) fn ret(&mut self, v: &'ll Value) {
110        unsafe {
111            llvm::LLVMBuildRet(self.llbuilder, v);
112        }
113    }
114
115    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
116        let bx = Self::with_cx(cx);
117        unsafe {
118            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
119        }
120        bx
121    }
122
123    // The generic builder has less functionality and thus (unlike the other alloca) we can not
124    // easily jump to the beginning of the function to place our allocas there. We trust the user
125    // to manually do that. FIXME(offload): improve the genericCx and add more llvm wrappers to
126    // handle this.
127    pub(crate) fn direct_alloca(&mut self, ty: &'ll Type, align: Align, name: &str) -> &'ll Value {
128        let val = unsafe {
129            let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED);
130            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
131            // Cast to default addrspace if necessary
132            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx.type_ptr(), UNNAMED)
133        };
134        if name != "" {
135            let name = std::ffi::CString::new(name).unwrap();
136            llvm::set_value_name(val, &name.as_bytes());
137        }
138        val
139    }
140
141    pub(crate) fn inbounds_gep(
142        &mut self,
143        ty: &'ll Type,
144        ptr: &'ll Value,
145        indices: &[&'ll Value],
146    ) -> &'ll Value {
147        unsafe {
148            llvm::LLVMBuildGEPWithNoWrapFlags(
149                self.llbuilder,
150                ty,
151                ptr,
152                indices.as_ptr(),
153                indices.len() as c_uint,
154                UNNAMED,
155                GEPNoWrapFlags::InBounds,
156            )
157        }
158    }
159
160    pub(crate) fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
161        debug!("Store {:?} -> {:?}", val, ptr);
162        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
163        unsafe {
164            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
165            llvm::LLVMSetAlignment(store, align.bytes() as c_uint);
166            store
167        }
168    }
169
170    pub(crate) fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
171        unsafe {
172            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
173            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
174            load
175        }
176    }
177
178    fn memset(&mut self, ptr: &'ll Value, fill_byte: &'ll Value, size: &'ll Value, align: Align) {
179        unsafe {
180            llvm::LLVMRustBuildMemSet(
181                self.llbuilder,
182                ptr,
183                align.bytes() as c_uint,
184                fill_byte,
185                size,
186                false,
187            );
188        }
189    }
190}
191
192/// Empty string, to be used where LLVM expects an instruction name, indicating
193/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
194// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
195pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
196
197impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
198    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
199    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
200    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
201    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
202    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
203    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
204
205    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
206    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
207    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
208}
209
210impl abi::HasDataLayout for Builder<'_, '_, '_> {
211    fn data_layout(&self) -> &abi::TargetDataLayout {
212        self.cx.data_layout()
213    }
214}
215
216impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
217    #[inline]
218    fn tcx(&self) -> TyCtxt<'tcx> {
219        self.cx.tcx
220    }
221}
222
223impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
224    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
225        self.cx.typing_env()
226    }
227}
228
229impl HasTargetSpec for Builder<'_, '_, '_> {
230    #[inline]
231    fn target_spec(&self) -> &Target {
232        self.cx.target_spec()
233    }
234}
235
236impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
237    #[inline]
238    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
239        self.cx.handle_layout_err(err, span, ty)
240    }
241}
242
243impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
244    #[inline]
245    fn handle_fn_abi_err(
246        &self,
247        err: FnAbiError<'tcx>,
248        span: Span,
249        fn_abi_request: FnAbiRequest<'tcx>,
250    ) -> ! {
251        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
252    }
253}
254
255impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
256    type Target = CodegenCx<'ll, 'tcx>;
257
258    #[inline]
259    fn deref(&self) -> &Self::Target {
260        self.cx
261    }
262}
263
264macro_rules! math_builder_methods {
265    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
266        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
267            unsafe {
268                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
269            }
270        })+
271    }
272}
273
274macro_rules! set_math_builder_methods {
275    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
276        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
277            unsafe {
278                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
279                llvm::$llvm_set_math(instr);
280                instr
281            }
282        })+
283    }
284}
285
286impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
287    type CodegenCx = CodegenCx<'ll, 'tcx>;
288
289    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
290        let bx = Builder::with_cx(cx);
291        unsafe {
292            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
293        }
294        bx
295    }
296
297    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
298        self.cx
299    }
300
301    fn llbb(&self) -> &'ll BasicBlock {
302        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
303    }
304
305    fn set_span(&mut self, _span: Span) {}
306
307    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
308        unsafe {
309            let name = SmallCStr::new(name);
310            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
311        }
312    }
313
314    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
315        Self::append_block(self.cx, self.llfn(), name)
316    }
317
318    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
319        *self = Self::build(self.cx, llbb)
320    }
321
322    fn ret_void(&mut self) {
323        llvm::LLVMBuildRetVoid(self.llbuilder);
324    }
325
326    fn ret(&mut self, v: &'ll Value) {
327        unsafe {
328            llvm::LLVMBuildRet(self.llbuilder, v);
329        }
330    }
331
332    fn br(&mut self, dest: &'ll BasicBlock) {
333        unsafe {
334            llvm::LLVMBuildBr(self.llbuilder, dest);
335        }
336    }
337
338    fn cond_br(
339        &mut self,
340        cond: &'ll Value,
341        then_llbb: &'ll BasicBlock,
342        else_llbb: &'ll BasicBlock,
343    ) {
344        unsafe {
345            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
346        }
347    }
348
349    fn switch(
350        &mut self,
351        v: &'ll Value,
352        else_llbb: &'ll BasicBlock,
353        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
354    ) {
355        let switch =
356            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
357        for (on_val, dest) in cases {
358            let on_val = self.const_uint_big(self.val_ty(v), on_val);
359            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
360        }
361    }
362
363    fn switch_with_weights(
364        &mut self,
365        v: Self::Value,
366        else_llbb: Self::BasicBlock,
367        else_is_cold: bool,
368        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
369    ) {
370        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
371            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
372            return;
373        }
374
375        let id = self.cx.create_metadata(b"branch_weights");
376
377        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
378        // This function handles switch instructions with more than 2 targets and it needs to
379        // emit branch weights metadata instead of using the intrinsic.
380        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
381        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
382        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
383        let weight =
384            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
385
386        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
387        md.push(id);
388        md.push(weight(else_is_cold));
389
390        let switch =
391            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
392        for (on_val, dest, is_cold) in cases {
393            let on_val = self.const_uint_big(self.val_ty(v), on_val);
394            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
395            md.push(weight(is_cold));
396        }
397
398        self.cx.set_metadata_node(switch, llvm::MD_prof, &md);
399    }
400
401    fn invoke(
402        &mut self,
403        llty: &'ll Type,
404        fn_attrs: Option<&CodegenFnAttrs>,
405        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
406        llfn: &'ll Value,
407        args: &[&'ll Value],
408        then: &'ll BasicBlock,
409        catch: &'ll BasicBlock,
410        funclet: Option<&Funclet<'ll>>,
411        instance: Option<Instance<'tcx>>,
412    ) -> &'ll Value {
413        debug!("invoke {:?} with args ({:?})", llfn, args);
414
415        let args = self.check_call("invoke", llty, llfn, args);
416        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
417        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
418        if let Some(funclet_bundle) = funclet_bundle {
419            bundles.push(funclet_bundle);
420        }
421
422        // Emit CFI pointer type membership test
423        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
424
425        // Emit KCFI operand bundle
426        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
427        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
428            bundles.push(kcfi_bundle);
429        }
430
431        let invoke = unsafe {
432            llvm::LLVMBuildInvokeWithOperandBundles(
433                self.llbuilder,
434                llty,
435                llfn,
436                args.as_ptr(),
437                args.len() as c_uint,
438                then,
439                catch,
440                bundles.as_ptr(),
441                bundles.len() as c_uint,
442                UNNAMED,
443            )
444        };
445        if let Some(fn_abi) = fn_abi {
446            fn_abi.apply_attrs_callsite(self, invoke);
447        }
448        invoke
449    }
450
451    fn unreachable(&mut self) {
452        unsafe {
453            llvm::LLVMBuildUnreachable(self.llbuilder);
454        }
455    }
456
457    math_builder_methods! {
458        add(a, b) => LLVMBuildAdd,
459        fadd(a, b) => LLVMBuildFAdd,
460        sub(a, b) => LLVMBuildSub,
461        fsub(a, b) => LLVMBuildFSub,
462        mul(a, b) => LLVMBuildMul,
463        fmul(a, b) => LLVMBuildFMul,
464        udiv(a, b) => LLVMBuildUDiv,
465        exactudiv(a, b) => LLVMBuildExactUDiv,
466        sdiv(a, b) => LLVMBuildSDiv,
467        exactsdiv(a, b) => LLVMBuildExactSDiv,
468        fdiv(a, b) => LLVMBuildFDiv,
469        urem(a, b) => LLVMBuildURem,
470        srem(a, b) => LLVMBuildSRem,
471        frem(a, b) => LLVMBuildFRem,
472        shl(a, b) => LLVMBuildShl,
473        lshr(a, b) => LLVMBuildLShr,
474        ashr(a, b) => LLVMBuildAShr,
475        and(a, b) => LLVMBuildAnd,
476        or(a, b) => LLVMBuildOr,
477        xor(a, b) => LLVMBuildXor,
478        neg(x) => LLVMBuildNeg,
479        fneg(x) => LLVMBuildFNeg,
480        not(x) => LLVMBuildNot,
481        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
482        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
483        unchecked_ssub(x, y) => LLVMBuildNSWSub,
484        unchecked_usub(x, y) => LLVMBuildNUWSub,
485        unchecked_smul(x, y) => LLVMBuildNSWMul,
486        unchecked_umul(x, y) => LLVMBuildNUWMul,
487    }
488
489    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
490        unsafe {
491            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
492            if llvm::LLVMIsAInstruction(add).is_some() {
493                llvm::LLVMSetNUW(add, TRUE);
494                llvm::LLVMSetNSW(add, TRUE);
495            }
496            add
497        }
498    }
499    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
500        unsafe {
501            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
502            if llvm::LLVMIsAInstruction(sub).is_some() {
503                llvm::LLVMSetNUW(sub, TRUE);
504                llvm::LLVMSetNSW(sub, TRUE);
505            }
506            sub
507        }
508    }
509    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
510        unsafe {
511            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
512            if llvm::LLVMIsAInstruction(mul).is_some() {
513                llvm::LLVMSetNUW(mul, TRUE);
514                llvm::LLVMSetNSW(mul, TRUE);
515            }
516            mul
517        }
518    }
519
520    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
521        unsafe {
522            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
523
524            // If a and b are both values, then `or` is a value, rather than
525            // an instruction, so we need to check before setting the flag.
526            // (See also `LLVMBuildNUWNeg` which also needs a check.)
527            if llvm::LLVMIsAInstruction(or).is_some() {
528                llvm::LLVMSetIsDisjoint(or, TRUE);
529            }
530            or
531        }
532    }
533
534    set_math_builder_methods! {
535        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
536        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
537        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
538        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
539        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
540        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
541        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
542        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
543        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
544        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
545    }
546
547    fn checked_binop(
548        &mut self,
549        oop: OverflowOp,
550        ty: Ty<'tcx>,
551        lhs: Self::Value,
552        rhs: Self::Value,
553    ) -> (Self::Value, Self::Value) {
554        let (size, signed) = ty.int_size_and_signed(self.tcx);
555        let width = size.bits();
556
557        if !signed {
558            match oop {
559                OverflowOp::Sub => {
560                    // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
561                    // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
562                    // in the backend if profitable.
563                    let sub = self.sub(lhs, rhs);
564                    let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
565                    return (sub, cmp);
566                }
567                OverflowOp::Add => {
568                    // Like with sub above, using icmp is the preferred form. See
569                    // <https://rust-lang.zulipchat.com/#narrow/channel/187780-t-compiler.2Fllvm/topic/.60uadd.2Ewith.2Eoverflow.60.20.28again.29/near/533041085>
570                    let add = self.add(lhs, rhs);
571                    let cmp = self.icmp(IntPredicate::IntULT, add, lhs);
572                    return (add, cmp);
573                }
574                OverflowOp::Mul => {}
575            }
576        }
577
578        let oop_str = match oop {
579            OverflowOp::Add => "add",
580            OverflowOp::Sub => "sub",
581            OverflowOp::Mul => "mul",
582        };
583
584        let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
585
586        let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
587        (self.extract_value(res, 0), self.extract_value(res, 1))
588    }
589
590    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
591        if self.cx().val_ty(val) == self.cx().type_i1() {
592            self.zext(val, self.cx().type_i8())
593        } else {
594            val
595        }
596    }
597
598    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
599        if scalar.is_bool() {
600            return self.unchecked_utrunc(val, self.cx().type_i1());
601        }
602        val
603    }
604
605    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
606        let mut bx = Builder::with_cx(self.cx);
607        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
608        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
609        unsafe {
610            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
611            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
612            // Cast to default addrspace if necessary
613            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
614        }
615    }
616
617    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
618        unsafe {
619            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
620            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
621            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
622            load
623        }
624    }
625
626    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
627        unsafe {
628            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
629            llvm::LLVMSetVolatile(load, llvm::TRUE);
630            load
631        }
632    }
633
634    fn atomic_load(
635        &mut self,
636        ty: &'ll Type,
637        ptr: &'ll Value,
638        order: rustc_middle::ty::AtomicOrdering,
639        size: Size,
640    ) -> &'ll Value {
641        unsafe {
642            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
643            // Set atomic ordering
644            llvm::LLVMSetOrdering(load, AtomicOrdering::from_generic(order));
645            // LLVM requires the alignment of atomic loads to be at least the size of the type.
646            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
647            load
648        }
649    }
650
651    #[instrument(level = "trace", skip(self))]
652    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
653        if place.layout.is_unsized() {
654            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
655            if matches!(tail.kind(), ty::Foreign(..)) {
656                // Unsized locals and, at least conceptually, even unsized arguments must be copied
657                // around, which requires dynamically determining their size. Therefore, we cannot
658                // allow `extern` types here. Consult t-opsem before removing this check.
659                panic!("unsized locals must not be `extern` types");
660            }
661        }
662        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
663
664        if place.layout.is_zst() {
665            return OperandRef::zero_sized(place.layout);
666        }
667
668        #[instrument(level = "trace", skip(bx))]
669        fn scalar_load_metadata<'a, 'll, 'tcx>(
670            bx: &mut Builder<'a, 'll, 'tcx>,
671            load: &'ll Value,
672            scalar: abi::Scalar,
673            layout: TyAndLayout<'tcx>,
674            offset: Size,
675        ) {
676            if bx.cx.sess().opts.optimize == OptLevel::No {
677                // Don't emit metadata we're not going to use
678                return;
679            }
680
681            if !scalar.is_uninit_valid() {
682                bx.noundef_metadata(load);
683            }
684
685            match scalar.primitive() {
686                abi::Primitive::Int(..) => {
687                    if !scalar.is_always_valid(bx) {
688                        bx.range_metadata(load, scalar.valid_range(bx));
689                    }
690                }
691                abi::Primitive::Pointer(_) => {
692                    if !scalar.valid_range(bx).contains(0) {
693                        bx.nonnull_metadata(load);
694                    }
695
696                    if let Some(pointee) = layout.pointee_info_at(bx, offset)
697                        && let Some(_) = pointee.safe
698                    {
699                        bx.align_metadata(load, pointee.align);
700                    }
701                }
702                abi::Primitive::Float(_) => {}
703            }
704        }
705
706        let val = if let Some(_) = place.val.llextra {
707            // FIXME: Merge with the `else` below?
708            OperandValue::Ref(place.val)
709        } else if place.layout.is_llvm_immediate() {
710            let mut const_llval = None;
711            let llty = place.layout.llvm_type(self);
712            if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
713                if llvm::LLVMIsGlobalConstant(global).is_true() {
714                    if let Some(init) = llvm::LLVMGetInitializer(global) {
715                        if self.val_ty(init) == llty {
716                            const_llval = Some(init);
717                        }
718                    }
719                }
720            }
721
722            let llval = const_llval.unwrap_or_else(|| {
723                let load = self.load(llty, place.val.llval, place.val.align);
724                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
725                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
726                    self.to_immediate_scalar(load, scalar)
727                } else {
728                    load
729                }
730            });
731            OperandValue::Immediate(llval)
732        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
733            let b_offset = a.size(self).align_to(b.align(self).abi);
734
735            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
736                let llptr = if i == 0 {
737                    place.val.llval
738                } else {
739                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
740                };
741                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
742                let load = self.load(llty, llptr, align);
743                scalar_load_metadata(self, load, scalar, layout, offset);
744                self.to_immediate_scalar(load, scalar)
745            };
746
747            OperandValue::Pair(
748                load(0, a, place.layout, place.val.align, Size::ZERO),
749                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
750            )
751        } else {
752            OperandValue::Ref(place.val)
753        };
754
755        OperandRef { val, layout: place.layout }
756    }
757
758    fn write_operand_repeatedly(
759        &mut self,
760        cg_elem: OperandRef<'tcx, &'ll Value>,
761        count: u64,
762        dest: PlaceRef<'tcx, &'ll Value>,
763    ) {
764        let zero = self.const_usize(0);
765        let count = self.const_usize(count);
766
767        let header_bb = self.append_sibling_block("repeat_loop_header");
768        let body_bb = self.append_sibling_block("repeat_loop_body");
769        let next_bb = self.append_sibling_block("repeat_loop_next");
770
771        self.br(header_bb);
772
773        let mut header_bx = Self::build(self.cx, header_bb);
774        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
775
776        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
777        header_bx.cond_br(keep_going, body_bb, next_bb);
778
779        let mut body_bx = Self::build(self.cx, body_bb);
780        let dest_elem = dest.project_index(&mut body_bx, i);
781        cg_elem.val.store(&mut body_bx, dest_elem);
782
783        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
784        body_bx.br(header_bb);
785        header_bx.add_incoming_to_phi(i, next, body_bb);
786
787        *self = Self::build(self.cx, next_bb);
788    }
789
790    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
791        if self.cx.sess().opts.optimize == OptLevel::No {
792            // Don't emit metadata we're not going to use
793            return;
794        }
795
796        let llty = self.cx.val_ty(load);
797        let md = [
798            llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
799            llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
800        ];
801        self.set_metadata_node(load, llvm::MD_range, &md);
802    }
803
804    fn nonnull_metadata(&mut self, load: &'ll Value) {
805        self.set_metadata_node(load, llvm::MD_nonnull, &[]);
806    }
807
808    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
809        self.store_with_flags(val, ptr, align, MemFlags::empty())
810    }
811
812    fn store_with_flags(
813        &mut self,
814        val: &'ll Value,
815        ptr: &'ll Value,
816        align: Align,
817        flags: MemFlags,
818    ) -> &'ll Value {
819        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
820        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
821        unsafe {
822            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
823            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
824            let align =
825                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
826            llvm::LLVMSetAlignment(store, align);
827            if flags.contains(MemFlags::VOLATILE) {
828                llvm::LLVMSetVolatile(store, llvm::TRUE);
829            }
830            if flags.contains(MemFlags::NONTEMPORAL) {
831                // Make sure that the current target architectures supports "sane" non-temporal
832                // stores, i.e., non-temporal stores that are equivalent to regular stores except
833                // for performance. LLVM doesn't seem to care about this, and will happily treat
834                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
835                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
836                // regular stores but require special fences. So we keep a list of architectures
837                // where `!nontemporal` is known to be truly just a hint, and use regular stores
838                // everywhere else. (In the future, we could alternatively ensure that an sfence
839                // gets emitted after a sequence of movnt before any kind of synchronizing
840                // operation. But it's not clear how to do that with LLVM.)
841                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
842                // <https://github.com/llvm/llvm-project/issues/64521>.
843                const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
844                    &["aarch64", "arm", "riscv32", "riscv64"];
845
846                let use_nontemporal =
847                    WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
848                if use_nontemporal {
849                    // According to LLVM [1] building a nontemporal store must
850                    // *always* point to a metadata value of the integer 1.
851                    //
852                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
853                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
854                    self.set_metadata_node(store, llvm::MD_nontemporal, &[one]);
855                }
856            }
857            store
858        }
859    }
860
861    fn atomic_store(
862        &mut self,
863        val: &'ll Value,
864        ptr: &'ll Value,
865        order: rustc_middle::ty::AtomicOrdering,
866        size: Size,
867    ) {
868        debug!("Store {:?} -> {:?}", val, ptr);
869        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
870        unsafe {
871            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
872            // Set atomic ordering
873            llvm::LLVMSetOrdering(store, AtomicOrdering::from_generic(order));
874            // LLVM requires the alignment of atomic stores to be at least the size of the type.
875            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
876        }
877    }
878
879    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
880        unsafe {
881            llvm::LLVMBuildGEPWithNoWrapFlags(
882                self.llbuilder,
883                ty,
884                ptr,
885                indices.as_ptr(),
886                indices.len() as c_uint,
887                UNNAMED,
888                GEPNoWrapFlags::default(),
889            )
890        }
891    }
892
893    fn inbounds_gep(
894        &mut self,
895        ty: &'ll Type,
896        ptr: &'ll Value,
897        indices: &[&'ll Value],
898    ) -> &'ll Value {
899        unsafe {
900            llvm::LLVMBuildGEPWithNoWrapFlags(
901                self.llbuilder,
902                ty,
903                ptr,
904                indices.as_ptr(),
905                indices.len() as c_uint,
906                UNNAMED,
907                GEPNoWrapFlags::InBounds,
908            )
909        }
910    }
911
912    fn inbounds_nuw_gep(
913        &mut self,
914        ty: &'ll Type,
915        ptr: &'ll Value,
916        indices: &[&'ll Value],
917    ) -> &'ll Value {
918        unsafe {
919            llvm::LLVMBuildGEPWithNoWrapFlags(
920                self.llbuilder,
921                ty,
922                ptr,
923                indices.as_ptr(),
924                indices.len() as c_uint,
925                UNNAMED,
926                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
927            )
928        }
929    }
930
931    /* Casts */
932    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
933        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
934    }
935
936    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
937        debug_assert_ne!(self.val_ty(val), dest_ty);
938
939        let trunc = self.trunc(val, dest_ty);
940        unsafe {
941            if llvm::LLVMIsAInstruction(trunc).is_some() {
942                llvm::LLVMSetNUW(trunc, TRUE);
943            }
944        }
945        trunc
946    }
947
948    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
949        debug_assert_ne!(self.val_ty(val), dest_ty);
950
951        let trunc = self.trunc(val, dest_ty);
952        unsafe {
953            if llvm::LLVMIsAInstruction(trunc).is_some() {
954                llvm::LLVMSetNSW(trunc, TRUE);
955            }
956        }
957        trunc
958    }
959
960    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
961        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
962    }
963
964    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
965        self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
966    }
967
968    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
969        self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
970    }
971
972    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
973        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
974        // poor codegen. The reason for this is that the corresponding wasm
975        // instructions, `i32.trunc_f32_s` for example, will trap when the float
976        // is out-of-bounds, infinity, or nan. This means that LLVM
977        // automatically inserts control flow around `fptoui` and `fptosi`
978        // because the LLVM instruction `fptoui` is defined as producing a
979        // poison value, not having UB on out-of-bounds values.
980        //
981        // This method, however, is only used with non-saturating casts that
982        // have UB on out-of-bounds values. This means that it's ok if we use
983        // the raw wasm instruction since out-of-bounds values can do whatever
984        // we like. To ensure that LLVM picks the right instruction we choose
985        // the raw wasm intrinsic functions which avoid LLVM inserting all the
986        // other control flow automatically.
987        if self.sess().target.is_like_wasm {
988            let src_ty = self.cx.val_ty(val);
989            if self.cx.type_kind(src_ty) != TypeKind::Vector {
990                let float_width = self.cx.float_width(src_ty);
991                let int_width = self.cx.int_width(dest_ty);
992                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
993                    return self.call_intrinsic(
994                        "llvm.wasm.trunc.unsigned",
995                        &[dest_ty, src_ty],
996                        &[val],
997                    );
998                }
999            }
1000        }
1001        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
1002    }
1003
1004    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1005        // see `fptoui` above for why wasm is different here
1006        if self.sess().target.is_like_wasm {
1007            let src_ty = self.cx.val_ty(val);
1008            if self.cx.type_kind(src_ty) != TypeKind::Vector {
1009                let float_width = self.cx.float_width(src_ty);
1010                let int_width = self.cx.int_width(dest_ty);
1011                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
1012                    return self.call_intrinsic(
1013                        "llvm.wasm.trunc.signed",
1014                        &[dest_ty, src_ty],
1015                        &[val],
1016                    );
1017                }
1018            }
1019        }
1020        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
1021    }
1022
1023    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1024        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1025    }
1026
1027    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1028        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1029    }
1030
1031    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1032        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
1033    }
1034
1035    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1036        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
1037    }
1038
1039    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1040        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
1041    }
1042
1043    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1044        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
1045    }
1046
1047    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1048        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
1049    }
1050
1051    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1052        unsafe {
1053            llvm::LLVMBuildIntCast2(self.llbuilder, val, dest_ty, is_signed.to_llvm_bool(), UNNAMED)
1054        }
1055    }
1056
1057    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1058        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1059    }
1060
1061    /* Comparisons */
1062    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1063        let op = llvm::IntPredicate::from_generic(op);
1064        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1065    }
1066
1067    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1068        let op = llvm::RealPredicate::from_generic(op);
1069        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1070    }
1071
1072    fn three_way_compare(
1073        &mut self,
1074        ty: Ty<'tcx>,
1075        lhs: Self::Value,
1076        rhs: Self::Value,
1077    ) -> Self::Value {
1078        let size = ty.primitive_size(self.tcx);
1079        let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1080
1081        self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])
1082    }
1083
1084    /* Miscellaneous instructions */
1085    fn memcpy(
1086        &mut self,
1087        dst: &'ll Value,
1088        dst_align: Align,
1089        src: &'ll Value,
1090        src_align: Align,
1091        size: &'ll Value,
1092        flags: MemFlags,
1093        tt: Option<FncTree>,
1094    ) {
1095        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1096        let size = self.intcast(size, self.type_isize(), false);
1097        let is_volatile = flags.contains(MemFlags::VOLATILE);
1098        let memcpy = unsafe {
1099            llvm::LLVMRustBuildMemCpy(
1100                self.llbuilder,
1101                dst,
1102                dst_align.bytes() as c_uint,
1103                src,
1104                src_align.bytes() as c_uint,
1105                size,
1106                is_volatile,
1107            )
1108        };
1109
1110        // TypeTree metadata for memcpy is especially important: when Enzyme encounters
1111        // a memcpy during autodiff, it needs to know the structure of the data being
1112        // copied to properly track derivatives. For example, copying an array of floats
1113        // vs. copying a struct with mixed types requires different derivative handling.
1114        // The TypeTree tells Enzyme exactly what memory layout to expect.
1115        if let Some(tt) = tt {
1116            crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt);
1117        }
1118    }
1119
1120    fn memmove(
1121        &mut self,
1122        dst: &'ll Value,
1123        dst_align: Align,
1124        src: &'ll Value,
1125        src_align: Align,
1126        size: &'ll Value,
1127        flags: MemFlags,
1128    ) {
1129        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1130        let size = self.intcast(size, self.type_isize(), false);
1131        let is_volatile = flags.contains(MemFlags::VOLATILE);
1132        unsafe {
1133            llvm::LLVMRustBuildMemMove(
1134                self.llbuilder,
1135                dst,
1136                dst_align.bytes() as c_uint,
1137                src,
1138                src_align.bytes() as c_uint,
1139                size,
1140                is_volatile,
1141            );
1142        }
1143    }
1144
1145    fn memset(
1146        &mut self,
1147        ptr: &'ll Value,
1148        fill_byte: &'ll Value,
1149        size: &'ll Value,
1150        align: Align,
1151        flags: MemFlags,
1152    ) {
1153        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1154        let is_volatile = flags.contains(MemFlags::VOLATILE);
1155        unsafe {
1156            llvm::LLVMRustBuildMemSet(
1157                self.llbuilder,
1158                ptr,
1159                align.bytes() as c_uint,
1160                fill_byte,
1161                size,
1162                is_volatile,
1163            );
1164        }
1165    }
1166
1167    fn select(
1168        &mut self,
1169        cond: &'ll Value,
1170        then_val: &'ll Value,
1171        else_val: &'ll Value,
1172    ) -> &'ll Value {
1173        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1174    }
1175
1176    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1177        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1178    }
1179
1180    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1181        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1182    }
1183
1184    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1185        unsafe {
1186            let elt_ty = self.cx.val_ty(elt);
1187            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1188            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1189            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1190            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1191        }
1192    }
1193
1194    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1195        assert_eq!(idx as c_uint as u64, idx);
1196        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1197    }
1198
1199    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1200        assert_eq!(idx as c_uint as u64, idx);
1201        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1202    }
1203
1204    fn set_personality_fn(&mut self, personality: &'ll Value) {
1205        unsafe {
1206            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1207        }
1208    }
1209
1210    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1211        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1212        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1213        unsafe {
1214            llvm::LLVMSetCleanup(landing_pad, llvm::TRUE);
1215        }
1216        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1217    }
1218
1219    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) {
1220        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1221        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1222        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1223    }
1224
1225    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1226        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1227        let mut exn = self.const_poison(ty);
1228        exn = self.insert_value(exn, exn0, 0);
1229        exn = self.insert_value(exn, exn1, 1);
1230        unsafe {
1231            llvm::LLVMBuildResume(self.llbuilder, exn);
1232        }
1233    }
1234
1235    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1236        let ret = unsafe {
1237            llvm::LLVMBuildCleanupPad(
1238                self.llbuilder,
1239                parent,
1240                args.as_ptr(),
1241                args.len() as c_uint,
1242                c"cleanuppad".as_ptr(),
1243            )
1244        };
1245        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1246    }
1247
1248    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1249        unsafe {
1250            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1251                .expect("LLVM does not have support for cleanupret");
1252        }
1253    }
1254
1255    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1256        let ret = unsafe {
1257            llvm::LLVMBuildCatchPad(
1258                self.llbuilder,
1259                parent,
1260                args.as_ptr(),
1261                args.len() as c_uint,
1262                c"catchpad".as_ptr(),
1263            )
1264        };
1265        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1266    }
1267
1268    fn catch_switch(
1269        &mut self,
1270        parent: Option<&'ll Value>,
1271        unwind: Option<&'ll BasicBlock>,
1272        handlers: &[&'ll BasicBlock],
1273    ) -> &'ll Value {
1274        let ret = unsafe {
1275            llvm::LLVMBuildCatchSwitch(
1276                self.llbuilder,
1277                parent,
1278                unwind,
1279                handlers.len() as c_uint,
1280                c"catchswitch".as_ptr(),
1281            )
1282        };
1283        let ret = ret.expect("LLVM does not have support for catchswitch");
1284        for handler in handlers {
1285            unsafe {
1286                llvm::LLVMAddHandler(ret, handler);
1287            }
1288        }
1289        ret
1290    }
1291
1292    // Atomic Operations
1293    fn atomic_cmpxchg(
1294        &mut self,
1295        dst: &'ll Value,
1296        cmp: &'ll Value,
1297        src: &'ll Value,
1298        order: rustc_middle::ty::AtomicOrdering,
1299        failure_order: rustc_middle::ty::AtomicOrdering,
1300        weak: bool,
1301    ) -> (&'ll Value, &'ll Value) {
1302        unsafe {
1303            let value = llvm::LLVMBuildAtomicCmpXchg(
1304                self.llbuilder,
1305                dst,
1306                cmp,
1307                src,
1308                AtomicOrdering::from_generic(order),
1309                AtomicOrdering::from_generic(failure_order),
1310                llvm::FALSE, // SingleThreaded
1311            );
1312            llvm::LLVMSetWeak(value, weak.to_llvm_bool());
1313            let val = self.extract_value(value, 0);
1314            let success = self.extract_value(value, 1);
1315            (val, success)
1316        }
1317    }
1318
1319    fn atomic_rmw(
1320        &mut self,
1321        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1322        dst: &'ll Value,
1323        src: &'ll Value,
1324        order: rustc_middle::ty::AtomicOrdering,
1325        ret_ptr: bool,
1326    ) -> &'ll Value {
1327        // FIXME: If `ret_ptr` is true and `src` is not a pointer, we *should* tell LLVM that the
1328        // LHS is a pointer and the operation should be provenance-preserving, but LLVM does not
1329        // currently support that (https://github.com/llvm/llvm-project/issues/120837).
1330        let mut res = unsafe {
1331            llvm::LLVMBuildAtomicRMW(
1332                self.llbuilder,
1333                AtomicRmwBinOp::from_generic(op),
1334                dst,
1335                src,
1336                AtomicOrdering::from_generic(order),
1337                llvm::FALSE, // SingleThreaded
1338            )
1339        };
1340        if ret_ptr && self.val_ty(res) != self.type_ptr() {
1341            res = self.inttoptr(res, self.type_ptr());
1342        }
1343        res
1344    }
1345
1346    fn atomic_fence(
1347        &mut self,
1348        order: rustc_middle::ty::AtomicOrdering,
1349        scope: SynchronizationScope,
1350    ) {
1351        let single_threaded = match scope {
1352            SynchronizationScope::SingleThread => true,
1353            SynchronizationScope::CrossThread => false,
1354        };
1355        unsafe {
1356            llvm::LLVMBuildFence(
1357                self.llbuilder,
1358                AtomicOrdering::from_generic(order),
1359                single_threaded.to_llvm_bool(),
1360                UNNAMED,
1361            );
1362        }
1363    }
1364
1365    fn set_invariant_load(&mut self, load: &'ll Value) {
1366        self.set_metadata_node(load, llvm::MD_invariant_load, &[]);
1367    }
1368
1369    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1370        self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1371    }
1372
1373    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1374        self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1375    }
1376
1377    fn call(
1378        &mut self,
1379        llty: &'ll Type,
1380        fn_call_attrs: Option<&CodegenFnAttrs>,
1381        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1382        llfn: &'ll Value,
1383        args: &[&'ll Value],
1384        funclet: Option<&Funclet<'ll>>,
1385        instance: Option<Instance<'tcx>>,
1386    ) -> &'ll Value {
1387        debug!("call {:?} with args ({:?})", llfn, args);
1388
1389        let args = self.check_call("call", llty, llfn, args);
1390        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1391        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1392        if let Some(funclet_bundle) = funclet_bundle {
1393            bundles.push(funclet_bundle);
1394        }
1395
1396        // Emit CFI pointer type membership test
1397        self.cfi_type_test(fn_call_attrs, fn_abi, instance, llfn);
1398
1399        // Emit KCFI operand bundle
1400        let kcfi_bundle = self.kcfi_operand_bundle(fn_call_attrs, fn_abi, instance, llfn);
1401        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1402            bundles.push(kcfi_bundle);
1403        }
1404
1405        let call = unsafe {
1406            llvm::LLVMBuildCallWithOperandBundles(
1407                self.llbuilder,
1408                llty,
1409                llfn,
1410                args.as_ptr() as *const &llvm::Value,
1411                args.len() as c_uint,
1412                bundles.as_ptr(),
1413                bundles.len() as c_uint,
1414                c"".as_ptr(),
1415            )
1416        };
1417
1418        if let Some(instance) = instance {
1419            // Attributes on the function definition being called
1420            let fn_defn_attrs = self.cx.tcx.codegen_fn_attrs(instance.def_id());
1421            if let Some(fn_call_attrs) = fn_call_attrs
1422                && !fn_call_attrs.target_features.is_empty()
1423                // If there is an inline attribute and a target feature that matches
1424                // we will add the attribute to the callsite otherwise we'll omit
1425                // this and not add the attribute to prevent soundness issues.
1426                && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance)
1427                && self.cx.tcx.is_target_feature_call_safe(
1428                    &fn_call_attrs.target_features,
1429                    &fn_defn_attrs.target_features,
1430                )
1431            {
1432                attributes::apply_to_callsite(
1433                    call,
1434                    llvm::AttributePlace::Function,
1435                    &[inlining_rule],
1436                );
1437            }
1438        }
1439
1440        if let Some(fn_abi) = fn_abi {
1441            fn_abi.apply_attrs_callsite(self, call);
1442        }
1443        call
1444    }
1445
1446    fn tail_call(
1447        &mut self,
1448        llty: Self::Type,
1449        fn_attrs: Option<&CodegenFnAttrs>,
1450        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
1451        llfn: Self::Value,
1452        args: &[Self::Value],
1453        funclet: Option<&Self::Funclet>,
1454        instance: Option<Instance<'tcx>>,
1455    ) {
1456        let call = self.call(llty, fn_attrs, Some(fn_abi), llfn, args, funclet, instance);
1457        llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail);
1458
1459        match &fn_abi.ret.mode {
1460            PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(),
1461            PassMode::Direct(_) | PassMode::Pair { .. } => self.ret(call),
1462            mode @ PassMode::Cast { .. } => {
1463                bug!("Encountered `PassMode::{mode:?}` during codegen")
1464            }
1465        }
1466    }
1467
1468    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1469        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1470    }
1471
1472    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1473        // Cleanup is always the cold path.
1474        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1475        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1476    }
1477}
1478
1479impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1480    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1481        // Forward to the `get_static` method of `CodegenCx`
1482        let global = self.cx().get_static(def_id);
1483        if self.cx().tcx.is_thread_local_static(def_id) {
1484            let pointer =
1485                self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1486            // Cast to default address space if globals are in a different addrspace
1487            self.pointercast(pointer, self.type_ptr())
1488        } else {
1489            // Cast to default address space if globals are in a different addrspace
1490            self.cx().const_pointercast(global, self.type_ptr())
1491        }
1492    }
1493}
1494
1495impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1496    pub(crate) fn llfn(&self) -> &'ll Value {
1497        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1498    }
1499}
1500
1501impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1502    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1503        unsafe {
1504            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1505        }
1506    }
1507}
1508impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1509    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1510        let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1511        self.set_metadata_node(load, llvm::MD_align, &md);
1512    }
1513
1514    fn noundef_metadata(&mut self, load: &'ll Value) {
1515        self.set_metadata_node(load, llvm::MD_noundef, &[]);
1516    }
1517
1518    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1519        self.set_metadata_node(inst, llvm::MD_unpredictable, &[]);
1520    }
1521
1522    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1523        self.call_intrinsic("llvm.minnum", &[self.val_ty(lhs)], &[lhs, rhs])
1524    }
1525
1526    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1527        self.call_intrinsic("llvm.maxnum", &[self.val_ty(lhs)], &[lhs, rhs])
1528    }
1529
1530    pub(crate) fn insert_element(
1531        &mut self,
1532        vec: &'ll Value,
1533        elt: &'ll Value,
1534        idx: &'ll Value,
1535    ) -> &'ll Value {
1536        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1537    }
1538
1539    pub(crate) fn shuffle_vector(
1540        &mut self,
1541        v1: &'ll Value,
1542        v2: &'ll Value,
1543        mask: &'ll Value,
1544    ) -> &'ll Value {
1545        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1546    }
1547
1548    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1549        self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src])
1550    }
1551    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1552        self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src])
1553    }
1554    pub(crate) fn vector_reduce_fadd_reassoc(
1555        &mut self,
1556        acc: &'ll Value,
1557        src: &'ll Value,
1558    ) -> &'ll Value {
1559        unsafe {
1560            let instr =
1561                self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]);
1562            llvm::LLVMRustSetAllowReassoc(instr);
1563            instr
1564        }
1565    }
1566    pub(crate) fn vector_reduce_fmul_reassoc(
1567        &mut self,
1568        acc: &'ll Value,
1569        src: &'ll Value,
1570    ) -> &'ll Value {
1571        unsafe {
1572            let instr =
1573                self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]);
1574            llvm::LLVMRustSetAllowReassoc(instr);
1575            instr
1576        }
1577    }
1578    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1579        self.call_intrinsic("llvm.vector.reduce.add", &[self.val_ty(src)], &[src])
1580    }
1581    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1582        self.call_intrinsic("llvm.vector.reduce.mul", &[self.val_ty(src)], &[src])
1583    }
1584    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1585        self.call_intrinsic("llvm.vector.reduce.and", &[self.val_ty(src)], &[src])
1586    }
1587    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1588        self.call_intrinsic("llvm.vector.reduce.or", &[self.val_ty(src)], &[src])
1589    }
1590    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1591        self.call_intrinsic("llvm.vector.reduce.xor", &[self.val_ty(src)], &[src])
1592    }
1593    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1594        self.call_intrinsic("llvm.vector.reduce.fmin", &[self.val_ty(src)], &[src])
1595    }
1596    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1597        self.call_intrinsic("llvm.vector.reduce.fmax", &[self.val_ty(src)], &[src])
1598    }
1599    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1600        self.call_intrinsic(
1601            if is_signed { "llvm.vector.reduce.smin" } else { "llvm.vector.reduce.umin" },
1602            &[self.val_ty(src)],
1603            &[src],
1604        )
1605    }
1606    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1607        self.call_intrinsic(
1608            if is_signed { "llvm.vector.reduce.smax" } else { "llvm.vector.reduce.umax" },
1609            &[self.val_ty(src)],
1610            &[src],
1611        )
1612    }
1613}
1614impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1615    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1616        unsafe {
1617            llvm::LLVMAddClause(landing_pad, clause);
1618        }
1619    }
1620
1621    pub(crate) fn catch_ret(
1622        &mut self,
1623        funclet: &Funclet<'ll>,
1624        unwind: &'ll BasicBlock,
1625    ) -> &'ll Value {
1626        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1627        ret.expect("LLVM does not have support for catchret")
1628    }
1629
1630    fn check_call<'b>(
1631        &mut self,
1632        typ: &str,
1633        fn_ty: &'ll Type,
1634        llfn: &'ll Value,
1635        args: &'b [&'ll Value],
1636    ) -> Cow<'b, [&'ll Value]> {
1637        assert!(
1638            self.cx.type_kind(fn_ty) == TypeKind::Function,
1639            "builder::{typ} not passed a function, but {fn_ty:?}"
1640        );
1641
1642        let param_tys = self.cx.func_params_types(fn_ty);
1643
1644        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1645            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1646
1647        if all_args_match {
1648            return Cow::Borrowed(args);
1649        }
1650
1651        let casted_args: Vec<_> = iter::zip(param_tys, args)
1652            .enumerate()
1653            .map(|(i, (expected_ty, &actual_val))| {
1654                let actual_ty = self.cx.val_ty(actual_val);
1655                if expected_ty != actual_ty {
1656                    debug!(
1657                        "type mismatch in function call of {:?}. \
1658                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1659                        llfn, expected_ty, i, actual_ty
1660                    );
1661                    self.bitcast(actual_val, expected_ty)
1662                } else {
1663                    actual_val
1664                }
1665            })
1666            .collect();
1667
1668        Cow::Owned(casted_args)
1669    }
1670
1671    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1672        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1673    }
1674}
1675
1676impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1677    pub(crate) fn call_intrinsic(
1678        &mut self,
1679        base_name: impl Into<Cow<'static, str>>,
1680        type_params: &[&'ll Type],
1681        args: &[&'ll Value],
1682    ) -> &'ll Value {
1683        let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1684        self.call(ty, None, None, f, args, None, None)
1685    }
1686
1687    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1688        let size = size.bytes();
1689        if size == 0 {
1690            return;
1691        }
1692
1693        if !self.cx().sess().emit_lifetime_markers() {
1694            return;
1695        }
1696
1697        if crate::llvm_util::get_version() >= (22, 0, 0) {
1698            self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]);
1699        } else {
1700            self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
1701        }
1702    }
1703}
1704impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1705    pub(crate) fn phi(
1706        &mut self,
1707        ty: &'ll Type,
1708        vals: &[&'ll Value],
1709        bbs: &[&'ll BasicBlock],
1710    ) -> &'ll Value {
1711        assert_eq!(vals.len(), bbs.len());
1712        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1713        unsafe {
1714            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1715            phi
1716        }
1717    }
1718
1719    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1720        unsafe {
1721            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1722        }
1723    }
1724}
1725impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1726    pub(crate) fn landing_pad(
1727        &mut self,
1728        ty: &'ll Type,
1729        pers_fn: &'ll Value,
1730        num_clauses: usize,
1731    ) -> &'ll Value {
1732        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1733        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1734        // personality lives on the parent function anyway.
1735        self.set_personality_fn(pers_fn);
1736        unsafe {
1737            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1738        }
1739    }
1740
1741    pub(crate) fn callbr(
1742        &mut self,
1743        llty: &'ll Type,
1744        fn_attrs: Option<&CodegenFnAttrs>,
1745        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1746        llfn: &'ll Value,
1747        args: &[&'ll Value],
1748        default_dest: &'ll BasicBlock,
1749        indirect_dest: &[&'ll BasicBlock],
1750        funclet: Option<&Funclet<'ll>>,
1751        instance: Option<Instance<'tcx>>,
1752    ) -> &'ll Value {
1753        debug!("invoke {:?} with args ({:?})", llfn, args);
1754
1755        let args = self.check_call("callbr", llty, llfn, args);
1756        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1757        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1758        if let Some(funclet_bundle) = funclet_bundle {
1759            bundles.push(funclet_bundle);
1760        }
1761
1762        // Emit CFI pointer type membership test
1763        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1764
1765        // Emit KCFI operand bundle
1766        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1767        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1768            bundles.push(kcfi_bundle);
1769        }
1770
1771        let callbr = unsafe {
1772            llvm::LLVMBuildCallBr(
1773                self.llbuilder,
1774                llty,
1775                llfn,
1776                default_dest,
1777                indirect_dest.as_ptr(),
1778                indirect_dest.len() as c_uint,
1779                args.as_ptr(),
1780                args.len() as c_uint,
1781                bundles.as_ptr(),
1782                bundles.len() as c_uint,
1783                UNNAMED,
1784            )
1785        };
1786        if let Some(fn_abi) = fn_abi {
1787            fn_abi.apply_attrs_callsite(self, callbr);
1788        }
1789        callbr
1790    }
1791
1792    // Emits CFI pointer type membership tests.
1793    fn cfi_type_test(
1794        &mut self,
1795        fn_attrs: Option<&CodegenFnAttrs>,
1796        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1797        instance: Option<Instance<'tcx>>,
1798        llfn: &'ll Value,
1799    ) {
1800        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1801        if self.tcx.sess.is_sanitizer_cfi_enabled()
1802            && let Some(fn_abi) = fn_abi
1803            && is_indirect_call
1804        {
1805            if let Some(fn_attrs) = fn_attrs
1806                && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1807            {
1808                return;
1809            }
1810
1811            let mut options = cfi::TypeIdOptions::empty();
1812            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1813                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1814            }
1815            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1816                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1817            }
1818
1819            let typeid = if let Some(instance) = instance {
1820                cfi::typeid_for_instance(self.tcx, instance, options)
1821            } else {
1822                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1823            };
1824            let typeid_metadata = self.cx.create_metadata(typeid.as_bytes());
1825            let dbg_loc = self.get_dbg_loc();
1826
1827            // Test whether the function pointer is associated with the type identifier using the
1828            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
1829            // calls to this intrinsic with code to test type membership.
1830            let typeid = self.get_metadata_value(typeid_metadata);
1831            let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
1832            let bb_pass = self.append_sibling_block("type_test.pass");
1833            let bb_fail = self.append_sibling_block("type_test.fail");
1834            self.cond_br(cond, bb_pass, bb_fail);
1835
1836            self.switch_to_block(bb_fail);
1837            if let Some(dbg_loc) = dbg_loc {
1838                self.set_dbg_loc(dbg_loc);
1839            }
1840            self.abort();
1841            self.unreachable();
1842
1843            self.switch_to_block(bb_pass);
1844            if let Some(dbg_loc) = dbg_loc {
1845                self.set_dbg_loc(dbg_loc);
1846            }
1847        }
1848    }
1849
1850    // Emits KCFI operand bundles.
1851    fn kcfi_operand_bundle(
1852        &mut self,
1853        fn_attrs: Option<&CodegenFnAttrs>,
1854        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1855        instance: Option<Instance<'tcx>>,
1856        llfn: &'ll Value,
1857    ) -> Option<llvm::OperandBundleBox<'ll>> {
1858        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1859        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1860            && let Some(fn_abi) = fn_abi
1861            && is_indirect_call
1862        {
1863            if let Some(fn_attrs) = fn_attrs
1864                && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1865            {
1866                return None;
1867            }
1868
1869            let mut options = kcfi::TypeIdOptions::empty();
1870            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1871                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1872            }
1873            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1874                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1875            }
1876
1877            let kcfi_typeid = if let Some(instance) = instance {
1878                kcfi::typeid_for_instance(self.tcx, instance, options)
1879            } else {
1880                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1881            };
1882
1883            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1884        } else {
1885            None
1886        };
1887        kcfi_bundle
1888    }
1889
1890    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1891    #[instrument(level = "debug", skip(self))]
1892    pub(crate) fn instrprof_increment(
1893        &mut self,
1894        fn_name: &'ll Value,
1895        hash: &'ll Value,
1896        num_counters: &'ll Value,
1897        index: &'ll Value,
1898    ) {
1899        self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
1900    }
1901}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
