rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::iter;
3use std::ops::Deref;
4
5use rustc_ast::expand::typetree::FncTree;
6pub(crate) mod autodiff;
7pub(crate) mod gpu_offload;
8
9use libc::{c_char, c_uint};
10use rustc_abi as abi;
11use rustc_abi::{Align, Size, WrappingRange};
12use rustc_codegen_ssa::MemFlags;
13use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
14use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
15use rustc_codegen_ssa::mir::place::PlaceRef;
16use rustc_codegen_ssa::traits::*;
17use rustc_data_structures::small_c_str::SmallCStr;
18use rustc_hir::def_id::DefId;
19use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, TargetFeature, TargetFeatureKind};
20use rustc_middle::ty::layout::{
21    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
22    TyAndLayout,
23};
24use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
25use rustc_sanitizers::{cfi, kcfi};
26use rustc_session::config::OptLevel;
27use rustc_span::Span;
28use rustc_target::callconv::{FnAbi, PassMode};
29use rustc_target::spec::{Arch, HasTargetSpec, SanitizerSet, Target};
30use smallvec::SmallVec;
31use tracing::{debug, instrument};
32
33use crate::abi::FnAbiLlvmExt;
34use crate::attributes;
35use crate::common::Funclet;
36use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
37use crate::llvm::{
38    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE,
39    ToLlvmBool, Type, Value,
40};
41use crate::type_of::LayoutLlvmExt;
42
43#[must_use]
44pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
45    pub llbuilder: &'ll mut llvm::Builder<'ll>,
46    pub cx: &'a GenericCx<'ll, CX>,
47}
48
49pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
50pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
51
52impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
53    fn drop(&mut self) {
54        unsafe {
55            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
56        }
57    }
58}
59
60impl<'a, 'll> SBuilder<'a, 'll> {
61    pub(crate) fn call(
62        &mut self,
63        llty: &'ll Type,
64        llfn: &'ll Value,
65        args: &[&'ll Value],
66        funclet: Option<&Funclet<'ll>>,
67    ) -> &'ll Value {
68        debug!("call {:?} with args ({:?})", llfn, args);
69
70        let args = self.check_call("call", llty, llfn, args);
71        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
72        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
73        if let Some(funclet_bundle) = funclet_bundle {
74            bundles.push(funclet_bundle);
75        }
76
77        let call = unsafe {
78            llvm::LLVMBuildCallWithOperandBundles(
79                self.llbuilder,
80                llty,
81                llfn,
82                args.as_ptr() as *const &llvm::Value,
83                args.len() as c_uint,
84                bundles.as_ptr(),
85                bundles.len() as c_uint,
86                c"".as_ptr(),
87            )
88        };
89        call
90    }
91}
92
93impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
94    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
95        // Create a fresh builder from the simple context.
96        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
97        GenericBuilder { llbuilder, cx: scx }
98    }
99
100    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
101        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
102    }
103
104    pub(crate) fn ret_void(&mut self) {
105        llvm::LLVMBuildRetVoid(self.llbuilder);
106    }
107
108    pub(crate) fn ret(&mut self, v: &'ll Value) {
109        unsafe {
110            llvm::LLVMBuildRet(self.llbuilder, v);
111        }
112    }
113
114    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
115        let bx = Self::with_cx(cx);
116        unsafe {
117            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
118        }
119        bx
120    }
121
122    // The generic builder has less functionality and thus (unlike the other alloca) we can not
123    // easily jump to the beginning of the function to place our allocas there. We trust the user
124    // to manually do that. FIXME(offload): improve the genericCx and add more llvm wrappers to
125    // handle this.
126    pub(crate) fn direct_alloca(&mut self, ty: &'ll Type, align: Align, name: &str) -> &'ll Value {
127        let val = unsafe {
128            let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED);
129            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
130            // Cast to default addrspace if necessary
131            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx.type_ptr(), UNNAMED)
132        };
133        if name != "" {
134            let name = std::ffi::CString::new(name).unwrap();
135            llvm::set_value_name(val, &name.as_bytes());
136        }
137        val
138    }
139
140    pub(crate) fn inbounds_gep(
141        &mut self,
142        ty: &'ll Type,
143        ptr: &'ll Value,
144        indices: &[&'ll Value],
145    ) -> &'ll Value {
146        unsafe {
147            llvm::LLVMBuildGEPWithNoWrapFlags(
148                self.llbuilder,
149                ty,
150                ptr,
151                indices.as_ptr(),
152                indices.len() as c_uint,
153                UNNAMED,
154                GEPNoWrapFlags::InBounds,
155            )
156        }
157    }
158
159    pub(crate) fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
160        debug!("Store {:?} -> {:?}", val, ptr);
161        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
162        unsafe {
163            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
164            llvm::LLVMSetAlignment(store, align.bytes() as c_uint);
165            store
166        }
167    }
168
169    pub(crate) fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
170        unsafe {
171            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
172            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
173            load
174        }
175    }
176
177    fn memset(&mut self, ptr: &'ll Value, fill_byte: &'ll Value, size: &'ll Value, align: Align) {
178        unsafe {
179            llvm::LLVMRustBuildMemSet(
180                self.llbuilder,
181                ptr,
182                align.bytes() as c_uint,
183                fill_byte,
184                size,
185                false,
186            );
187        }
188    }
189}
190
191/// Empty string, to be used where LLVM expects an instruction name, indicating
192/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
193// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
194pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
195
196impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
197    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
198    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
199    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
200    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
201    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
202    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
203
204    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
205    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
206    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
207}
208
209impl abi::HasDataLayout for Builder<'_, '_, '_> {
210    fn data_layout(&self) -> &abi::TargetDataLayout {
211        self.cx.data_layout()
212    }
213}
214
215impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
216    #[inline]
217    fn tcx(&self) -> TyCtxt<'tcx> {
218        self.cx.tcx
219    }
220}
221
222impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
223    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
224        self.cx.typing_env()
225    }
226}
227
228impl HasTargetSpec for Builder<'_, '_, '_> {
229    #[inline]
230    fn target_spec(&self) -> &Target {
231        self.cx.target_spec()
232    }
233}
234
235impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
236    #[inline]
237    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
238        self.cx.handle_layout_err(err, span, ty)
239    }
240}
241
242impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
243    #[inline]
244    fn handle_fn_abi_err(
245        &self,
246        err: FnAbiError<'tcx>,
247        span: Span,
248        fn_abi_request: FnAbiRequest<'tcx>,
249    ) -> ! {
250        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
251    }
252}
253
254impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
255    type Target = CodegenCx<'ll, 'tcx>;
256
257    #[inline]
258    fn deref(&self) -> &Self::Target {
259        self.cx
260    }
261}
262
263macro_rules! math_builder_methods {
264    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
265        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
266            unsafe {
267                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
268            }
269        })+
270    }
271}
272
273macro_rules! set_math_builder_methods {
274    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
275        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
276            unsafe {
277                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
278                llvm::$llvm_set_math(instr);
279                instr
280            }
281        })+
282    }
283}
284
285impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
286    type CodegenCx = CodegenCx<'ll, 'tcx>;
287
288    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
289        let bx = Builder::with_cx(cx);
290        unsafe {
291            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
292        }
293        bx
294    }
295
296    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
297        self.cx
298    }
299
300    fn llbb(&self) -> &'ll BasicBlock {
301        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
302    }
303
304    fn set_span(&mut self, _span: Span) {}
305
306    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
307        unsafe {
308            let name = SmallCStr::new(name);
309            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
310        }
311    }
312
313    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
314        Self::append_block(self.cx, self.llfn(), name)
315    }
316
317    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
318        *self = Self::build(self.cx, llbb)
319    }
320
321    fn ret_void(&mut self) {
322        llvm::LLVMBuildRetVoid(self.llbuilder);
323    }
324
325    fn ret(&mut self, v: &'ll Value) {
326        unsafe {
327            llvm::LLVMBuildRet(self.llbuilder, v);
328        }
329    }
330
331    fn br(&mut self, dest: &'ll BasicBlock) {
332        unsafe {
333            llvm::LLVMBuildBr(self.llbuilder, dest);
334        }
335    }
336
337    fn cond_br(
338        &mut self,
339        cond: &'ll Value,
340        then_llbb: &'ll BasicBlock,
341        else_llbb: &'ll BasicBlock,
342    ) {
343        unsafe {
344            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
345        }
346    }
347
348    fn switch(
349        &mut self,
350        v: &'ll Value,
351        else_llbb: &'ll BasicBlock,
352        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
353    ) {
354        let switch =
355            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
356        for (on_val, dest) in cases {
357            let on_val = self.const_uint_big(self.val_ty(v), on_val);
358            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
359        }
360    }
361
362    fn switch_with_weights(
363        &mut self,
364        v: Self::Value,
365        else_llbb: Self::BasicBlock,
366        else_is_cold: bool,
367        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
368    ) {
369        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
370            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
371            return;
372        }
373
374        let id = self.cx.create_metadata(b"branch_weights");
375
376        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
377        // This function handles switch instructions with more than 2 targets and it needs to
378        // emit branch weights metadata instead of using the intrinsic.
379        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
380        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
381        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
382        let weight =
383            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
384
385        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
386        md.push(id);
387        md.push(weight(else_is_cold));
388
389        let switch =
390            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
391        for (on_val, dest, is_cold) in cases {
392            let on_val = self.const_uint_big(self.val_ty(v), on_val);
393            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
394            md.push(weight(is_cold));
395        }
396
397        self.cx.set_metadata_node(switch, llvm::MD_prof, &md);
398    }
399
400    fn invoke(
401        &mut self,
402        llty: &'ll Type,
403        fn_attrs: Option<&CodegenFnAttrs>,
404        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
405        llfn: &'ll Value,
406        args: &[&'ll Value],
407        then: &'ll BasicBlock,
408        catch: &'ll BasicBlock,
409        funclet: Option<&Funclet<'ll>>,
410        instance: Option<Instance<'tcx>>,
411    ) -> &'ll Value {
412        debug!("invoke {:?} with args ({:?})", llfn, args);
413
414        let args = self.check_call("invoke", llty, llfn, args);
415        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
416        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
417        if let Some(funclet_bundle) = funclet_bundle {
418            bundles.push(funclet_bundle);
419        }
420
421        // Emit CFI pointer type membership test
422        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
423
424        // Emit KCFI operand bundle
425        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
426        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
427            bundles.push(kcfi_bundle);
428        }
429
430        let invoke = unsafe {
431            llvm::LLVMBuildInvokeWithOperandBundles(
432                self.llbuilder,
433                llty,
434                llfn,
435                args.as_ptr(),
436                args.len() as c_uint,
437                then,
438                catch,
439                bundles.as_ptr(),
440                bundles.len() as c_uint,
441                UNNAMED,
442            )
443        };
444        if let Some(fn_abi) = fn_abi {
445            fn_abi.apply_attrs_callsite(self, invoke);
446        }
447        invoke
448    }
449
450    fn unreachable(&mut self) {
451        unsafe {
452            llvm::LLVMBuildUnreachable(self.llbuilder);
453        }
454    }
455
456    math_builder_methods! {
457        add(a, b) => LLVMBuildAdd,
458        fadd(a, b) => LLVMBuildFAdd,
459        sub(a, b) => LLVMBuildSub,
460        fsub(a, b) => LLVMBuildFSub,
461        mul(a, b) => LLVMBuildMul,
462        fmul(a, b) => LLVMBuildFMul,
463        udiv(a, b) => LLVMBuildUDiv,
464        exactudiv(a, b) => LLVMBuildExactUDiv,
465        sdiv(a, b) => LLVMBuildSDiv,
466        exactsdiv(a, b) => LLVMBuildExactSDiv,
467        fdiv(a, b) => LLVMBuildFDiv,
468        urem(a, b) => LLVMBuildURem,
469        srem(a, b) => LLVMBuildSRem,
470        frem(a, b) => LLVMBuildFRem,
471        shl(a, b) => LLVMBuildShl,
472        lshr(a, b) => LLVMBuildLShr,
473        ashr(a, b) => LLVMBuildAShr,
474        and(a, b) => LLVMBuildAnd,
475        or(a, b) => LLVMBuildOr,
476        xor(a, b) => LLVMBuildXor,
477        neg(x) => LLVMBuildNeg,
478        fneg(x) => LLVMBuildFNeg,
479        not(x) => LLVMBuildNot,
480        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
481        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
482        unchecked_ssub(x, y) => LLVMBuildNSWSub,
483        unchecked_usub(x, y) => LLVMBuildNUWSub,
484        unchecked_smul(x, y) => LLVMBuildNSWMul,
485        unchecked_umul(x, y) => LLVMBuildNUWMul,
486    }
487
488    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
489        unsafe {
490            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
491            if llvm::LLVMIsAInstruction(add).is_some() {
492                llvm::LLVMSetNUW(add, TRUE);
493                llvm::LLVMSetNSW(add, TRUE);
494            }
495            add
496        }
497    }
498    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
499        unsafe {
500            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
501            if llvm::LLVMIsAInstruction(sub).is_some() {
502                llvm::LLVMSetNUW(sub, TRUE);
503                llvm::LLVMSetNSW(sub, TRUE);
504            }
505            sub
506        }
507    }
508    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
509        unsafe {
510            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
511            if llvm::LLVMIsAInstruction(mul).is_some() {
512                llvm::LLVMSetNUW(mul, TRUE);
513                llvm::LLVMSetNSW(mul, TRUE);
514            }
515            mul
516        }
517    }
518
519    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
520        unsafe {
521            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
522
523            // If a and b are both values, then `or` is a value, rather than
524            // an instruction, so we need to check before setting the flag.
525            // (See also `LLVMBuildNUWNeg` which also needs a check.)
526            if llvm::LLVMIsAInstruction(or).is_some() {
527                llvm::LLVMSetIsDisjoint(or, TRUE);
528            }
529            or
530        }
531    }
532
533    set_math_builder_methods! {
534        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
535        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
536        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
537        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
538        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
539        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
540        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
541        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
542        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
543        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
544    }
545
546    fn checked_binop(
547        &mut self,
548        oop: OverflowOp,
549        ty: Ty<'tcx>,
550        lhs: Self::Value,
551        rhs: Self::Value,
552    ) -> (Self::Value, Self::Value) {
553        let (size, signed) = ty.int_size_and_signed(self.tcx);
554        let width = size.bits();
555
556        if !signed {
557            match oop {
558                OverflowOp::Sub => {
559                    // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
560                    // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
561                    // in the backend if profitable.
562                    let sub = self.sub(lhs, rhs);
563                    let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
564                    return (sub, cmp);
565                }
566                OverflowOp::Add => {
567                    // Like with sub above, using icmp is the preferred form. See
568                    // <https://rust-lang.zulipchat.com/#narrow/channel/187780-t-compiler.2Fllvm/topic/.60uadd.2Ewith.2Eoverflow.60.20.28again.29/near/533041085>
569                    let add = self.add(lhs, rhs);
570                    let cmp = self.icmp(IntPredicate::IntULT, add, lhs);
571                    return (add, cmp);
572                }
573                OverflowOp::Mul => {}
574            }
575        }
576
577        let oop_str = match oop {
578            OverflowOp::Add => "add",
579            OverflowOp::Sub => "sub",
580            OverflowOp::Mul => "mul",
581        };
582
583        let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
584
585        let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
586        (self.extract_value(res, 0), self.extract_value(res, 1))
587    }
588
589    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
590        if self.cx().val_ty(val) == self.cx().type_i1() {
591            self.zext(val, self.cx().type_i8())
592        } else {
593            val
594        }
595    }
596
597    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
598        if scalar.is_bool() {
599            return self.unchecked_utrunc(val, self.cx().type_i1());
600        }
601        val
602    }
603
604    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
605        let mut bx = Builder::with_cx(self.cx);
606        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
607        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
608        unsafe {
609            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
610            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
611            // Cast to default addrspace if necessary
612            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
613        }
614    }
615
616    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
617        unsafe {
618            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
619            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
620            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
621            load
622        }
623    }
624
625    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
626        unsafe {
627            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
628            llvm::LLVMSetVolatile(load, llvm::TRUE);
629            load
630        }
631    }
632
633    fn atomic_load(
634        &mut self,
635        ty: &'ll Type,
636        ptr: &'ll Value,
637        order: rustc_middle::ty::AtomicOrdering,
638        size: Size,
639    ) -> &'ll Value {
640        unsafe {
641            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
642            // Set atomic ordering
643            llvm::LLVMSetOrdering(load, AtomicOrdering::from_generic(order));
644            // LLVM requires the alignment of atomic loads to be at least the size of the type.
645            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
646            load
647        }
648    }
649
650    #[instrument(level = "trace", skip(self))]
651    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
652        if place.layout.is_unsized() {
653            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
654            if matches!(tail.kind(), ty::Foreign(..)) {
655                // Unsized locals and, at least conceptually, even unsized arguments must be copied
656                // around, which requires dynamically determining their size. Therefore, we cannot
657                // allow `extern` types here. Consult t-opsem before removing this check.
658                panic!("unsized locals must not be `extern` types");
659            }
660        }
661        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
662
663        if place.layout.is_zst() {
664            return OperandRef::zero_sized(place.layout);
665        }
666
667        #[instrument(level = "trace", skip(bx))]
668        fn scalar_load_metadata<'a, 'll, 'tcx>(
669            bx: &mut Builder<'a, 'll, 'tcx>,
670            load: &'ll Value,
671            scalar: abi::Scalar,
672            layout: TyAndLayout<'tcx>,
673            offset: Size,
674        ) {
675            if bx.cx.sess().opts.optimize == OptLevel::No {
676                // Don't emit metadata we're not going to use
677                return;
678            }
679
680            if !scalar.is_uninit_valid() {
681                bx.noundef_metadata(load);
682            }
683
684            match scalar.primitive() {
685                abi::Primitive::Int(..) => {
686                    if !scalar.is_always_valid(bx) {
687                        bx.range_metadata(load, scalar.valid_range(bx));
688                    }
689                }
690                abi::Primitive::Pointer(_) => {
691                    if !scalar.valid_range(bx).contains(0) {
692                        bx.nonnull_metadata(load);
693                    }
694
695                    if let Some(pointee) = layout.pointee_info_at(bx, offset)
696                        && let Some(_) = pointee.safe
697                    {
698                        bx.align_metadata(load, pointee.align);
699                    }
700                }
701                abi::Primitive::Float(_) => {}
702            }
703        }
704
705        let val = if let Some(_) = place.val.llextra {
706            // FIXME: Merge with the `else` below?
707            OperandValue::Ref(place.val)
708        } else if place.layout.is_llvm_immediate() {
709            let mut const_llval = None;
710            let llty = place.layout.llvm_type(self);
711            if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
712                if llvm::LLVMIsGlobalConstant(global).is_true() {
713                    if let Some(init) = llvm::LLVMGetInitializer(global) {
714                        if self.val_ty(init) == llty {
715                            const_llval = Some(init);
716                        }
717                    }
718                }
719            }
720
721            let llval = const_llval.unwrap_or_else(|| {
722                let load = self.load(llty, place.val.llval, place.val.align);
723                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
724                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
725                    self.to_immediate_scalar(load, scalar)
726                } else {
727                    load
728                }
729            });
730            OperandValue::Immediate(llval)
731        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
732            let b_offset = a.size(self).align_to(b.align(self).abi);
733
734            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
735                let llptr = if i == 0 {
736                    place.val.llval
737                } else {
738                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
739                };
740                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
741                let load = self.load(llty, llptr, align);
742                scalar_load_metadata(self, load, scalar, layout, offset);
743                self.to_immediate_scalar(load, scalar)
744            };
745
746            OperandValue::Pair(
747                load(0, a, place.layout, place.val.align, Size::ZERO),
748                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
749            )
750        } else {
751            OperandValue::Ref(place.val)
752        };
753
754        OperandRef { val, layout: place.layout, move_annotation: None }
755    }
756
757    fn write_operand_repeatedly(
758        &mut self,
759        cg_elem: OperandRef<'tcx, &'ll Value>,
760        count: u64,
761        dest: PlaceRef<'tcx, &'ll Value>,
762    ) {
763        if self.cx.sess().opts.optimize == OptLevel::No {
764            // To let debuggers single-step over lines like
765            //
766            //     let foo = ["bar"; 42];
767            //
768            // we need the debugger-friendly LLVM IR that `_unoptimized()`
769            // provides. The `_optimized()` version generates trickier LLVM IR.
770            // See PR #148058 for a failed attempt at handling that.
771            self.write_operand_repeatedly_unoptimized(cg_elem, count, dest);
772        } else {
773            self.write_operand_repeatedly_optimized(cg_elem, count, dest);
774        }
775    }
776
777    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
778        if self.cx.sess().opts.optimize == OptLevel::No {
779            // Don't emit metadata we're not going to use
780            return;
781        }
782
783        let llty = self.cx.val_ty(load);
784        let md = [
785            llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
786            llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
787        ];
788        self.set_metadata_node(load, llvm::MD_range, &md);
789    }
790
791    fn nonnull_metadata(&mut self, load: &'ll Value) {
792        self.set_metadata_node(load, llvm::MD_nonnull, &[]);
793    }
794
795    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
796        self.store_with_flags(val, ptr, align, MemFlags::empty())
797    }
798
799    fn store_with_flags(
800        &mut self,
801        val: &'ll Value,
802        ptr: &'ll Value,
803        align: Align,
804        flags: MemFlags,
805    ) -> &'ll Value {
806        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
807        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
808        unsafe {
809            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
810            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
811            let align =
812                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
813            llvm::LLVMSetAlignment(store, align);
814            if flags.contains(MemFlags::VOLATILE) {
815                llvm::LLVMSetVolatile(store, llvm::TRUE);
816            }
817            if flags.contains(MemFlags::NONTEMPORAL) {
818                // Make sure that the current target architectures supports "sane" non-temporal
819                // stores, i.e., non-temporal stores that are equivalent to regular stores except
820                // for performance. LLVM doesn't seem to care about this, and will happily treat
821                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
822                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
823                // regular stores but require special fences. So we keep a list of architectures
824                // where `!nontemporal` is known to be truly just a hint, and use regular stores
825                // everywhere else. (In the future, we could alternatively ensure that an sfence
826                // gets emitted after a sequence of movnt before any kind of synchronizing
827                // operation. But it's not clear how to do that with LLVM.)
828                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
829                // <https://github.com/llvm/llvm-project/issues/64521>.
830                let use_nontemporal = matches!(
831                    self.cx.tcx.sess.target.arch,
832                    Arch::AArch64 | Arch::Arm | Arch::RiscV32 | Arch::RiscV64
833                );
834                if use_nontemporal {
835                    // According to LLVM [1] building a nontemporal store must
836                    // *always* point to a metadata value of the integer 1.
837                    //
838                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
839                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
840                    self.set_metadata_node(store, llvm::MD_nontemporal, &[one]);
841                }
842            }
843            store
844        }
845    }
846
847    fn atomic_store(
848        &mut self,
849        val: &'ll Value,
850        ptr: &'ll Value,
851        order: rustc_middle::ty::AtomicOrdering,
852        size: Size,
853    ) {
854        debug!("Store {:?} -> {:?}", val, ptr);
855        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
856        unsafe {
857            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
858            // Set atomic ordering
859            llvm::LLVMSetOrdering(store, AtomicOrdering::from_generic(order));
860            // LLVM requires the alignment of atomic stores to be at least the size of the type.
861            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
862        }
863    }
864
865    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
866        unsafe {
867            llvm::LLVMBuildGEPWithNoWrapFlags(
868                self.llbuilder,
869                ty,
870                ptr,
871                indices.as_ptr(),
872                indices.len() as c_uint,
873                UNNAMED,
874                GEPNoWrapFlags::default(),
875            )
876        }
877    }
878
879    fn inbounds_gep(
880        &mut self,
881        ty: &'ll Type,
882        ptr: &'ll Value,
883        indices: &[&'ll Value],
884    ) -> &'ll Value {
885        unsafe {
886            llvm::LLVMBuildGEPWithNoWrapFlags(
887                self.llbuilder,
888                ty,
889                ptr,
890                indices.as_ptr(),
891                indices.len() as c_uint,
892                UNNAMED,
893                GEPNoWrapFlags::InBounds,
894            )
895        }
896    }
897
898    fn inbounds_nuw_gep(
899        &mut self,
900        ty: &'ll Type,
901        ptr: &'ll Value,
902        indices: &[&'ll Value],
903    ) -> &'ll Value {
904        unsafe {
905            llvm::LLVMBuildGEPWithNoWrapFlags(
906                self.llbuilder,
907                ty,
908                ptr,
909                indices.as_ptr(),
910                indices.len() as c_uint,
911                UNNAMED,
912                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
913            )
914        }
915    }
916
917    /* Casts */
918    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
919        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
920    }
921
922    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
923        debug_assert_ne!(self.val_ty(val), dest_ty);
924
925        let trunc = self.trunc(val, dest_ty);
926        unsafe {
927            if llvm::LLVMIsAInstruction(trunc).is_some() {
928                llvm::LLVMSetNUW(trunc, TRUE);
929            }
930        }
931        trunc
932    }
933
934    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
935        debug_assert_ne!(self.val_ty(val), dest_ty);
936
937        let trunc = self.trunc(val, dest_ty);
938        unsafe {
939            if llvm::LLVMIsAInstruction(trunc).is_some() {
940                llvm::LLVMSetNSW(trunc, TRUE);
941            }
942        }
943        trunc
944    }
945
946    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
947        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
948    }
949
950    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
951        self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
952    }
953
954    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
955        self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
956    }
957
958    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
959        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
960        // poor codegen. The reason for this is that the corresponding wasm
961        // instructions, `i32.trunc_f32_s` for example, will trap when the float
962        // is out-of-bounds, infinity, or nan. This means that LLVM
963        // automatically inserts control flow around `fptoui` and `fptosi`
964        // because the LLVM instruction `fptoui` is defined as producing a
965        // poison value, not having UB on out-of-bounds values.
966        //
967        // This method, however, is only used with non-saturating casts that
968        // have UB on out-of-bounds values. This means that it's ok if we use
969        // the raw wasm instruction since out-of-bounds values can do whatever
970        // we like. To ensure that LLVM picks the right instruction we choose
971        // the raw wasm intrinsic functions which avoid LLVM inserting all the
972        // other control flow automatically.
973        if self.sess().target.is_like_wasm {
974            let src_ty = self.cx.val_ty(val);
975            if self.cx.type_kind(src_ty) != TypeKind::Vector {
976                let float_width = self.cx.float_width(src_ty);
977                let int_width = self.cx.int_width(dest_ty);
978                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
979                    return self.call_intrinsic(
980                        "llvm.wasm.trunc.unsigned",
981                        &[dest_ty, src_ty],
982                        &[val],
983                    );
984                }
985            }
986        }
987        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
988    }
989
990    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
991        // see `fptoui` above for why wasm is different here
992        if self.sess().target.is_like_wasm {
993            let src_ty = self.cx.val_ty(val);
994            if self.cx.type_kind(src_ty) != TypeKind::Vector {
995                let float_width = self.cx.float_width(src_ty);
996                let int_width = self.cx.int_width(dest_ty);
997                if matches!((int_width, float_width), (32 | 64, 32 | 64)) {
998                    return self.call_intrinsic(
999                        "llvm.wasm.trunc.signed",
1000                        &[dest_ty, src_ty],
1001                        &[val],
1002                    );
1003                }
1004            }
1005        }
1006        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
1007    }
1008
1009    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1010        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1011    }
1012
1013    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1014        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1015    }
1016
1017    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1018        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
1019    }
1020
1021    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1022        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
1023    }
1024
1025    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1026        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
1027    }
1028
1029    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1030        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
1031    }
1032
1033    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1034        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
1035    }
1036
1037    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1038        unsafe {
1039            llvm::LLVMBuildIntCast2(self.llbuilder, val, dest_ty, is_signed.to_llvm_bool(), UNNAMED)
1040        }
1041    }
1042
1043    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1044        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1045    }
1046
1047    /* Comparisons */
1048    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1049        let op = llvm::IntPredicate::from_generic(op);
1050        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1051    }
1052
1053    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1054        let op = llvm::RealPredicate::from_generic(op);
1055        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1056    }
1057
1058    fn three_way_compare(
1059        &mut self,
1060        ty: Ty<'tcx>,
1061        lhs: Self::Value,
1062        rhs: Self::Value,
1063    ) -> Self::Value {
1064        let size = ty.primitive_size(self.tcx);
1065        let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1066
1067        self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])
1068    }
1069
1070    /* Miscellaneous instructions */
1071    fn memcpy(
1072        &mut self,
1073        dst: &'ll Value,
1074        dst_align: Align,
1075        src: &'ll Value,
1076        src_align: Align,
1077        size: &'ll Value,
1078        flags: MemFlags,
1079        tt: Option<FncTree>,
1080    ) {
1081        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1082        let size = self.intcast(size, self.type_isize(), false);
1083        let is_volatile = flags.contains(MemFlags::VOLATILE);
1084        let memcpy = unsafe {
1085            llvm::LLVMRustBuildMemCpy(
1086                self.llbuilder,
1087                dst,
1088                dst_align.bytes() as c_uint,
1089                src,
1090                src_align.bytes() as c_uint,
1091                size,
1092                is_volatile,
1093            )
1094        };
1095
1096        // TypeTree metadata for memcpy is especially important: when Enzyme encounters
1097        // a memcpy during autodiff, it needs to know the structure of the data being
1098        // copied to properly track derivatives. For example, copying an array of floats
1099        // vs. copying a struct with mixed types requires different derivative handling.
1100        // The TypeTree tells Enzyme exactly what memory layout to expect.
1101        if let Some(tt) = tt {
1102            crate::typetree::add_tt(self.cx().llmod, self.cx().llcx, memcpy, tt);
1103        }
1104    }
1105
1106    fn memmove(
1107        &mut self,
1108        dst: &'ll Value,
1109        dst_align: Align,
1110        src: &'ll Value,
1111        src_align: Align,
1112        size: &'ll Value,
1113        flags: MemFlags,
1114    ) {
1115        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1116        let size = self.intcast(size, self.type_isize(), false);
1117        let is_volatile = flags.contains(MemFlags::VOLATILE);
1118        unsafe {
1119            llvm::LLVMRustBuildMemMove(
1120                self.llbuilder,
1121                dst,
1122                dst_align.bytes() as c_uint,
1123                src,
1124                src_align.bytes() as c_uint,
1125                size,
1126                is_volatile,
1127            );
1128        }
1129    }
1130
1131    fn memset(
1132        &mut self,
1133        ptr: &'ll Value,
1134        fill_byte: &'ll Value,
1135        size: &'ll Value,
1136        align: Align,
1137        flags: MemFlags,
1138    ) {
1139        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1140        let is_volatile = flags.contains(MemFlags::VOLATILE);
1141        unsafe {
1142            llvm::LLVMRustBuildMemSet(
1143                self.llbuilder,
1144                ptr,
1145                align.bytes() as c_uint,
1146                fill_byte,
1147                size,
1148                is_volatile,
1149            );
1150        }
1151    }
1152
1153    fn select(
1154        &mut self,
1155        cond: &'ll Value,
1156        then_val: &'ll Value,
1157        else_val: &'ll Value,
1158    ) -> &'ll Value {
1159        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1160    }
1161
1162    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1163        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1164    }
1165
1166    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1167        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1168    }
1169
1170    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1171        unsafe {
1172            let elt_ty = self.cx.val_ty(elt);
1173            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1174            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1175            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1176            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1177        }
1178    }
1179
1180    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1181        assert_eq!(idx as c_uint as u64, idx);
1182        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1183    }
1184
1185    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1186        assert_eq!(idx as c_uint as u64, idx);
1187        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1188    }
1189
1190    fn set_personality_fn(&mut self, personality: &'ll Value) {
1191        unsafe {
1192            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1193        }
1194    }
1195
1196    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1197        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1198        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1199        unsafe {
1200            llvm::LLVMSetCleanup(landing_pad, llvm::TRUE);
1201        }
1202        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1203    }
1204
1205    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) {
1206        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1207        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1208        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1209    }
1210
1211    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1212        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1213        let mut exn = self.const_poison(ty);
1214        exn = self.insert_value(exn, exn0, 0);
1215        exn = self.insert_value(exn, exn1, 1);
1216        unsafe {
1217            llvm::LLVMBuildResume(self.llbuilder, exn);
1218        }
1219    }
1220
1221    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1222        let ret = unsafe {
1223            llvm::LLVMBuildCleanupPad(
1224                self.llbuilder,
1225                parent,
1226                args.as_ptr(),
1227                args.len() as c_uint,
1228                c"cleanuppad".as_ptr(),
1229            )
1230        };
1231        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1232    }
1233
1234    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1235        unsafe {
1236            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1237                .expect("LLVM does not have support for cleanupret");
1238        }
1239    }
1240
1241    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1242        let ret = unsafe {
1243            llvm::LLVMBuildCatchPad(
1244                self.llbuilder,
1245                parent,
1246                args.as_ptr(),
1247                args.len() as c_uint,
1248                c"catchpad".as_ptr(),
1249            )
1250        };
1251        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1252    }
1253
1254    fn catch_switch(
1255        &mut self,
1256        parent: Option<&'ll Value>,
1257        unwind: Option<&'ll BasicBlock>,
1258        handlers: &[&'ll BasicBlock],
1259    ) -> &'ll Value {
1260        let ret = unsafe {
1261            llvm::LLVMBuildCatchSwitch(
1262                self.llbuilder,
1263                parent,
1264                unwind,
1265                handlers.len() as c_uint,
1266                c"catchswitch".as_ptr(),
1267            )
1268        };
1269        let ret = ret.expect("LLVM does not have support for catchswitch");
1270        for handler in handlers {
1271            unsafe {
1272                llvm::LLVMAddHandler(ret, handler);
1273            }
1274        }
1275        ret
1276    }
1277
1278    // Atomic Operations
1279    fn atomic_cmpxchg(
1280        &mut self,
1281        dst: &'ll Value,
1282        cmp: &'ll Value,
1283        src: &'ll Value,
1284        order: rustc_middle::ty::AtomicOrdering,
1285        failure_order: rustc_middle::ty::AtomicOrdering,
1286        weak: bool,
1287    ) -> (&'ll Value, &'ll Value) {
1288        unsafe {
1289            let value = llvm::LLVMBuildAtomicCmpXchg(
1290                self.llbuilder,
1291                dst,
1292                cmp,
1293                src,
1294                AtomicOrdering::from_generic(order),
1295                AtomicOrdering::from_generic(failure_order),
1296                llvm::FALSE, // SingleThreaded
1297            );
1298            llvm::LLVMSetWeak(value, weak.to_llvm_bool());
1299            let val = self.extract_value(value, 0);
1300            let success = self.extract_value(value, 1);
1301            (val, success)
1302        }
1303    }
1304
1305    fn atomic_rmw(
1306        &mut self,
1307        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1308        dst: &'ll Value,
1309        src: &'ll Value,
1310        order: rustc_middle::ty::AtomicOrdering,
1311        ret_ptr: bool,
1312    ) -> &'ll Value {
1313        // FIXME: If `ret_ptr` is true and `src` is not a pointer, we *should* tell LLVM that the
1314        // LHS is a pointer and the operation should be provenance-preserving, but LLVM does not
1315        // currently support that (https://github.com/llvm/llvm-project/issues/120837).
1316        let mut res = unsafe {
1317            llvm::LLVMBuildAtomicRMW(
1318                self.llbuilder,
1319                AtomicRmwBinOp::from_generic(op),
1320                dst,
1321                src,
1322                AtomicOrdering::from_generic(order),
1323                llvm::FALSE, // SingleThreaded
1324            )
1325        };
1326        if ret_ptr && self.val_ty(res) != self.type_ptr() {
1327            res = self.inttoptr(res, self.type_ptr());
1328        }
1329        res
1330    }
1331
1332    fn atomic_fence(
1333        &mut self,
1334        order: rustc_middle::ty::AtomicOrdering,
1335        scope: SynchronizationScope,
1336    ) {
1337        let single_threaded = match scope {
1338            SynchronizationScope::SingleThread => true,
1339            SynchronizationScope::CrossThread => false,
1340        };
1341        unsafe {
1342            llvm::LLVMBuildFence(
1343                self.llbuilder,
1344                AtomicOrdering::from_generic(order),
1345                single_threaded.to_llvm_bool(),
1346                UNNAMED,
1347            );
1348        }
1349    }
1350
1351    fn set_invariant_load(&mut self, load: &'ll Value) {
1352        self.set_metadata_node(load, llvm::MD_invariant_load, &[]);
1353    }
1354
1355    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1356        self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1357    }
1358
1359    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1360        self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1361    }
1362
1363    fn call(
1364        &mut self,
1365        llty: &'ll Type,
1366        fn_call_attrs: Option<&CodegenFnAttrs>,
1367        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1368        llfn: &'ll Value,
1369        args: &[&'ll Value],
1370        funclet: Option<&Funclet<'ll>>,
1371        instance: Option<Instance<'tcx>>,
1372    ) -> &'ll Value {
1373        debug!("call {:?} with args ({:?})", llfn, args);
1374
1375        let args = self.check_call("call", llty, llfn, args);
1376        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1377        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1378        if let Some(funclet_bundle) = funclet_bundle {
1379            bundles.push(funclet_bundle);
1380        }
1381
1382        // Emit CFI pointer type membership test
1383        self.cfi_type_test(fn_call_attrs, fn_abi, instance, llfn);
1384
1385        // Emit KCFI operand bundle
1386        let kcfi_bundle = self.kcfi_operand_bundle(fn_call_attrs, fn_abi, instance, llfn);
1387        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1388            bundles.push(kcfi_bundle);
1389        }
1390
1391        let call = unsafe {
1392            llvm::LLVMBuildCallWithOperandBundles(
1393                self.llbuilder,
1394                llty,
1395                llfn,
1396                args.as_ptr() as *const &llvm::Value,
1397                args.len() as c_uint,
1398                bundles.as_ptr(),
1399                bundles.len() as c_uint,
1400                c"".as_ptr(),
1401            )
1402        };
1403
1404        if let Some(instance) = instance {
1405            // Attributes on the function definition being called
1406            let fn_defn_attrs = self.cx.tcx.codegen_fn_attrs(instance.def_id());
1407            if let Some(fn_call_attrs) = fn_call_attrs
1408                // If there is an inline attribute and a target feature that matches
1409                // we will add the attribute to the callsite otherwise we'll omit
1410                // this and not add the attribute to prevent soundness issues.
1411                && let Some(inlining_rule) = attributes::inline_attr(&self.cx, self.cx.tcx, instance)
1412                && self.cx.tcx.is_target_feature_call_safe(
1413                    &fn_defn_attrs.target_features,
1414                    &fn_call_attrs.target_features.iter().cloned().chain(
1415                        self.cx.tcx.sess.target_features.iter().map(|feat| TargetFeature {
1416                            name: *feat,
1417                            kind: TargetFeatureKind::Implied,
1418                        })
1419                    ).collect::<Vec<_>>(),
1420                )
1421            {
1422                attributes::apply_to_callsite(
1423                    call,
1424                    llvm::AttributePlace::Function,
1425                    &[inlining_rule],
1426                );
1427            }
1428        }
1429
1430        if let Some(fn_abi) = fn_abi {
1431            fn_abi.apply_attrs_callsite(self, call);
1432        }
1433        call
1434    }
1435
1436    fn tail_call(
1437        &mut self,
1438        llty: Self::Type,
1439        fn_attrs: Option<&CodegenFnAttrs>,
1440        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
1441        llfn: Self::Value,
1442        args: &[Self::Value],
1443        funclet: Option<&Self::Funclet>,
1444        instance: Option<Instance<'tcx>>,
1445    ) {
1446        let call = self.call(llty, fn_attrs, Some(fn_abi), llfn, args, funclet, instance);
1447        llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail);
1448
1449        match &fn_abi.ret.mode {
1450            PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(),
1451            PassMode::Direct(_) | PassMode::Pair { .. } | PassMode::Cast { .. } => self.ret(call),
1452        }
1453    }
1454
1455    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1456        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1457    }
1458
1459    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1460        // Cleanup is always the cold path.
1461        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1462        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1463    }
1464}
1465
1466impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1467    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1468        // Forward to the `get_static` method of `CodegenCx`
1469        let global = self.cx().get_static(def_id);
1470        if self.cx().tcx.is_thread_local_static(def_id) {
1471            let pointer =
1472                self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1473            // Cast to default address space if globals are in a different addrspace
1474            self.pointercast(pointer, self.type_ptr())
1475        } else {
1476            // Cast to default address space if globals are in a different addrspace
1477            self.cx().const_pointercast(global, self.type_ptr())
1478        }
1479    }
1480}
1481
1482impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1483    pub(crate) fn llfn(&self) -> &'ll Value {
1484        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1485    }
1486}
1487
1488impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1489    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1490        unsafe {
1491            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1492        }
1493    }
1494}
1495impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1496    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1497        let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1498        self.set_metadata_node(load, llvm::MD_align, &md);
1499    }
1500
1501    fn noundef_metadata(&mut self, load: &'ll Value) {
1502        self.set_metadata_node(load, llvm::MD_noundef, &[]);
1503    }
1504
1505    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1506        self.set_metadata_node(inst, llvm::MD_unpredictable, &[]);
1507    }
1508
1509    fn write_operand_repeatedly_optimized(
1510        &mut self,
1511        cg_elem: OperandRef<'tcx, &'ll Value>,
1512        count: u64,
1513        dest: PlaceRef<'tcx, &'ll Value>,
1514    ) {
1515        let zero = self.const_usize(0);
1516        let count = self.const_usize(count);
1517
1518        let header_bb = self.append_sibling_block("repeat_loop_header");
1519        let body_bb = self.append_sibling_block("repeat_loop_body");
1520        let next_bb = self.append_sibling_block("repeat_loop_next");
1521
1522        self.br(header_bb);
1523
1524        let mut header_bx = Self::build(self.cx, header_bb);
1525        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
1526
1527        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
1528        header_bx.cond_br(keep_going, body_bb, next_bb);
1529
1530        let mut body_bx = Self::build(self.cx, body_bb);
1531        let dest_elem = dest.project_index(&mut body_bx, i);
1532        cg_elem.val.store(&mut body_bx, dest_elem);
1533
1534        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
1535        body_bx.br(header_bb);
1536        header_bx.add_incoming_to_phi(i, next, body_bb);
1537
1538        *self = Self::build(self.cx, next_bb);
1539    }
1540
1541    fn write_operand_repeatedly_unoptimized(
1542        &mut self,
1543        cg_elem: OperandRef<'tcx, &'ll Value>,
1544        count: u64,
1545        dest: PlaceRef<'tcx, &'ll Value>,
1546    ) {
1547        let zero = self.const_usize(0);
1548        let count = self.const_usize(count);
1549        let start = dest.project_index(self, zero).val.llval;
1550        let end = dest.project_index(self, count).val.llval;
1551
1552        let header_bb = self.append_sibling_block("repeat_loop_header");
1553        let body_bb = self.append_sibling_block("repeat_loop_body");
1554        let next_bb = self.append_sibling_block("repeat_loop_next");
1555
1556        self.br(header_bb);
1557
1558        let mut header_bx = Self::build(self.cx, header_bb);
1559        let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]);
1560
1561        let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end);
1562        header_bx.cond_br(keep_going, body_bb, next_bb);
1563
1564        let mut body_bx = Self::build(self.cx, body_bb);
1565        let align = dest.val.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
1566        cg_elem
1567            .val
1568            .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align));
1569
1570        let next = body_bx.inbounds_gep(
1571            self.backend_type(cg_elem.layout),
1572            current,
1573            &[self.const_usize(1)],
1574        );
1575        body_bx.br(header_bb);
1576        header_bx.add_incoming_to_phi(current, next, body_bb);
1577
1578        *self = Self::build(self.cx, next_bb);
1579    }
1580
1581    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1582        self.call_intrinsic("llvm.minnum", &[self.val_ty(lhs)], &[lhs, rhs])
1583    }
1584
1585    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1586        self.call_intrinsic("llvm.maxnum", &[self.val_ty(lhs)], &[lhs, rhs])
1587    }
1588
1589    pub(crate) fn insert_element(
1590        &mut self,
1591        vec: &'ll Value,
1592        elt: &'ll Value,
1593        idx: &'ll Value,
1594    ) -> &'ll Value {
1595        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1596    }
1597
1598    pub(crate) fn shuffle_vector(
1599        &mut self,
1600        v1: &'ll Value,
1601        v2: &'ll Value,
1602        mask: &'ll Value,
1603    ) -> &'ll Value {
1604        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1605    }
1606
1607    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1608        self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src])
1609    }
1610    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1611        self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src])
1612    }
1613    pub(crate) fn vector_reduce_fadd_reassoc(
1614        &mut self,
1615        acc: &'ll Value,
1616        src: &'ll Value,
1617    ) -> &'ll Value {
1618        unsafe {
1619            let instr =
1620                self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]);
1621            llvm::LLVMRustSetAllowReassoc(instr);
1622            instr
1623        }
1624    }
1625    pub(crate) fn vector_reduce_fmul_reassoc(
1626        &mut self,
1627        acc: &'ll Value,
1628        src: &'ll Value,
1629    ) -> &'ll Value {
1630        unsafe {
1631            let instr =
1632                self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]);
1633            llvm::LLVMRustSetAllowReassoc(instr);
1634            instr
1635        }
1636    }
1637    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1638        self.call_intrinsic("llvm.vector.reduce.add", &[self.val_ty(src)], &[src])
1639    }
1640    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1641        self.call_intrinsic("llvm.vector.reduce.mul", &[self.val_ty(src)], &[src])
1642    }
1643    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1644        self.call_intrinsic("llvm.vector.reduce.and", &[self.val_ty(src)], &[src])
1645    }
1646    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1647        self.call_intrinsic("llvm.vector.reduce.or", &[self.val_ty(src)], &[src])
1648    }
1649    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1650        self.call_intrinsic("llvm.vector.reduce.xor", &[self.val_ty(src)], &[src])
1651    }
1652    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1653        self.call_intrinsic("llvm.vector.reduce.fmin", &[self.val_ty(src)], &[src])
1654    }
1655    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1656        self.call_intrinsic("llvm.vector.reduce.fmax", &[self.val_ty(src)], &[src])
1657    }
1658    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1659        self.call_intrinsic(
1660            if is_signed { "llvm.vector.reduce.smin" } else { "llvm.vector.reduce.umin" },
1661            &[self.val_ty(src)],
1662            &[src],
1663        )
1664    }
1665    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1666        self.call_intrinsic(
1667            if is_signed { "llvm.vector.reduce.smax" } else { "llvm.vector.reduce.umax" },
1668            &[self.val_ty(src)],
1669            &[src],
1670        )
1671    }
1672}
1673impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1674    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1675        unsafe {
1676            llvm::LLVMAddClause(landing_pad, clause);
1677        }
1678    }
1679
1680    pub(crate) fn catch_ret(
1681        &mut self,
1682        funclet: &Funclet<'ll>,
1683        unwind: &'ll BasicBlock,
1684    ) -> &'ll Value {
1685        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1686        ret.expect("LLVM does not have support for catchret")
1687    }
1688
1689    fn check_call<'b>(
1690        &mut self,
1691        typ: &str,
1692        fn_ty: &'ll Type,
1693        llfn: &'ll Value,
1694        args: &'b [&'ll Value],
1695    ) -> Cow<'b, [&'ll Value]> {
1696        assert!(
1697            self.cx.type_kind(fn_ty) == TypeKind::Function,
1698            "builder::{typ} not passed a function, but {fn_ty:?}"
1699        );
1700
1701        let param_tys = self.cx.func_params_types(fn_ty);
1702
1703        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1704            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1705
1706        if all_args_match {
1707            return Cow::Borrowed(args);
1708        }
1709
1710        let casted_args: Vec<_> = iter::zip(param_tys, args)
1711            .enumerate()
1712            .map(|(i, (expected_ty, &actual_val))| {
1713                let actual_ty = self.cx.val_ty(actual_val);
1714                if expected_ty != actual_ty {
1715                    debug!(
1716                        "type mismatch in function call of {:?}. \
1717                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1718                        llfn, expected_ty, i, actual_ty
1719                    );
1720                    self.bitcast(actual_val, expected_ty)
1721                } else {
1722                    actual_val
1723                }
1724            })
1725            .collect();
1726
1727        Cow::Owned(casted_args)
1728    }
1729
1730    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1731        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1732    }
1733}
1734
1735impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1736    pub(crate) fn call_intrinsic(
1737        &mut self,
1738        base_name: impl Into<Cow<'static, str>>,
1739        type_params: &[&'ll Type],
1740        args: &[&'ll Value],
1741    ) -> &'ll Value {
1742        let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1743        self.call(ty, None, None, f, args, None, None)
1744    }
1745
1746    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1747        let size = size.bytes();
1748        if size == 0 {
1749            return;
1750        }
1751
1752        if !self.cx().sess().emit_lifetime_markers() {
1753            return;
1754        }
1755
1756        if crate::llvm_util::get_version() >= (22, 0, 0) {
1757            self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]);
1758        } else {
1759            self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
1760        }
1761    }
1762}
1763impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1764    pub(crate) fn phi(
1765        &mut self,
1766        ty: &'ll Type,
1767        vals: &[&'ll Value],
1768        bbs: &[&'ll BasicBlock],
1769    ) -> &'ll Value {
1770        assert_eq!(vals.len(), bbs.len());
1771        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1772        unsafe {
1773            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1774            phi
1775        }
1776    }
1777
1778    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1779        unsafe {
1780            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1781        }
1782    }
1783}
1784impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1785    pub(crate) fn landing_pad(
1786        &mut self,
1787        ty: &'ll Type,
1788        pers_fn: &'ll Value,
1789        num_clauses: usize,
1790    ) -> &'ll Value {
1791        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1792        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1793        // personality lives on the parent function anyway.
1794        self.set_personality_fn(pers_fn);
1795        unsafe {
1796            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1797        }
1798    }
1799
1800    pub(crate) fn callbr(
1801        &mut self,
1802        llty: &'ll Type,
1803        fn_attrs: Option<&CodegenFnAttrs>,
1804        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1805        llfn: &'ll Value,
1806        args: &[&'ll Value],
1807        default_dest: &'ll BasicBlock,
1808        indirect_dest: &[&'ll BasicBlock],
1809        funclet: Option<&Funclet<'ll>>,
1810        instance: Option<Instance<'tcx>>,
1811    ) -> &'ll Value {
1812        debug!("invoke {:?} with args ({:?})", llfn, args);
1813
1814        let args = self.check_call("callbr", llty, llfn, args);
1815        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1816        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1817        if let Some(funclet_bundle) = funclet_bundle {
1818            bundles.push(funclet_bundle);
1819        }
1820
1821        // Emit CFI pointer type membership test
1822        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1823
1824        // Emit KCFI operand bundle
1825        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1826        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1827            bundles.push(kcfi_bundle);
1828        }
1829
1830        let callbr = unsafe {
1831            llvm::LLVMBuildCallBr(
1832                self.llbuilder,
1833                llty,
1834                llfn,
1835                default_dest,
1836                indirect_dest.as_ptr(),
1837                indirect_dest.len() as c_uint,
1838                args.as_ptr(),
1839                args.len() as c_uint,
1840                bundles.as_ptr(),
1841                bundles.len() as c_uint,
1842                UNNAMED,
1843            )
1844        };
1845        if let Some(fn_abi) = fn_abi {
1846            fn_abi.apply_attrs_callsite(self, callbr);
1847        }
1848        callbr
1849    }
1850
1851    // Emits CFI pointer type membership tests.
1852    fn cfi_type_test(
1853        &mut self,
1854        fn_attrs: Option<&CodegenFnAttrs>,
1855        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1856        instance: Option<Instance<'tcx>>,
1857        llfn: &'ll Value,
1858    ) {
1859        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1860        if self.tcx.sess.is_sanitizer_cfi_enabled()
1861            && let Some(fn_abi) = fn_abi
1862            && is_indirect_call
1863        {
1864            if let Some(fn_attrs) = fn_attrs
1865                && fn_attrs.sanitizers.disabled.contains(SanitizerSet::CFI)
1866            {
1867                return;
1868            }
1869
1870            let mut options = cfi::TypeIdOptions::empty();
1871            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1872                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1873            }
1874            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1875                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1876            }
1877
1878            let typeid = if let Some(instance) = instance {
1879                cfi::typeid_for_instance(self.tcx, instance, options)
1880            } else {
1881                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1882            };
1883            let typeid_metadata = self.cx.create_metadata(typeid.as_bytes());
1884            let dbg_loc = self.get_dbg_loc();
1885
1886            // Test whether the function pointer is associated with the type identifier using the
1887            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
1888            // calls to this intrinsic with code to test type membership.
1889            let typeid = self.get_metadata_value(typeid_metadata);
1890            let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
1891            let bb_pass = self.append_sibling_block("type_test.pass");
1892            let bb_fail = self.append_sibling_block("type_test.fail");
1893            self.cond_br(cond, bb_pass, bb_fail);
1894
1895            self.switch_to_block(bb_fail);
1896            if let Some(dbg_loc) = dbg_loc {
1897                self.set_dbg_loc(dbg_loc);
1898            }
1899            self.abort();
1900            self.unreachable();
1901
1902            self.switch_to_block(bb_pass);
1903            if let Some(dbg_loc) = dbg_loc {
1904                self.set_dbg_loc(dbg_loc);
1905            }
1906        }
1907    }
1908
1909    // Emits KCFI operand bundles.
1910    fn kcfi_operand_bundle(
1911        &mut self,
1912        fn_attrs: Option<&CodegenFnAttrs>,
1913        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1914        instance: Option<Instance<'tcx>>,
1915        llfn: &'ll Value,
1916    ) -> Option<llvm::OperandBundleBox<'ll>> {
1917        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1918        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1919            && let Some(fn_abi) = fn_abi
1920            && is_indirect_call
1921        {
1922            if let Some(fn_attrs) = fn_attrs
1923                && fn_attrs.sanitizers.disabled.contains(SanitizerSet::KCFI)
1924            {
1925                return None;
1926            }
1927
1928            let mut options = kcfi::TypeIdOptions::empty();
1929            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1930                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1931            }
1932            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1933                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1934            }
1935
1936            let kcfi_typeid = if let Some(instance) = instance {
1937                kcfi::typeid_for_instance(self.tcx, instance, options)
1938            } else {
1939                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1940            };
1941
1942            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1943        } else {
1944            None
1945        };
1946        kcfi_bundle
1947    }
1948
1949    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1950    #[instrument(level = "debug", skip(self))]
1951    pub(crate) fn instrprof_increment(
1952        &mut self,
1953        fn_name: &'ll Value,
1954        hash: &'ll Value,
1955        num_counters: &'ll Value,
1956        index: &'ll Value,
1957    ) {
1958        self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
1959    }
1960}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
