rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5pub(crate) mod autodiff;
6
7use libc::{c_char, c_uint, size_t};
8use rustc_abi as abi;
9use rustc_abi::{Align, Size, WrappingRange};
10use rustc_codegen_ssa::MemFlags;
11use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
12use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
13use rustc_codegen_ssa::mir::place::PlaceRef;
14use rustc_codegen_ssa::traits::*;
15use rustc_data_structures::small_c_str::SmallCStr;
16use rustc_hir::def_id::DefId;
17use rustc_middle::bug;
18use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
19use rustc_middle::ty::layout::{
20    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
21    TyAndLayout,
22};
23use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
24use rustc_sanitizers::{cfi, kcfi};
25use rustc_session::config::OptLevel;
26use rustc_span::Span;
27use rustc_target::callconv::FnAbi;
28use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
29use smallvec::SmallVec;
30use tracing::{debug, instrument};
31
32use crate::abi::FnAbiLlvmExt;
33use crate::attributes;
34use crate::common::Funclet;
35use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
36use crate::llvm::{
37    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, False, GEPNoWrapFlags, Metadata, True,
38};
39use crate::type_::Type;
40use crate::type_of::LayoutLlvmExt;
41use crate::value::Value;
42
43#[must_use]
44pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
45    pub llbuilder: &'ll mut llvm::Builder<'ll>,
46    pub cx: &'a GenericCx<'ll, CX>,
47}
48
49pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
50pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
51
52impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
53    fn drop(&mut self) {
54        unsafe {
55            llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
56        }
57    }
58}
59
60impl<'a, 'll> SBuilder<'a, 'll> {
61    pub(crate) fn call(
62        &mut self,
63        llty: &'ll Type,
64        llfn: &'ll Value,
65        args: &[&'ll Value],
66        funclet: Option<&Funclet<'ll>>,
67    ) -> &'ll Value {
68        debug!("call {:?} with args ({:?})", llfn, args);
69
70        let args = self.check_call("call", llty, llfn, args);
71        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
72        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
73        if let Some(funclet_bundle) = funclet_bundle {
74            bundles.push(funclet_bundle);
75        }
76
77        let call = unsafe {
78            llvm::LLVMBuildCallWithOperandBundles(
79                self.llbuilder,
80                llty,
81                llfn,
82                args.as_ptr() as *const &llvm::Value,
83                args.len() as c_uint,
84                bundles.as_ptr(),
85                bundles.len() as c_uint,
86                c"".as_ptr(),
87            )
88        };
89        call
90    }
91}
92
93impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
94    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
95        // Create a fresh builder from the simple context.
96        let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(scx.deref().borrow().llcx) };
97        GenericBuilder { llbuilder, cx: scx }
98    }
99
100    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
101        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
102    }
103
104    pub(crate) fn ret_void(&mut self) {
105        llvm::LLVMBuildRetVoid(self.llbuilder);
106    }
107
108    pub(crate) fn ret(&mut self, v: &'ll Value) {
109        unsafe {
110            llvm::LLVMBuildRet(self.llbuilder, v);
111        }
112    }
113
114    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
115        let bx = Self::with_cx(cx);
116        unsafe {
117            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
118        }
119        bx
120    }
121}
122
123/// Empty string, to be used where LLVM expects an instruction name, indicating
124/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
125// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
126pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
127
128impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
129    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
130    type Metadata = <GenericCx<'ll, CX> as BackendTypes>::Metadata;
131    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
132    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
133    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
134    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
135
136    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
137    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
138    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
139}
140
141impl abi::HasDataLayout for Builder<'_, '_, '_> {
142    fn data_layout(&self) -> &abi::TargetDataLayout {
143        self.cx.data_layout()
144    }
145}
146
147impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
148    #[inline]
149    fn tcx(&self) -> TyCtxt<'tcx> {
150        self.cx.tcx
151    }
152}
153
154impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
155    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
156        self.cx.typing_env()
157    }
158}
159
160impl HasTargetSpec for Builder<'_, '_, '_> {
161    #[inline]
162    fn target_spec(&self) -> &Target {
163        self.cx.target_spec()
164    }
165}
166
167impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
168    #[inline]
169    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
170        self.cx.handle_layout_err(err, span, ty)
171    }
172}
173
174impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
175    #[inline]
176    fn handle_fn_abi_err(
177        &self,
178        err: FnAbiError<'tcx>,
179        span: Span,
180        fn_abi_request: FnAbiRequest<'tcx>,
181    ) -> ! {
182        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
183    }
184}
185
186impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
187    type Target = CodegenCx<'ll, 'tcx>;
188
189    #[inline]
190    fn deref(&self) -> &Self::Target {
191        self.cx
192    }
193}
194
195macro_rules! math_builder_methods {
196    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
197        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
198            unsafe {
199                llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
200            }
201        })+
202    }
203}
204
205macro_rules! set_math_builder_methods {
206    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
207        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
208            unsafe {
209                let instr = llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED);
210                llvm::$llvm_set_math(instr);
211                instr
212            }
213        })+
214    }
215}
216
217impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
218    type CodegenCx = CodegenCx<'ll, 'tcx>;
219
220    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
221        let bx = Builder::with_cx(cx);
222        unsafe {
223            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
224        }
225        bx
226    }
227
228    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
229        self.cx
230    }
231
232    fn llbb(&self) -> &'ll BasicBlock {
233        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
234    }
235
236    fn set_span(&mut self, _span: Span) {}
237
238    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
239        unsafe {
240            let name = SmallCStr::new(name);
241            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
242        }
243    }
244
245    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
246        Self::append_block(self.cx, self.llfn(), name)
247    }
248
249    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
250        *self = Self::build(self.cx, llbb)
251    }
252
253    fn ret_void(&mut self) {
254        llvm::LLVMBuildRetVoid(self.llbuilder);
255    }
256
257    fn ret(&mut self, v: &'ll Value) {
258        unsafe {
259            llvm::LLVMBuildRet(self.llbuilder, v);
260        }
261    }
262
263    fn br(&mut self, dest: &'ll BasicBlock) {
264        unsafe {
265            llvm::LLVMBuildBr(self.llbuilder, dest);
266        }
267    }
268
269    fn cond_br(
270        &mut self,
271        cond: &'ll Value,
272        then_llbb: &'ll BasicBlock,
273        else_llbb: &'ll BasicBlock,
274    ) {
275        unsafe {
276            llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
277        }
278    }
279
280    fn switch(
281        &mut self,
282        v: &'ll Value,
283        else_llbb: &'ll BasicBlock,
284        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
285    ) {
286        let switch =
287            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
288        for (on_val, dest) in cases {
289            let on_val = self.const_uint_big(self.val_ty(v), on_val);
290            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
291        }
292    }
293
294    fn switch_with_weights(
295        &mut self,
296        v: Self::Value,
297        else_llbb: Self::BasicBlock,
298        else_is_cold: bool,
299        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
300    ) {
301        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
302            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
303            return;
304        }
305
306        let id_str = "branch_weights";
307        let id = unsafe {
308            llvm::LLVMMDStringInContext2(self.cx.llcx, id_str.as_ptr().cast(), id_str.len())
309        };
310
311        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
312        // This function handles switch instructions with more than 2 targets and it needs to
313        // emit branch weights metadata instead of using the intrinsic.
314        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
315        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
316        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
317        let weight =
318            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
319
320        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
321        md.push(id);
322        md.push(weight(else_is_cold));
323
324        let switch =
325            unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
326        for (on_val, dest, is_cold) in cases {
327            let on_val = self.const_uint_big(self.val_ty(v), on_val);
328            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
329            md.push(weight(is_cold));
330        }
331
332        unsafe {
333            let md_node = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len() as size_t);
334            self.cx.set_metadata(switch, llvm::MD_prof, md_node);
335        }
336    }
337
338    fn invoke(
339        &mut self,
340        llty: &'ll Type,
341        fn_attrs: Option<&CodegenFnAttrs>,
342        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
343        llfn: &'ll Value,
344        args: &[&'ll Value],
345        then: &'ll BasicBlock,
346        catch: &'ll BasicBlock,
347        funclet: Option<&Funclet<'ll>>,
348        instance: Option<Instance<'tcx>>,
349    ) -> &'ll Value {
350        debug!("invoke {:?} with args ({:?})", llfn, args);
351
352        let args = self.check_call("invoke", llty, llfn, args);
353        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
354        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
355        if let Some(funclet_bundle) = funclet_bundle {
356            bundles.push(funclet_bundle);
357        }
358
359        // Emit CFI pointer type membership test
360        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
361
362        // Emit KCFI operand bundle
363        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
364        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
365            bundles.push(kcfi_bundle);
366        }
367
368        let invoke = unsafe {
369            llvm::LLVMBuildInvokeWithOperandBundles(
370                self.llbuilder,
371                llty,
372                llfn,
373                args.as_ptr(),
374                args.len() as c_uint,
375                then,
376                catch,
377                bundles.as_ptr(),
378                bundles.len() as c_uint,
379                UNNAMED,
380            )
381        };
382        if let Some(fn_abi) = fn_abi {
383            fn_abi.apply_attrs_callsite(self, invoke);
384        }
385        invoke
386    }
387
388    fn unreachable(&mut self) {
389        unsafe {
390            llvm::LLVMBuildUnreachable(self.llbuilder);
391        }
392    }
393
394    math_builder_methods! {
395        add(a, b) => LLVMBuildAdd,
396        fadd(a, b) => LLVMBuildFAdd,
397        sub(a, b) => LLVMBuildSub,
398        fsub(a, b) => LLVMBuildFSub,
399        mul(a, b) => LLVMBuildMul,
400        fmul(a, b) => LLVMBuildFMul,
401        udiv(a, b) => LLVMBuildUDiv,
402        exactudiv(a, b) => LLVMBuildExactUDiv,
403        sdiv(a, b) => LLVMBuildSDiv,
404        exactsdiv(a, b) => LLVMBuildExactSDiv,
405        fdiv(a, b) => LLVMBuildFDiv,
406        urem(a, b) => LLVMBuildURem,
407        srem(a, b) => LLVMBuildSRem,
408        frem(a, b) => LLVMBuildFRem,
409        shl(a, b) => LLVMBuildShl,
410        lshr(a, b) => LLVMBuildLShr,
411        ashr(a, b) => LLVMBuildAShr,
412        and(a, b) => LLVMBuildAnd,
413        or(a, b) => LLVMBuildOr,
414        xor(a, b) => LLVMBuildXor,
415        neg(x) => LLVMBuildNeg,
416        fneg(x) => LLVMBuildFNeg,
417        not(x) => LLVMBuildNot,
418        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
419        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
420        unchecked_ssub(x, y) => LLVMBuildNSWSub,
421        unchecked_usub(x, y) => LLVMBuildNUWSub,
422        unchecked_smul(x, y) => LLVMBuildNSWMul,
423        unchecked_umul(x, y) => LLVMBuildNUWMul,
424    }
425
426    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
427        unsafe {
428            let add = llvm::LLVMBuildAdd(self.llbuilder, a, b, UNNAMED);
429            if llvm::LLVMIsAInstruction(add).is_some() {
430                llvm::LLVMSetNUW(add, True);
431                llvm::LLVMSetNSW(add, True);
432            }
433            add
434        }
435    }
436    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
437        unsafe {
438            let sub = llvm::LLVMBuildSub(self.llbuilder, a, b, UNNAMED);
439            if llvm::LLVMIsAInstruction(sub).is_some() {
440                llvm::LLVMSetNUW(sub, True);
441                llvm::LLVMSetNSW(sub, True);
442            }
443            sub
444        }
445    }
446    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
447        unsafe {
448            let mul = llvm::LLVMBuildMul(self.llbuilder, a, b, UNNAMED);
449            if llvm::LLVMIsAInstruction(mul).is_some() {
450                llvm::LLVMSetNUW(mul, True);
451                llvm::LLVMSetNSW(mul, True);
452            }
453            mul
454        }
455    }
456
457    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
458        unsafe {
459            let or = llvm::LLVMBuildOr(self.llbuilder, a, b, UNNAMED);
460
461            // If a and b are both values, then `or` is a value, rather than
462            // an instruction, so we need to check before setting the flag.
463            // (See also `LLVMBuildNUWNeg` which also needs a check.)
464            if llvm::LLVMIsAInstruction(or).is_some() {
465                llvm::LLVMSetIsDisjoint(or, True);
466            }
467            or
468        }
469    }
470
471    set_math_builder_methods! {
472        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
473        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
474        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
475        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
476        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
477        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
478        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
479        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
480        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
481        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
482    }
483
484    fn checked_binop(
485        &mut self,
486        oop: OverflowOp,
487        ty: Ty<'_>,
488        lhs: Self::Value,
489        rhs: Self::Value,
490    ) -> (Self::Value, Self::Value) {
491        use rustc_middle::ty::IntTy::*;
492        use rustc_middle::ty::UintTy::*;
493        use rustc_middle::ty::{Int, Uint};
494
495        let new_kind = match ty.kind() {
496            Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)),
497            Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)),
498            t @ (Uint(_) | Int(_)) => *t,
499            _ => panic!("tried to get overflow intrinsic for op applied to non-int type"),
500        };
501
502        let name = match oop {
503            OverflowOp::Add => match new_kind {
504                Int(I8) => "llvm.sadd.with.overflow.i8",
505                Int(I16) => "llvm.sadd.with.overflow.i16",
506                Int(I32) => "llvm.sadd.with.overflow.i32",
507                Int(I64) => "llvm.sadd.with.overflow.i64",
508                Int(I128) => "llvm.sadd.with.overflow.i128",
509
510                Uint(U8) => "llvm.uadd.with.overflow.i8",
511                Uint(U16) => "llvm.uadd.with.overflow.i16",
512                Uint(U32) => "llvm.uadd.with.overflow.i32",
513                Uint(U64) => "llvm.uadd.with.overflow.i64",
514                Uint(U128) => "llvm.uadd.with.overflow.i128",
515
516                _ => unreachable!(),
517            },
518            OverflowOp::Sub => match new_kind {
519                Int(I8) => "llvm.ssub.with.overflow.i8",
520                Int(I16) => "llvm.ssub.with.overflow.i16",
521                Int(I32) => "llvm.ssub.with.overflow.i32",
522                Int(I64) => "llvm.ssub.with.overflow.i64",
523                Int(I128) => "llvm.ssub.with.overflow.i128",
524
525                Uint(_) => {
526                    // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
527                    // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
528                    // in the backend if profitable.
529                    let sub = self.sub(lhs, rhs);
530                    let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
531                    return (sub, cmp);
532                }
533
534                _ => unreachable!(),
535            },
536            OverflowOp::Mul => match new_kind {
537                Int(I8) => "llvm.smul.with.overflow.i8",
538                Int(I16) => "llvm.smul.with.overflow.i16",
539                Int(I32) => "llvm.smul.with.overflow.i32",
540                Int(I64) => "llvm.smul.with.overflow.i64",
541                Int(I128) => "llvm.smul.with.overflow.i128",
542
543                Uint(U8) => "llvm.umul.with.overflow.i8",
544                Uint(U16) => "llvm.umul.with.overflow.i16",
545                Uint(U32) => "llvm.umul.with.overflow.i32",
546                Uint(U64) => "llvm.umul.with.overflow.i64",
547                Uint(U128) => "llvm.umul.with.overflow.i128",
548
549                _ => unreachable!(),
550            },
551        };
552
553        let res = self.call_intrinsic(name, &[lhs, rhs]);
554        (self.extract_value(res, 0), self.extract_value(res, 1))
555    }
556
557    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
558        if self.cx().val_ty(val) == self.cx().type_i1() {
559            self.zext(val, self.cx().type_i8())
560        } else {
561            val
562        }
563    }
564
565    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
566        if scalar.is_bool() {
567            return self.unchecked_utrunc(val, self.cx().type_i1());
568        }
569        val
570    }
571
572    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
573        let mut bx = Builder::with_cx(self.cx);
574        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
575        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
576        unsafe {
577            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED);
578            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
579            // Cast to default addrspace if necessary
580            llvm::LLVMBuildPointerCast(bx.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
581        }
582    }
583
584    fn dynamic_alloca(&mut self, size: &'ll Value, align: Align) -> &'ll Value {
585        unsafe {
586            let alloca =
587                llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), size, UNNAMED);
588            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
589            // Cast to default addrspace if necessary
590            llvm::LLVMBuildPointerCast(self.llbuilder, alloca, self.cx().type_ptr(), UNNAMED)
591        }
592    }
593
594    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
595        unsafe {
596            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
597            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
598            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
599            load
600        }
601    }
602
603    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
604        unsafe {
605            let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
606            llvm::LLVMSetVolatile(load, llvm::True);
607            load
608        }
609    }
610
611    fn atomic_load(
612        &mut self,
613        ty: &'ll Type,
614        ptr: &'ll Value,
615        order: rustc_codegen_ssa::common::AtomicOrdering,
616        size: Size,
617    ) -> &'ll Value {
618        unsafe {
619            let load = llvm::LLVMRustBuildAtomicLoad(
620                self.llbuilder,
621                ty,
622                ptr,
623                UNNAMED,
624                AtomicOrdering::from_generic(order),
625            );
626            // LLVM requires the alignment of atomic loads to be at least the size of the type.
627            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
628            load
629        }
630    }
631
632    #[instrument(level = "trace", skip(self))]
633    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
634        if place.layout.is_unsized() {
635            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
636            if matches!(tail.kind(), ty::Foreign(..)) {
637                // Unsized locals and, at least conceptually, even unsized arguments must be copied
638                // around, which requires dynamically determining their size. Therefore, we cannot
639                // allow `extern` types here. Consult t-opsem before removing this check.
640                panic!("unsized locals must not be `extern` types");
641            }
642        }
643        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
644
645        if place.layout.is_zst() {
646            return OperandRef::zero_sized(place.layout);
647        }
648
649        #[instrument(level = "trace", skip(bx))]
650        fn scalar_load_metadata<'a, 'll, 'tcx>(
651            bx: &mut Builder<'a, 'll, 'tcx>,
652            load: &'ll Value,
653            scalar: abi::Scalar,
654            layout: TyAndLayout<'tcx>,
655            offset: Size,
656        ) {
657            if bx.cx.sess().opts.optimize == OptLevel::No {
658                // Don't emit metadata we're not going to use
659                return;
660            }
661
662            if !scalar.is_uninit_valid() {
663                bx.noundef_metadata(load);
664            }
665
666            match scalar.primitive() {
667                abi::Primitive::Int(..) => {
668                    if !scalar.is_always_valid(bx) {
669                        bx.range_metadata(load, scalar.valid_range(bx));
670                    }
671                }
672                abi::Primitive::Pointer(_) => {
673                    if !scalar.valid_range(bx).contains(0) {
674                        bx.nonnull_metadata(load);
675                    }
676
677                    if let Some(pointee) = layout.pointee_info_at(bx, offset) {
678                        if let Some(_) = pointee.safe {
679                            bx.align_metadata(load, pointee.align);
680                        }
681                    }
682                }
683                abi::Primitive::Float(_) => {}
684            }
685        }
686
687        let val = if let Some(_) = place.val.llextra {
688            // FIXME: Merge with the `else` below?
689            OperandValue::Ref(place.val)
690        } else if place.layout.is_llvm_immediate() {
691            let mut const_llval = None;
692            let llty = place.layout.llvm_type(self);
693            unsafe {
694                if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
695                    if llvm::LLVMIsGlobalConstant(global) == llvm::True {
696                        if let Some(init) = llvm::LLVMGetInitializer(global) {
697                            if self.val_ty(init) == llty {
698                                const_llval = Some(init);
699                            }
700                        }
701                    }
702                }
703            }
704            let llval = const_llval.unwrap_or_else(|| {
705                let load = self.load(llty, place.val.llval, place.val.align);
706                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
707                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
708                    self.to_immediate_scalar(load, scalar)
709                } else {
710                    load
711                }
712            });
713            OperandValue::Immediate(llval)
714        } else if let abi::BackendRepr::ScalarPair(a, b) = place.layout.backend_repr {
715            let b_offset = a.size(self).align_to(b.align(self).abi);
716
717            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
718                let llptr = if i == 0 {
719                    place.val.llval
720                } else {
721                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
722                };
723                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
724                let load = self.load(llty, llptr, align);
725                scalar_load_metadata(self, load, scalar, layout, offset);
726                self.to_immediate_scalar(load, scalar)
727            };
728
729            OperandValue::Pair(
730                load(0, a, place.layout, place.val.align, Size::ZERO),
731                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
732            )
733        } else {
734            OperandValue::Ref(place.val)
735        };
736
737        OperandRef { val, layout: place.layout }
738    }
739
740    fn write_operand_repeatedly(
741        &mut self,
742        cg_elem: OperandRef<'tcx, &'ll Value>,
743        count: u64,
744        dest: PlaceRef<'tcx, &'ll Value>,
745    ) {
746        let zero = self.const_usize(0);
747        let count = self.const_usize(count);
748
749        let header_bb = self.append_sibling_block("repeat_loop_header");
750        let body_bb = self.append_sibling_block("repeat_loop_body");
751        let next_bb = self.append_sibling_block("repeat_loop_next");
752
753        self.br(header_bb);
754
755        let mut header_bx = Self::build(self.cx, header_bb);
756        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
757
758        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
759        header_bx.cond_br(keep_going, body_bb, next_bb);
760
761        let mut body_bx = Self::build(self.cx, body_bb);
762        let dest_elem = dest.project_index(&mut body_bx, i);
763        cg_elem.val.store(&mut body_bx, dest_elem);
764
765        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
766        body_bx.br(header_bb);
767        header_bx.add_incoming_to_phi(i, next, body_bb);
768
769        *self = Self::build(self.cx, next_bb);
770    }
771
772    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
773        if self.cx.sess().opts.optimize == OptLevel::No {
774            // Don't emit metadata we're not going to use
775            return;
776        }
777
778        unsafe {
779            let llty = self.cx.val_ty(load);
780            let md = [
781                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
782                llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
783            ];
784            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
785            self.set_metadata(load, llvm::MD_range, md);
786        }
787    }
788
789    fn nonnull_metadata(&mut self, load: &'ll Value) {
790        unsafe {
791            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
792            self.set_metadata(load, llvm::MD_nonnull, md);
793        }
794    }
795
796    fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
797        self.store_with_flags(val, ptr, align, MemFlags::empty())
798    }
799
800    fn store_with_flags(
801        &mut self,
802        val: &'ll Value,
803        ptr: &'ll Value,
804        align: Align,
805        flags: MemFlags,
806    ) -> &'ll Value {
807        debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
808        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
809        unsafe {
810            let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
811            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
812            let align =
813                if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
814            llvm::LLVMSetAlignment(store, align);
815            if flags.contains(MemFlags::VOLATILE) {
816                llvm::LLVMSetVolatile(store, llvm::True);
817            }
818            if flags.contains(MemFlags::NONTEMPORAL) {
819                // Make sure that the current target architectures supports "sane" non-temporal
820                // stores, i.e., non-temporal stores that are equivalent to regular stores except
821                // for performance. LLVM doesn't seem to care about this, and will happily treat
822                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
823                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
824                // regular stores but require special fences. So we keep a list of architectures
825                // where `!nontemporal` is known to be truly just a hint, and use regular stores
826                // everywhere else. (In the future, we could alternatively ensure that an sfence
827                // gets emitted after a sequence of movnt before any kind of synchronizing
828                // operation. But it's not clear how to do that with LLVM.)
829                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
830                // <https://github.com/llvm/llvm-project/issues/64521>.
831                const WELL_BEHAVED_NONTEMPORAL_ARCHS: &[&str] =
832                    &["aarch64", "arm", "riscv32", "riscv64"];
833
834                let use_nontemporal =
835                    WELL_BEHAVED_NONTEMPORAL_ARCHS.contains(&&*self.cx.tcx.sess.target.arch);
836                if use_nontemporal {
837                    // According to LLVM [1] building a nontemporal store must
838                    // *always* point to a metadata value of the integer 1.
839                    //
840                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
841                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
842                    let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, &one, 1);
843                    self.set_metadata(store, llvm::MD_nontemporal, md);
844                }
845            }
846            store
847        }
848    }
849
850    fn atomic_store(
851        &mut self,
852        val: &'ll Value,
853        ptr: &'ll Value,
854        order: rustc_codegen_ssa::common::AtomicOrdering,
855        size: Size,
856    ) {
857        debug!("Store {:?} -> {:?}", val, ptr);
858        assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
859        unsafe {
860            let store = llvm::LLVMRustBuildAtomicStore(
861                self.llbuilder,
862                val,
863                ptr,
864                AtomicOrdering::from_generic(order),
865            );
866            // LLVM requires the alignment of atomic stores to be at least the size of the type.
867            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
868        }
869    }
870
871    fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
872        unsafe {
873            llvm::LLVMBuildGEPWithNoWrapFlags(
874                self.llbuilder,
875                ty,
876                ptr,
877                indices.as_ptr(),
878                indices.len() as c_uint,
879                UNNAMED,
880                GEPNoWrapFlags::default(),
881            )
882        }
883    }
884
885    fn inbounds_gep(
886        &mut self,
887        ty: &'ll Type,
888        ptr: &'ll Value,
889        indices: &[&'ll Value],
890    ) -> &'ll Value {
891        unsafe {
892            llvm::LLVMBuildGEPWithNoWrapFlags(
893                self.llbuilder,
894                ty,
895                ptr,
896                indices.as_ptr(),
897                indices.len() as c_uint,
898                UNNAMED,
899                GEPNoWrapFlags::InBounds,
900            )
901        }
902    }
903
904    fn inbounds_nuw_gep(
905        &mut self,
906        ty: &'ll Type,
907        ptr: &'ll Value,
908        indices: &[&'ll Value],
909    ) -> &'ll Value {
910        unsafe {
911            llvm::LLVMBuildGEPWithNoWrapFlags(
912                self.llbuilder,
913                ty,
914                ptr,
915                indices.as_ptr(),
916                indices.len() as c_uint,
917                UNNAMED,
918                GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW,
919            )
920        }
921    }
922
923    /* Casts */
924    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
925        unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
926    }
927
928    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
929        debug_assert_ne!(self.val_ty(val), dest_ty);
930
931        let trunc = self.trunc(val, dest_ty);
932        unsafe {
933            if llvm::LLVMIsAInstruction(trunc).is_some() {
934                llvm::LLVMSetNUW(trunc, True);
935            }
936        }
937        trunc
938    }
939
940    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
941        debug_assert_ne!(self.val_ty(val), dest_ty);
942
943        let trunc = self.trunc(val, dest_ty);
944        unsafe {
945            if llvm::LLVMIsAInstruction(trunc).is_some() {
946                llvm::LLVMSetNSW(trunc, True);
947            }
948        }
949        trunc
950    }
951
952    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
953        unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
954    }
955
956    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
957        self.fptoint_sat(false, val, dest_ty)
958    }
959
960    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
961        self.fptoint_sat(true, val, dest_ty)
962    }
963
964    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
965        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
966        // poor codegen. The reason for this is that the corresponding wasm
967        // instructions, `i32.trunc_f32_s` for example, will trap when the float
968        // is out-of-bounds, infinity, or nan. This means that LLVM
969        // automatically inserts control flow around `fptoui` and `fptosi`
970        // because the LLVM instruction `fptoui` is defined as producing a
971        // poison value, not having UB on out-of-bounds values.
972        //
973        // This method, however, is only used with non-saturating casts that
974        // have UB on out-of-bounds values. This means that it's ok if we use
975        // the raw wasm instruction since out-of-bounds values can do whatever
976        // we like. To ensure that LLVM picks the right instruction we choose
977        // the raw wasm intrinsic functions which avoid LLVM inserting all the
978        // other control flow automatically.
979        if self.sess().target.is_like_wasm {
980            let src_ty = self.cx.val_ty(val);
981            if self.cx.type_kind(src_ty) != TypeKind::Vector {
982                let float_width = self.cx.float_width(src_ty);
983                let int_width = self.cx.int_width(dest_ty);
984                let name = match (int_width, float_width) {
985                    (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"),
986                    (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"),
987                    (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"),
988                    (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"),
989                    _ => None,
990                };
991                if let Some(name) = name {
992                    return self.call_intrinsic(name, &[val]);
993                }
994            }
995        }
996        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
997    }
998
999    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1000        // see `fptoui` above for why wasm is different here
1001        if self.sess().target.is_like_wasm {
1002            let src_ty = self.cx.val_ty(val);
1003            if self.cx.type_kind(src_ty) != TypeKind::Vector {
1004                let float_width = self.cx.float_width(src_ty);
1005                let int_width = self.cx.int_width(dest_ty);
1006                let name = match (int_width, float_width) {
1007                    (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"),
1008                    (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"),
1009                    (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"),
1010                    (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"),
1011                    _ => None,
1012                };
1013                if let Some(name) = name {
1014                    return self.call_intrinsic(name, &[val]);
1015                }
1016            }
1017        }
1018        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
1019    }
1020
1021    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1022        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1023    }
1024
1025    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1026        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
1027    }
1028
1029    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1030        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
1031    }
1032
1033    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1034        unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
1035    }
1036
1037    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1038        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
1039    }
1040
1041    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1042        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
1043    }
1044
1045    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1046        unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
1047    }
1048
1049    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1050        unsafe {
1051            llvm::LLVMBuildIntCast2(
1052                self.llbuilder,
1053                val,
1054                dest_ty,
1055                if is_signed { True } else { False },
1056                UNNAMED,
1057            )
1058        }
1059    }
1060
1061    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1062        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
1063    }
1064
1065    /* Comparisons */
1066    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1067        let op = llvm::IntPredicate::from_generic(op);
1068        unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1069    }
1070
1071    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1072        let op = llvm::RealPredicate::from_generic(op);
1073        unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
1074    }
1075
1076    fn three_way_compare(
1077        &mut self,
1078        ty: Ty<'tcx>,
1079        lhs: Self::Value,
1080        rhs: Self::Value,
1081    ) -> Option<Self::Value> {
1082        // FIXME: See comment on the definition of `three_way_compare`.
1083        if crate::llvm_util::get_version() < (20, 0, 0) {
1084            return None;
1085        }
1086
1087        let name = match (ty.is_signed(), ty.primitive_size(self.tcx).bits()) {
1088            (true, 8) => "llvm.scmp.i8.i8",
1089            (true, 16) => "llvm.scmp.i8.i16",
1090            (true, 32) => "llvm.scmp.i8.i32",
1091            (true, 64) => "llvm.scmp.i8.i64",
1092            (true, 128) => "llvm.scmp.i8.i128",
1093
1094            (false, 8) => "llvm.ucmp.i8.i8",
1095            (false, 16) => "llvm.ucmp.i8.i16",
1096            (false, 32) => "llvm.ucmp.i8.i32",
1097            (false, 64) => "llvm.ucmp.i8.i64",
1098            (false, 128) => "llvm.ucmp.i8.i128",
1099
1100            _ => bug!("three-way compare unsupported for type {ty:?}"),
1101        };
1102        Some(self.call_intrinsic(name, &[lhs, rhs]))
1103    }
1104
1105    /* Miscellaneous instructions */
1106    fn memcpy(
1107        &mut self,
1108        dst: &'ll Value,
1109        dst_align: Align,
1110        src: &'ll Value,
1111        src_align: Align,
1112        size: &'ll Value,
1113        flags: MemFlags,
1114    ) {
1115        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1116        let size = self.intcast(size, self.type_isize(), false);
1117        let is_volatile = flags.contains(MemFlags::VOLATILE);
1118        unsafe {
1119            llvm::LLVMRustBuildMemCpy(
1120                self.llbuilder,
1121                dst,
1122                dst_align.bytes() as c_uint,
1123                src,
1124                src_align.bytes() as c_uint,
1125                size,
1126                is_volatile,
1127            );
1128        }
1129    }
1130
1131    fn memmove(
1132        &mut self,
1133        dst: &'ll Value,
1134        dst_align: Align,
1135        src: &'ll Value,
1136        src_align: Align,
1137        size: &'ll Value,
1138        flags: MemFlags,
1139    ) {
1140        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1141        let size = self.intcast(size, self.type_isize(), false);
1142        let is_volatile = flags.contains(MemFlags::VOLATILE);
1143        unsafe {
1144            llvm::LLVMRustBuildMemMove(
1145                self.llbuilder,
1146                dst,
1147                dst_align.bytes() as c_uint,
1148                src,
1149                src_align.bytes() as c_uint,
1150                size,
1151                is_volatile,
1152            );
1153        }
1154    }
1155
1156    fn memset(
1157        &mut self,
1158        ptr: &'ll Value,
1159        fill_byte: &'ll Value,
1160        size: &'ll Value,
1161        align: Align,
1162        flags: MemFlags,
1163    ) {
1164        assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1165        let is_volatile = flags.contains(MemFlags::VOLATILE);
1166        unsafe {
1167            llvm::LLVMRustBuildMemSet(
1168                self.llbuilder,
1169                ptr,
1170                align.bytes() as c_uint,
1171                fill_byte,
1172                size,
1173                is_volatile,
1174            );
1175        }
1176    }
1177
1178    fn select(
1179        &mut self,
1180        cond: &'ll Value,
1181        then_val: &'ll Value,
1182        else_val: &'ll Value,
1183    ) -> &'ll Value {
1184        unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
1185    }
1186
1187    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1188        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1189    }
1190
1191    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1192        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
1193    }
1194
1195    fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
1196        unsafe {
1197            let elt_ty = self.cx.val_ty(elt);
1198            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
1199            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1200            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
1201            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1202        }
1203    }
1204
1205    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1206        assert_eq!(idx as c_uint as u64, idx);
1207        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
1208    }
1209
1210    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1211        assert_eq!(idx as c_uint as u64, idx);
1212        unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
1213    }
1214
1215    fn set_personality_fn(&mut self, personality: &'ll Value) {
1216        unsafe {
1217            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1218        }
1219    }
1220
1221    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1222        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1223        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1224        unsafe {
1225            llvm::LLVMSetCleanup(landing_pad, llvm::True);
1226        }
1227        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1228    }
1229
1230    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1231        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1232        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1233        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1234        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1235    }
1236
1237    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1238        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()], false);
1239        let mut exn = self.const_poison(ty);
1240        exn = self.insert_value(exn, exn0, 0);
1241        exn = self.insert_value(exn, exn1, 1);
1242        unsafe {
1243            llvm::LLVMBuildResume(self.llbuilder, exn);
1244        }
1245    }
1246
1247    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1248        let ret = unsafe {
1249            llvm::LLVMBuildCleanupPad(
1250                self.llbuilder,
1251                parent,
1252                args.as_ptr(),
1253                args.len() as c_uint,
1254                c"cleanuppad".as_ptr(),
1255            )
1256        };
1257        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1258    }
1259
1260    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1261        unsafe {
1262            llvm::LLVMBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind)
1263                .expect("LLVM does not have support for cleanupret");
1264        }
1265    }
1266
1267    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1268        let ret = unsafe {
1269            llvm::LLVMBuildCatchPad(
1270                self.llbuilder,
1271                parent,
1272                args.as_ptr(),
1273                args.len() as c_uint,
1274                c"catchpad".as_ptr(),
1275            )
1276        };
1277        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1278    }
1279
1280    fn catch_switch(
1281        &mut self,
1282        parent: Option<&'ll Value>,
1283        unwind: Option<&'ll BasicBlock>,
1284        handlers: &[&'ll BasicBlock],
1285    ) -> &'ll Value {
1286        let ret = unsafe {
1287            llvm::LLVMBuildCatchSwitch(
1288                self.llbuilder,
1289                parent,
1290                unwind,
1291                handlers.len() as c_uint,
1292                c"catchswitch".as_ptr(),
1293            )
1294        };
1295        let ret = ret.expect("LLVM does not have support for catchswitch");
1296        for handler in handlers {
1297            unsafe {
1298                llvm::LLVMAddHandler(ret, handler);
1299            }
1300        }
1301        ret
1302    }
1303
1304    // Atomic Operations
1305    fn atomic_cmpxchg(
1306        &mut self,
1307        dst: &'ll Value,
1308        cmp: &'ll Value,
1309        src: &'ll Value,
1310        order: rustc_codegen_ssa::common::AtomicOrdering,
1311        failure_order: rustc_codegen_ssa::common::AtomicOrdering,
1312        weak: bool,
1313    ) -> (&'ll Value, &'ll Value) {
1314        let weak = if weak { llvm::True } else { llvm::False };
1315        unsafe {
1316            let value = llvm::LLVMBuildAtomicCmpXchg(
1317                self.llbuilder,
1318                dst,
1319                cmp,
1320                src,
1321                AtomicOrdering::from_generic(order),
1322                AtomicOrdering::from_generic(failure_order),
1323                llvm::False, // SingleThreaded
1324            );
1325            llvm::LLVMSetWeak(value, weak);
1326            let val = self.extract_value(value, 0);
1327            let success = self.extract_value(value, 1);
1328            (val, success)
1329        }
1330    }
1331
1332    fn atomic_rmw(
1333        &mut self,
1334        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1335        dst: &'ll Value,
1336        mut src: &'ll Value,
1337        order: rustc_codegen_ssa::common::AtomicOrdering,
1338    ) -> &'ll Value {
1339        // The only RMW operation that LLVM supports on pointers is compare-exchange.
1340        let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
1341            && op != rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg;
1342        if requires_cast_to_int {
1343            src = self.ptrtoint(src, self.type_isize());
1344        }
1345        let mut res = unsafe {
1346            llvm::LLVMBuildAtomicRMW(
1347                self.llbuilder,
1348                AtomicRmwBinOp::from_generic(op),
1349                dst,
1350                src,
1351                AtomicOrdering::from_generic(order),
1352                llvm::False, // SingleThreaded
1353            )
1354        };
1355        if requires_cast_to_int {
1356            res = self.inttoptr(res, self.type_ptr());
1357        }
1358        res
1359    }
1360
1361    fn atomic_fence(
1362        &mut self,
1363        order: rustc_codegen_ssa::common::AtomicOrdering,
1364        scope: SynchronizationScope,
1365    ) {
1366        let single_threaded = match scope {
1367            SynchronizationScope::SingleThread => llvm::True,
1368            SynchronizationScope::CrossThread => llvm::False,
1369        };
1370        unsafe {
1371            llvm::LLVMBuildFence(
1372                self.llbuilder,
1373                AtomicOrdering::from_generic(order),
1374                single_threaded,
1375                UNNAMED,
1376            );
1377        }
1378    }
1379
1380    fn set_invariant_load(&mut self, load: &'ll Value) {
1381        unsafe {
1382            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1383            self.set_metadata(load, llvm::MD_invariant_load, md);
1384        }
1385    }
1386
1387    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1388        self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size);
1389    }
1390
1391    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1392        self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
1393    }
1394
1395    fn call(
1396        &mut self,
1397        llty: &'ll Type,
1398        fn_attrs: Option<&CodegenFnAttrs>,
1399        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1400        llfn: &'ll Value,
1401        args: &[&'ll Value],
1402        funclet: Option<&Funclet<'ll>>,
1403        instance: Option<Instance<'tcx>>,
1404    ) -> &'ll Value {
1405        debug!("call {:?} with args ({:?})", llfn, args);
1406
1407        let args = self.check_call("call", llty, llfn, args);
1408        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1409        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1410        if let Some(funclet_bundle) = funclet_bundle {
1411            bundles.push(funclet_bundle);
1412        }
1413
1414        // Emit CFI pointer type membership test
1415        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1416
1417        // Emit KCFI operand bundle
1418        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1419        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1420            bundles.push(kcfi_bundle);
1421        }
1422
1423        let call = unsafe {
1424            llvm::LLVMBuildCallWithOperandBundles(
1425                self.llbuilder,
1426                llty,
1427                llfn,
1428                args.as_ptr() as *const &llvm::Value,
1429                args.len() as c_uint,
1430                bundles.as_ptr(),
1431                bundles.len() as c_uint,
1432                c"".as_ptr(),
1433            )
1434        };
1435        if let Some(fn_abi) = fn_abi {
1436            fn_abi.apply_attrs_callsite(self, call);
1437        }
1438        call
1439    }
1440
1441    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1442        unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1443    }
1444
1445    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1446        // Cleanup is always the cold path.
1447        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1448        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1449    }
1450}
1451
1452impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1453    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1454        // Forward to the `get_static` method of `CodegenCx`
1455        let s = self.cx().get_static(def_id);
1456        // Cast to default address space if globals are in a different addrspace
1457        self.cx().const_pointercast(s, self.type_ptr())
1458    }
1459}
1460
1461impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1462    pub(crate) fn llfn(&self) -> &'ll Value {
1463        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1464    }
1465}
1466
1467impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1468    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1469        unsafe {
1470            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1471        }
1472    }
1473}
1474impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1475    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1476        unsafe {
1477            let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1478            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, md.as_ptr(), md.len());
1479            self.set_metadata(load, llvm::MD_align, md);
1480        }
1481    }
1482
1483    fn noundef_metadata(&mut self, load: &'ll Value) {
1484        unsafe {
1485            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1486            self.set_metadata(load, llvm::MD_noundef, md);
1487        }
1488    }
1489
1490    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1491        unsafe {
1492            let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, ptr::null(), 0);
1493            self.set_metadata(inst, llvm::MD_unpredictable, md);
1494        }
1495    }
1496}
1497impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1498    pub(crate) fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1499        unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1500    }
1501
1502    pub(crate) fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1503        unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1504    }
1505
1506    pub(crate) fn insert_element(
1507        &mut self,
1508        vec: &'ll Value,
1509        elt: &'ll Value,
1510        idx: &'ll Value,
1511    ) -> &'ll Value {
1512        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1513    }
1514
1515    pub(crate) fn shuffle_vector(
1516        &mut self,
1517        v1: &'ll Value,
1518        v2: &'ll Value,
1519        mask: &'ll Value,
1520    ) -> &'ll Value {
1521        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1522    }
1523
1524    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1525        unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1526    }
1527    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1528        unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1529    }
1530    pub(crate) fn vector_reduce_fadd_reassoc(
1531        &mut self,
1532        acc: &'ll Value,
1533        src: &'ll Value,
1534    ) -> &'ll Value {
1535        unsafe {
1536            let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1537            llvm::LLVMRustSetAllowReassoc(instr);
1538            instr
1539        }
1540    }
1541    pub(crate) fn vector_reduce_fmul_reassoc(
1542        &mut self,
1543        acc: &'ll Value,
1544        src: &'ll Value,
1545    ) -> &'ll Value {
1546        unsafe {
1547            let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1548            llvm::LLVMRustSetAllowReassoc(instr);
1549            instr
1550        }
1551    }
1552    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1553        unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1554    }
1555    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1556        unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1557    }
1558    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1559        unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1560    }
1561    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1562        unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1563    }
1564    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1565        unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1566    }
1567    pub(crate) fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1568        unsafe {
1569            llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
1570        }
1571    }
1572    pub(crate) fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1573        unsafe {
1574            llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
1575        }
1576    }
1577    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1578        unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1579    }
1580    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1581        unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1582    }
1583
1584    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1585        unsafe {
1586            llvm::LLVMAddClause(landing_pad, clause);
1587        }
1588    }
1589
1590    pub(crate) fn catch_ret(
1591        &mut self,
1592        funclet: &Funclet<'ll>,
1593        unwind: &'ll BasicBlock,
1594    ) -> &'ll Value {
1595        let ret = unsafe { llvm::LLVMBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1596        ret.expect("LLVM does not have support for catchret")
1597    }
1598
1599    fn check_call<'b>(
1600        &mut self,
1601        typ: &str,
1602        fn_ty: &'ll Type,
1603        llfn: &'ll Value,
1604        args: &'b [&'ll Value],
1605    ) -> Cow<'b, [&'ll Value]> {
1606        assert!(
1607            self.cx.type_kind(fn_ty) == TypeKind::Function,
1608            "builder::{typ} not passed a function, but {fn_ty:?}"
1609        );
1610
1611        let param_tys = self.cx.func_params_types(fn_ty);
1612
1613        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1614            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1615
1616        if all_args_match {
1617            return Cow::Borrowed(args);
1618        }
1619
1620        let casted_args: Vec<_> = iter::zip(param_tys, args)
1621            .enumerate()
1622            .map(|(i, (expected_ty, &actual_val))| {
1623                let actual_ty = self.cx.val_ty(actual_val);
1624                if expected_ty != actual_ty {
1625                    debug!(
1626                        "type mismatch in function call of {:?}. \
1627                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1628                        llfn, expected_ty, i, actual_ty
1629                    );
1630                    self.bitcast(actual_val, expected_ty)
1631                } else {
1632                    actual_val
1633                }
1634            })
1635            .collect();
1636
1637        Cow::Owned(casted_args)
1638    }
1639
1640    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1641        unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1642    }
1643}
1644
1645impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1646    pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value {
1647        let (ty, f) = self.cx.get_intrinsic(intrinsic);
1648        self.call(ty, None, None, f, args, None, None)
1649    }
1650
1651    fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
1652        let size = size.bytes();
1653        if size == 0 {
1654            return;
1655        }
1656
1657        if !self.cx().sess().emit_lifetime_markers() {
1658            return;
1659        }
1660
1661        self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]);
1662    }
1663}
1664impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1665    pub(crate) fn phi(
1666        &mut self,
1667        ty: &'ll Type,
1668        vals: &[&'ll Value],
1669        bbs: &[&'ll BasicBlock],
1670    ) -> &'ll Value {
1671        assert_eq!(vals.len(), bbs.len());
1672        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1673        unsafe {
1674            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1675            phi
1676        }
1677    }
1678
1679    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1680        unsafe {
1681            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1682        }
1683    }
1684}
1685impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1686    fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1687        let src_ty = self.cx.val_ty(val);
1688        let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector {
1689            assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty));
1690            (
1691                self.cx.element_type(src_ty),
1692                self.cx.element_type(dest_ty),
1693                Some(self.cx.vector_length(src_ty)),
1694            )
1695        } else {
1696            (src_ty, dest_ty, None)
1697        };
1698        let float_width = self.cx.float_width(float_ty);
1699        let int_width = self.cx.int_width(int_ty);
1700
1701        let instr = if signed { "fptosi" } else { "fptoui" };
1702        let name = if let Some(vector_length) = vector_length {
1703            format!("llvm.{instr}.sat.v{vector_length}i{int_width}.v{vector_length}f{float_width}")
1704        } else {
1705            format!("llvm.{instr}.sat.i{int_width}.f{float_width}")
1706        };
1707        let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty));
1708        self.call(self.type_func(&[src_ty], dest_ty), None, None, f, &[val], None, None)
1709    }
1710
1711    pub(crate) fn landing_pad(
1712        &mut self,
1713        ty: &'ll Type,
1714        pers_fn: &'ll Value,
1715        num_clauses: usize,
1716    ) -> &'ll Value {
1717        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1718        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1719        // personality lives on the parent function anyway.
1720        self.set_personality_fn(pers_fn);
1721        unsafe {
1722            llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
1723        }
1724    }
1725
1726    pub(crate) fn callbr(
1727        &mut self,
1728        llty: &'ll Type,
1729        fn_attrs: Option<&CodegenFnAttrs>,
1730        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1731        llfn: &'ll Value,
1732        args: &[&'ll Value],
1733        default_dest: &'ll BasicBlock,
1734        indirect_dest: &[&'ll BasicBlock],
1735        funclet: Option<&Funclet<'ll>>,
1736        instance: Option<Instance<'tcx>>,
1737    ) -> &'ll Value {
1738        debug!("invoke {:?} with args ({:?})", llfn, args);
1739
1740        let args = self.check_call("callbr", llty, llfn, args);
1741        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1742        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1743        if let Some(funclet_bundle) = funclet_bundle {
1744            bundles.push(funclet_bundle);
1745        }
1746
1747        // Emit CFI pointer type membership test
1748        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1749
1750        // Emit KCFI operand bundle
1751        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1752        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1753            bundles.push(kcfi_bundle);
1754        }
1755
1756        let callbr = unsafe {
1757            llvm::LLVMBuildCallBr(
1758                self.llbuilder,
1759                llty,
1760                llfn,
1761                default_dest,
1762                indirect_dest.as_ptr(),
1763                indirect_dest.len() as c_uint,
1764                args.as_ptr(),
1765                args.len() as c_uint,
1766                bundles.as_ptr(),
1767                bundles.len() as c_uint,
1768                UNNAMED,
1769            )
1770        };
1771        if let Some(fn_abi) = fn_abi {
1772            fn_abi.apply_attrs_callsite(self, callbr);
1773        }
1774        callbr
1775    }
1776
1777    // Emits CFI pointer type membership tests.
1778    fn cfi_type_test(
1779        &mut self,
1780        fn_attrs: Option<&CodegenFnAttrs>,
1781        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1782        instance: Option<Instance<'tcx>>,
1783        llfn: &'ll Value,
1784    ) {
1785        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1786        if self.tcx.sess.is_sanitizer_cfi_enabled()
1787            && let Some(fn_abi) = fn_abi
1788            && is_indirect_call
1789        {
1790            if let Some(fn_attrs) = fn_attrs
1791                && fn_attrs.no_sanitize.contains(SanitizerSet::CFI)
1792            {
1793                return;
1794            }
1795
1796            let mut options = cfi::TypeIdOptions::empty();
1797            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1798                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
1799            }
1800            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1801                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
1802            }
1803
1804            let typeid = if let Some(instance) = instance {
1805                cfi::typeid_for_instance(self.tcx, instance, options)
1806            } else {
1807                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1808            };
1809            let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
1810            let dbg_loc = self.get_dbg_loc();
1811
1812            // Test whether the function pointer is associated with the type identifier.
1813            let cond = self.type_test(llfn, typeid_metadata);
1814            let bb_pass = self.append_sibling_block("type_test.pass");
1815            let bb_fail = self.append_sibling_block("type_test.fail");
1816            self.cond_br(cond, bb_pass, bb_fail);
1817
1818            self.switch_to_block(bb_fail);
1819            if let Some(dbg_loc) = dbg_loc {
1820                self.set_dbg_loc(dbg_loc);
1821            }
1822            self.abort();
1823            self.unreachable();
1824
1825            self.switch_to_block(bb_pass);
1826            if let Some(dbg_loc) = dbg_loc {
1827                self.set_dbg_loc(dbg_loc);
1828            }
1829        }
1830    }
1831
1832    // Emits KCFI operand bundles.
1833    fn kcfi_operand_bundle(
1834        &mut self,
1835        fn_attrs: Option<&CodegenFnAttrs>,
1836        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1837        instance: Option<Instance<'tcx>>,
1838        llfn: &'ll Value,
1839    ) -> Option<llvm::OperandBundleBox<'ll>> {
1840        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
1841        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
1842            && let Some(fn_abi) = fn_abi
1843            && is_indirect_call
1844        {
1845            if let Some(fn_attrs) = fn_attrs
1846                && fn_attrs.no_sanitize.contains(SanitizerSet::KCFI)
1847            {
1848                return None;
1849            }
1850
1851            let mut options = kcfi::TypeIdOptions::empty();
1852            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
1853                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
1854            }
1855            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
1856                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
1857            }
1858
1859            let kcfi_typeid = if let Some(instance) = instance {
1860                kcfi::typeid_for_instance(self.tcx, instance, options)
1861            } else {
1862                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
1863            };
1864
1865            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
1866        } else {
1867            None
1868        };
1869        kcfi_bundle
1870    }
1871
1872    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
1873    #[instrument(level = "debug", skip(self))]
1874    pub(crate) fn instrprof_increment(
1875        &mut self,
1876        fn_name: &'ll Value,
1877        hash: &'ll Value,
1878        num_counters: &'ll Value,
1879        index: &'ll Value,
1880    ) {
1881        self.call_intrinsic("llvm.instrprof.increment", &[fn_name, hash, num_counters, index]);
1882    }
1883
1884    /// Emits a call to `llvm.instrprof.mcdc.parameters`.
1885    ///
1886    /// This doesn't produce any code directly, but is used as input by
1887    /// the LLVM pass that handles coverage instrumentation.
1888    ///
1889    /// (See clang's [`CodeGenPGO::emitMCDCParameters`] for comparison.)
1890    ///
1891    /// [`CodeGenPGO::emitMCDCParameters`]:
1892    ///     https://github.com/rust-lang/llvm-project/blob/5399a24/clang/lib/CodeGen/CodeGenPGO.cpp#L1124
1893    #[instrument(level = "debug", skip(self))]
1894    pub(crate) fn mcdc_parameters(
1895        &mut self,
1896        fn_name: &'ll Value,
1897        hash: &'ll Value,
1898        bitmap_bits: &'ll Value,
1899    ) {
1900        self.call_intrinsic("llvm.instrprof.mcdc.parameters", &[fn_name, hash, bitmap_bits]);
1901    }
1902
1903    #[instrument(level = "debug", skip(self))]
1904    pub(crate) fn mcdc_tvbitmap_update(
1905        &mut self,
1906        fn_name: &'ll Value,
1907        hash: &'ll Value,
1908        bitmap_index: &'ll Value,
1909        mcdc_temp: &'ll Value,
1910    ) {
1911        let args = &[fn_name, hash, bitmap_index, mcdc_temp];
1912        self.call_intrinsic("llvm.instrprof.mcdc.tvbitmap.update", args);
1913    }
1914
1915    #[instrument(level = "debug", skip(self))]
1916    pub(crate) fn mcdc_condbitmap_reset(&mut self, mcdc_temp: &'ll Value) {
1917        self.store(self.const_i32(0), mcdc_temp, self.tcx.data_layout.i32_align.abi);
1918    }
1919
1920    #[instrument(level = "debug", skip(self))]
1921    pub(crate) fn mcdc_condbitmap_update(&mut self, cond_index: &'ll Value, mcdc_temp: &'ll Value) {
1922        let align = self.tcx.data_layout.i32_align.abi;
1923        let current_tv_index = self.load(self.cx.type_i32(), mcdc_temp, align);
1924        let new_tv_index = self.add(current_tv_index, cond_index);
1925        self.store(new_tv_index, mcdc_temp, align);
1926    }
1927}
rustc_codegen_llvm/builder.rs

rustc_codegen_llvm/
builder.rs
