Skip to main content

rustc_codegen_llvm/
builder.rs

1use std::borrow::{Borrow, Cow};
2use std::ops::Deref;
3use std::{iter, ptr};
4
5use rustc_ast::expand::typetree::FncTree;
6pub(crate) mod autodiff;
7pub(crate) mod gpu_offload;
8
9use libc::{c_char, c_uint};
10use rustc_abi::{self as abi, Align, CanonAbi, Float, Size, WrappingRange};
11use rustc_attr_ir::UnrollAttr;
12use rustc_codegen_ssa::MemFlags;
13use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope};
14use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
15use rustc_codegen_ssa::mir::place::PlaceRef;
16use rustc_codegen_ssa::traits::*;
17use rustc_data_structures::small_c_str::SmallCStr;
18use rustc_hir::def_id::DefId;
19use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
20use rustc_middle::ty::layout::{
21    FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTypingEnv, LayoutError, LayoutOfHelpers,
22    TyAndLayout,
23};
24use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
25use rustc_sanitizers::{cfi, kcfi};
26use rustc_session::config::OptLevel;
27use rustc_span::{Span, bug};
28use rustc_target::callconv::{FnAbi, PassMode};
29use rustc_target::spec::{Arch, HasTargetSpec, SanitizerSet, Target};
30use smallvec::SmallVec;
31use tracing::{debug, instrument};
32
33use crate::abi::FnAbiLlvmExt;
34use crate::attributes;
35use crate::common::Funclet;
36use crate::context::{CodegenCx, FullCx, GenericCx, SCx};
37use crate::llvm::{
38    self, AtomicOrdering, AtomicRmwBinOp, BasicBlock, FromGeneric, GEPNoWrapFlags, Metadata, TRUE,
39    ToLlvmBool, Type, TypeKind, Value,
40};
41use crate::type_of::LayoutLlvmExt;
42
43#[must_use]
44pub(crate) struct GenericBuilder<'a, 'll, CX: Borrow<SCx<'ll>>> {
45    /// Owning pointer that corresponds to an `LLVMBuilderRef`. Disposed on drop.
46    llbuilder_raw: ptr::NonNull<llvm::Builder<'ll>>,
47    pub cx: &'a GenericCx<'ll, CX>,
48    pub span: rustc_span::Span,
49}
50
51pub(crate) type SBuilder<'a, 'll> = GenericBuilder<'a, 'll, SCx<'ll>>;
52pub(crate) type Builder<'a, 'll, 'tcx> = GenericBuilder<'a, 'll, FullCx<'ll, 'tcx>>;
53
54impl<'a, 'll, CX: Borrow<SCx<'ll>>> Drop for GenericBuilder<'a, 'll, CX> {
55    fn drop(&mut self) {
56        unsafe { llvm::LLVMDisposeBuilder(self.llbuilder_raw) };
57    }
58}
59
60impl<'a, 'll> SBuilder<'a, 'll> {
61    pub(crate) fn call(
62        &mut self,
63        llty: &'ll Type,
64        llfn: &'ll Value,
65        args: &[&'ll Value],
66        funclet: Option<&Funclet<'ll>>,
67    ) -> &'ll Value {
68        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:68",
                        "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                        ::tracing_core::__macro_support::Option::Some(68u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("call {0:?} with args ({1:?})",
                                                    llfn, args) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("call {:?} with args ({:?})", llfn, args);
69
70        let args = self.check_call("call", llty, llfn, args);
71        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
72        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
73        if let Some(funclet_bundle) = funclet_bundle {
74            bundles.push(funclet_bundle);
75        }
76
77        let call = unsafe {
78            llvm::LLVMBuildCallWithOperandBundles(
79                self.llbuilder(),
80                llty,
81                llfn,
82                args.as_ptr() as *const &llvm::Value,
83                args.len() as c_uint,
84                bundles.as_ptr(),
85                bundles.len() as c_uint,
86                c"".as_ptr(),
87            )
88        };
89        call
90    }
91}
92
93impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
94    fn with_cx(scx: &'a GenericCx<'ll, CX>) -> Self {
95        // Create a fresh builder from the simple context.
96        let llbuilder_raw = unsafe { llvm::LLVMCreateBuilderForModule(scx.deref().borrow().llmod) };
97        GenericBuilder { llbuilder_raw, cx: scx, span: rustc_span::DUMMY_SP }
98    }
99
100    pub(crate) fn append_block(
101        cx: &'a GenericCx<'ll, CX>,
102        llfn: &'ll Value,
103        name: &str,
104    ) -> &'ll BasicBlock {
105        unsafe {
106            let name = SmallCStr::new(name);
107            llvm::LLVMAppendBasicBlockInContext(cx.llcx(), llfn, name.as_ptr())
108        }
109    }
110
111    /// Returns a reference to this builder's underlying [`llvm::Builder`],
112    /// tied to the lifetime of self.
113    pub(crate) fn llbuilder<'s>(&'s self) -> &'s llvm::Builder<'ll> {
114        // SAFETY: We own this builder, and the reference lifetime is tied to `&self`
115        // (not to 'll), so it won't be disposed by our Drop while the ref is live.
116        unsafe { self.llbuilder_raw.as_ref() }
117    }
118
119    pub(crate) fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
120        unsafe { llvm::LLVMBuildTrunc(self.llbuilder(), val, dest_ty, UNNAMED) }
121    }
122
123    pub(crate) fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
124        unsafe { llvm::LLVMBuildBitCast(self.llbuilder(), val, dest_ty, UNNAMED) }
125    }
126
127    pub(crate) fn ret_void(&mut self) {
128        llvm::LLVMBuildRetVoid(self.llbuilder());
129    }
130
131    pub(crate) fn ret(&mut self, v: &'ll Value) {
132        unsafe {
133            llvm::LLVMBuildRet(self.llbuilder(), v);
134        }
135    }
136
137    pub(crate) fn build(cx: &'a GenericCx<'ll, CX>, llbb: &'ll BasicBlock) -> Self {
138        let bx = Self::with_cx(cx);
139        unsafe {
140            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder(), llbb);
141        }
142        bx
143    }
144
145    // The generic builder has less functionality and thus (unlike the other alloca) we can not
146    // easily jump to the beginning of the function to place our allocas there. We trust the user
147    // to manually do that. FIXME(offload): improve the genericCx and add more llvm wrappers to
148    // handle this.
149    pub(crate) fn direct_alloca(&mut self, ty: &'ll Type, align: Align, name: &str) -> &'ll Value {
150        let val = unsafe {
151            let alloca = llvm::LLVMBuildAlloca(self.llbuilder(), ty, UNNAMED);
152            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
153            // Cast to default addrspace if necessary
154            llvm::LLVMBuildPointerCast(self.llbuilder(), alloca, self.cx.type_ptr(), UNNAMED)
155        };
156        if name != "" {
157            let name = std::ffi::CString::new(name).unwrap();
158            llvm::set_value_name(val, &name.as_bytes());
159        }
160        val
161    }
162
163    /// Don't use this method: instead use `inbounds_gep` or `inbounds_ptradd`.
164    // FIXME: Remove once unused.
165    pub(crate) fn typed_inbounds_gep(
166        &mut self,
167        ty: &'ll Type,
168        ptr: &'ll Value,
169        indices: &[&'ll Value],
170    ) -> &'ll Value {
171        unsafe {
172            llvm::LLVMBuildGEPWithNoWrapFlags(
173                self.llbuilder(),
174                ty,
175                ptr,
176                indices.as_ptr(),
177                indices.len() as c_uint,
178                UNNAMED,
179                GEPNoWrapFlags::InBounds,
180            )
181        }
182    }
183
184    fn build_gep(
185        &mut self,
186        stride: Size,
187        ptr: &'ll Value,
188        index: &'ll Value,
189        flags: GEPNoWrapFlags,
190    ) -> &'ll Value {
191        let ty = if stride.bytes() == 1 {
192            // LLVM prefers `i8` as the canonical form for a single-byte GEP.
193            self.cx.type_i8()
194        } else {
195            self.cx.type_array(self.cx.type_i8(), stride.bytes())
196        };
197        let indices = &[index];
198        unsafe {
199            llvm::LLVMBuildGEPWithNoWrapFlags(
200                self.llbuilder(),
201                ty,
202                ptr,
203                indices.as_ptr(),
204                indices.len() as c_uint,
205                UNNAMED,
206                flags,
207            )
208        }
209    }
210
211    pub(crate) fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
212        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:212",
                        "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                        ::tracing_core::__macro_support::Option::Some(212u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Store {0:?} -> {1:?}",
                                                    val, ptr) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("Store {:?} -> {:?}", val, ptr);
213        {
    match (&self.cx.type_kind(self.cx.val_ty(ptr)), &TypeKind::Pointer) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
214        unsafe {
215            let store = llvm::LLVMBuildStore(self.llbuilder(), val, ptr);
216            llvm::LLVMSetAlignment(store, align.bytes() as c_uint);
217            store
218        }
219    }
220
221    pub(crate) fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
222        unsafe {
223            let load = llvm::LLVMBuildLoad2(self.llbuilder(), ty, ptr, UNNAMED);
224            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
225            load
226        }
227    }
228}
229
230/// Empty string, to be used where LLVM expects an instruction name, indicating
231/// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
232// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
233pub(crate) const UNNAMED: *const c_char = c"".as_ptr();
234
235impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericBuilder<'_, 'll, CX> {
236    type Function = <GenericCx<'ll, CX> as BackendTypes>::Function;
237    type BasicBlock = <GenericCx<'ll, CX> as BackendTypes>::BasicBlock;
238    type Funclet = <GenericCx<'ll, CX> as BackendTypes>::Funclet;
239
240    type Value = <GenericCx<'ll, CX> as BackendTypes>::Value;
241    type Type = <GenericCx<'ll, CX> as BackendTypes>::Type;
242    type FunctionSignature = <GenericCx<'ll, CX> as BackendTypes>::FunctionSignature;
243
244    type DIScope = <GenericCx<'ll, CX> as BackendTypes>::DIScope;
245    type DILocation = <GenericCx<'ll, CX> as BackendTypes>::DILocation;
246    type DIVariable = <GenericCx<'ll, CX> as BackendTypes>::DIVariable;
247}
248
249impl abi::HasDataLayout for Builder<'_, '_, '_> {
250    fn data_layout(&self) -> &abi::TargetDataLayout {
251        self.cx.data_layout()
252    }
253}
254
255impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
256    #[inline]
257    fn tcx(&self) -> TyCtxt<'tcx> {
258        self.cx.tcx
259    }
260}
261
262impl<'tcx> ty::layout::HasTypingEnv<'tcx> for Builder<'_, '_, 'tcx> {
263    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
264        self.cx.typing_env()
265    }
266}
267
268impl HasTargetSpec for Builder<'_, '_, '_> {
269    #[inline]
270    fn target_spec(&self) -> &Target {
271        self.cx.target_spec()
272    }
273}
274
275impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
276    #[inline]
277    fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! {
278        self.cx.handle_layout_err(err, span, ty)
279    }
280}
281
282impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> {
283    #[inline]
284    fn handle_fn_abi_err(
285        &self,
286        err: FnAbiError<'tcx>,
287        span: Span,
288        fn_abi_request: FnAbiRequest<'tcx>,
289    ) -> ! {
290        self.cx.handle_fn_abi_err(err, span, fn_abi_request)
291    }
292}
293
294impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> {
295    type Target = CodegenCx<'ll, 'tcx>;
296
297    #[inline]
298    fn deref(&self) -> &Self::Target {
299        self.cx
300    }
301}
302
303macro_rules! math_builder_methods {
304    ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
305        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
306            unsafe {
307                llvm::$llvm_capi(self.llbuilder(), $($arg,)* UNNAMED)
308            }
309        })+
310    }
311}
312
313macro_rules! set_math_builder_methods {
314    ($($name:ident($($arg:ident),*) => ($llvm_capi:ident, $llvm_set_math:ident)),+ $(,)?) => {
315        $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
316            unsafe {
317                let instr = llvm::$llvm_capi(self.llbuilder(), $($arg,)* UNNAMED);
318                llvm::$llvm_set_math(instr);
319                instr
320            }
321        })+
322    }
323}
324
325impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
326    type CodegenCx = CodegenCx<'ll, 'tcx>;
327
328    fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
329        let bx = Builder::with_cx(cx);
330        unsafe {
331            llvm::LLVMPositionBuilderAtEnd(bx.llbuilder(), llbb);
332        }
333        bx
334    }
335
336    fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
337        self.cx
338    }
339
340    fn llbb(&self) -> &'ll BasicBlock {
341        unsafe { llvm::LLVMGetInsertBlock(self.llbuilder()) }
342    }
343
344    fn set_span(&mut self, span: rustc_span::Span) {
345        self.span = span;
346    }
347
348    fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
349        unsafe {
350            let name = SmallCStr::new(name);
351            llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
352        }
353    }
354
355    fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
356        Self::append_block(self.cx, self.llfn(), name)
357    }
358
359    fn switch_to_block(&mut self, llbb: Self::BasicBlock) {
360        *self = Self::build(self.cx, llbb)
361    }
362
363    fn ret_void(&mut self) {
364        llvm::LLVMBuildRetVoid(self.llbuilder());
365    }
366
367    fn ret(&mut self, v: &'ll Value) {
368        unsafe {
369            llvm::LLVMBuildRet(self.llbuilder(), v);
370        }
371    }
372
373    fn br(&mut self, dest: &'ll BasicBlock) {
374        unsafe {
375            llvm::LLVMBuildBr(self.llbuilder(), dest);
376        }
377    }
378
379    fn br_with_attrs(&mut self, dest: &'ll BasicBlock, loop_hint_attrs: &[UnrollAttr]) {
380        unsafe {
381            let val = llvm::LLVMBuildBr(self.llbuilder(), dest);
382
383            let mut nodes = Vec::new();
384
385            for unroll in loop_hint_attrs {
386                // UnrollAttr::Count needs a second operand, the provided count, but the other
387                // unroll hints do not.
388                let md_node = if let UnrollAttr::Count(count) = unroll {
389                    let unroll_meta = self.create_metadata("llvm.loop.unroll.count".as_bytes());
390                    let count = llvm::LLVMValueAsMetadata(self.get_const_i32(u64::from(*count)));
391                    self.md_node_in_context(&[unroll_meta, count])
392                } else {
393                    let metadata_str = match unroll {
394                        UnrollAttr::Hint => "llvm.loop.unroll.enable",
395                        UnrollAttr::Full => "llvm.loop.unroll.full",
396                        UnrollAttr::Never => "llvm.loop.unroll.disable",
397                        UnrollAttr::Count(_) => ::core::panicking::unreachable_default_msg()unreachable!(),
398                    };
399                    let unroll_meta = self.create_metadata(metadata_str.as_bytes());
400                    self.md_node_in_context(&[unroll_meta])
401                };
402                nodes.push(md_node);
403            }
404
405            if let [first, ..] = nodes[..] {
406                nodes.insert(0, first);
407
408                // Create the loop metadata node
409                let loop_meta_mdnode = self.set_metadata_node(val, llvm::MD_loop, &nodes);
410
411                // Look up the metadata node as a value
412                let loop_meta_val = llvm::LLVMGetMetadata(val, llvm::MD_loop).unwrap();
413
414                // Replace the first entry with a reference to itself
415                // This is required by LLVM. See the LangRef page for llvm.loop metadata.
416                llvm::LLVMReplaceMDNodeOperandWith(loop_meta_val, 0, loop_meta_mdnode);
417            }
418        }
419    }
420
421    fn cond_br(
422        &mut self,
423        cond: &'ll Value,
424        then_llbb: &'ll BasicBlock,
425        else_llbb: &'ll BasicBlock,
426    ) {
427        unsafe {
428            llvm::LLVMBuildCondBr(self.llbuilder(), cond, then_llbb, else_llbb);
429        }
430    }
431
432    fn switch(
433        &mut self,
434        v: &'ll Value,
435        else_llbb: &'ll BasicBlock,
436        cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
437    ) {
438        let switch =
439            unsafe { llvm::LLVMBuildSwitch(self.llbuilder(), v, else_llbb, cases.len() as c_uint) };
440        for (on_val, dest) in cases {
441            let on_val = self.const_uint_big(self.val_ty(v), on_val);
442            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
443        }
444    }
445
446    fn switch_with_weights(
447        &mut self,
448        v: Self::Value,
449        else_llbb: Self::BasicBlock,
450        else_is_cold: bool,
451        cases: impl ExactSizeIterator<Item = (u128, Self::BasicBlock, bool)>,
452    ) {
453        if self.cx.sess().opts.optimize == rustc_session::config::OptLevel::No {
454            self.switch(v, else_llbb, cases.map(|(val, dest, _)| (val, dest)));
455            return;
456        }
457
458        let id = self.cx.create_metadata(b"branch_weights");
459
460        // For switch instructions with 2 targets, the `llvm.expect` intrinsic is used.
461        // This function handles switch instructions with more than 2 targets and it needs to
462        // emit branch weights metadata instead of using the intrinsic.
463        // The values 1 and 2000 are the same as the values used by the `llvm.expect` intrinsic.
464        let cold_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(1));
465        let hot_weight = llvm::LLVMValueAsMetadata(self.cx.const_u32(2000));
466        let weight =
467            |is_cold: bool| -> &Metadata { if is_cold { cold_weight } else { hot_weight } };
468
469        let mut md: SmallVec<[&Metadata; 16]> = SmallVec::with_capacity(cases.len() + 2);
470        md.push(id);
471        md.push(weight(else_is_cold));
472
473        let switch =
474            unsafe { llvm::LLVMBuildSwitch(self.llbuilder(), v, else_llbb, cases.len() as c_uint) };
475        for (on_val, dest, is_cold) in cases {
476            let on_val = self.const_uint_big(self.val_ty(v), on_val);
477            unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
478            md.push(weight(is_cold));
479        }
480
481        self.cx.set_metadata_node(switch, llvm::MD_prof, &md);
482    }
483
484    fn invoke(
485        &mut self,
486        llty: &'ll Type,
487        fn_attrs: Option<&CodegenFnAttrs>,
488        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
489        llfn: &'ll Value,
490        return_slot: ReturnSlot<&'ll Value>,
491        args: &[&'ll Value],
492        then: &'ll BasicBlock,
493        catch: &'ll BasicBlock,
494        funclet: Option<&Funclet<'ll>>,
495        instance: Option<Instance<'tcx>>,
496    ) -> &'ll Value {
497        // If this function returns indirectly (`PassMode::Indirect`),
498        // the `return_slot` should be the first argument.
499        let args = match return_slot {
500            ReturnSlot::Direct => args.to_vec(),
501            ReturnSlot::Indirect(sret_ptr) => {
502                let mut args = args.to_vec();
503                args.insert(0, sret_ptr);
504                args
505            }
506        };
507        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:507",
                        "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                        ::tracing_core::__macro_support::Option::Some(507u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("invoke {0:?} with args ({1:?})",
                                                    llfn, args) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("invoke {:?} with args ({:?})", llfn, args);
508        let args = self.check_call("invoke", llty, llfn, &args);
509        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
510        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
511        if let Some(funclet_bundle) = funclet_bundle {
512            bundles.push(funclet_bundle);
513        }
514
515        // Emit CFI pointer type membership test
516        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
517
518        // Emit KCFI operand bundle
519        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
520        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
521            bundles.push(kcfi_bundle);
522        }
523
524        let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
525        if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
526            bundles.push(p);
527        }
528
529        let invoke = unsafe {
530            llvm::LLVMBuildInvokeWithOperandBundles(
531                self.llbuilder(),
532                llty,
533                llfn,
534                args.as_ptr(),
535                args.len() as c_uint,
536                then,
537                catch,
538                bundles.as_ptr(),
539                bundles.len() as c_uint,
540                UNNAMED,
541            )
542        };
543        if let Some(fn_abi) = fn_abi {
544            fn_abi.apply_attrs_callsite(self, invoke);
545        }
546        invoke
547    }
548
549    fn unreachable(&mut self) {
550        unsafe {
551            llvm::LLVMBuildUnreachable(self.llbuilder());
552        }
553    }
554
555    fn add(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildAdd(self.llbuilder(), a, b, UNNAMED) }
}
fn fadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildFAdd(self.llbuilder(), a, b, UNNAMED) }
}
fn sub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildSub(self.llbuilder(), a, b, UNNAMED) }
}
fn fsub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildFSub(self.llbuilder(), a, b, UNNAMED) }
}
fn mul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildMul(self.llbuilder(), a, b, UNNAMED) }
}
fn fmul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildFMul(self.llbuilder(), a, b, UNNAMED) }
}
fn udiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildUDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn exactudiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildExactUDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn sdiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildSDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn exactsdiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildExactSDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn fdiv(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildFDiv(self.llbuilder(), a, b, UNNAMED) }
}
fn urem(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildURem(self.llbuilder(), a, b, UNNAMED) }
}
fn srem(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildSRem(self.llbuilder(), a, b, UNNAMED) }
}
fn frem(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildFRem(self.llbuilder(), a, b, UNNAMED) }
}
fn shl(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildShl(self.llbuilder(), a, b, UNNAMED) }
}
fn lshr(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildLShr(self.llbuilder(), a, b, UNNAMED) }
}
fn ashr(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildAShr(self.llbuilder(), a, b, UNNAMED) }
}
fn and(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildAnd(self.llbuilder(), a, b, UNNAMED) }
}
fn or(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildOr(self.llbuilder(), a, b, UNNAMED) }
}
fn xor(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildXor(self.llbuilder(), a, b, UNNAMED) }
}
fn neg(&mut self, x: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildNeg(self.llbuilder(), x, UNNAMED) }
}
fn fneg(&mut self, x: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildFNeg(self.llbuilder(), x, UNNAMED) }
}
fn not(&mut self, x: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildNot(self.llbuilder(), x, UNNAMED) }
}
fn unchecked_sadd(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildNSWAdd(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_uadd(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildNUWAdd(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_ssub(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildNSWSub(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_usub(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildNUWSub(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_smul(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildNSWMul(self.llbuilder(), x, y, UNNAMED) }
}
fn unchecked_umul(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe { llvm::LLVMBuildNUWMul(self.llbuilder(), x, y, UNNAMED) }
}math_builder_methods! {
556        add(a, b) => LLVMBuildAdd,
557        fadd(a, b) => LLVMBuildFAdd,
558        sub(a, b) => LLVMBuildSub,
559        fsub(a, b) => LLVMBuildFSub,
560        mul(a, b) => LLVMBuildMul,
561        fmul(a, b) => LLVMBuildFMul,
562        udiv(a, b) => LLVMBuildUDiv,
563        exactudiv(a, b) => LLVMBuildExactUDiv,
564        sdiv(a, b) => LLVMBuildSDiv,
565        exactsdiv(a, b) => LLVMBuildExactSDiv,
566        fdiv(a, b) => LLVMBuildFDiv,
567        urem(a, b) => LLVMBuildURem,
568        srem(a, b) => LLVMBuildSRem,
569        frem(a, b) => LLVMBuildFRem,
570        shl(a, b) => LLVMBuildShl,
571        lshr(a, b) => LLVMBuildLShr,
572        ashr(a, b) => LLVMBuildAShr,
573        and(a, b) => LLVMBuildAnd,
574        or(a, b) => LLVMBuildOr,
575        xor(a, b) => LLVMBuildXor,
576        neg(x) => LLVMBuildNeg,
577        fneg(x) => LLVMBuildFNeg,
578        not(x) => LLVMBuildNot,
579        unchecked_sadd(x, y) => LLVMBuildNSWAdd,
580        unchecked_uadd(x, y) => LLVMBuildNUWAdd,
581        unchecked_ssub(x, y) => LLVMBuildNSWSub,
582        unchecked_usub(x, y) => LLVMBuildNUWSub,
583        unchecked_smul(x, y) => LLVMBuildNSWMul,
584        unchecked_umul(x, y) => LLVMBuildNUWMul,
585    }
586
587    fn unchecked_suadd(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
588        unsafe {
589            let add = llvm::LLVMBuildAdd(self.llbuilder(), a, b, UNNAMED);
590            if llvm::LLVMIsAInstruction(add).is_some() {
591                llvm::LLVMSetNUW(add, TRUE);
592                llvm::LLVMSetNSW(add, TRUE);
593            }
594            add
595        }
596    }
597    fn unchecked_susub(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
598        unsafe {
599            let sub = llvm::LLVMBuildSub(self.llbuilder(), a, b, UNNAMED);
600            if llvm::LLVMIsAInstruction(sub).is_some() {
601                llvm::LLVMSetNUW(sub, TRUE);
602                llvm::LLVMSetNSW(sub, TRUE);
603            }
604            sub
605        }
606    }
607    fn unchecked_sumul(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
608        unsafe {
609            let mul = llvm::LLVMBuildMul(self.llbuilder(), a, b, UNNAMED);
610            if llvm::LLVMIsAInstruction(mul).is_some() {
611                llvm::LLVMSetNUW(mul, TRUE);
612                llvm::LLVMSetNSW(mul, TRUE);
613            }
614            mul
615        }
616    }
617
618    fn or_disjoint(&mut self, a: &'ll Value, b: &'ll Value) -> &'ll Value {
619        unsafe {
620            let or = llvm::LLVMBuildOr(self.llbuilder(), a, b, UNNAMED);
621
622            // If a and b are both values, then `or` is a value, rather than
623            // an instruction, so we need to check before setting the flag.
624            // (See also `LLVMBuildNUWNeg` which also needs a check.)
625            if llvm::LLVMIsAInstruction(or).is_some() {
626                llvm::LLVMSetIsDisjoint(or, TRUE);
627            }
628            or
629        }
630    }
631
632    fn fadd_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFAdd(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetFastMath(instr);
        instr
    }
}
fn fsub_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFSub(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetFastMath(instr);
        instr
    }
}
fn fmul_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFMul(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetFastMath(instr);
        instr
    }
}
fn fdiv_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFDiv(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetFastMath(instr);
        instr
    }
}
fn frem_fast(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFRem(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetFastMath(instr);
        instr
    }
}
fn fadd_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFAdd(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetAlgebraicMath(instr);
        instr
    }
}
fn fsub_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFSub(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetAlgebraicMath(instr);
        instr
    }
}
fn fmul_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFMul(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetAlgebraicMath(instr);
        instr
    }
}
fn fdiv_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFDiv(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetAlgebraicMath(instr);
        instr
    }
}
fn frem_algebraic(&mut self, x: &'ll Value, y: &'ll Value) -> &'ll Value {
    unsafe {
        let instr = llvm::LLVMBuildFRem(self.llbuilder(), x, y, UNNAMED);
        llvm::LLVMRustSetAlgebraicMath(instr);
        instr
    }
}set_math_builder_methods! {
633        fadd_fast(x, y) => (LLVMBuildFAdd, LLVMRustSetFastMath),
634        fsub_fast(x, y) => (LLVMBuildFSub, LLVMRustSetFastMath),
635        fmul_fast(x, y) => (LLVMBuildFMul, LLVMRustSetFastMath),
636        fdiv_fast(x, y) => (LLVMBuildFDiv, LLVMRustSetFastMath),
637        frem_fast(x, y) => (LLVMBuildFRem, LLVMRustSetFastMath),
638        fadd_algebraic(x, y) => (LLVMBuildFAdd, LLVMRustSetAlgebraicMath),
639        fsub_algebraic(x, y) => (LLVMBuildFSub, LLVMRustSetAlgebraicMath),
640        fmul_algebraic(x, y) => (LLVMBuildFMul, LLVMRustSetAlgebraicMath),
641        fdiv_algebraic(x, y) => (LLVMBuildFDiv, LLVMRustSetAlgebraicMath),
642        frem_algebraic(x, y) => (LLVMBuildFRem, LLVMRustSetAlgebraicMath),
643    }
644
645    fn checked_binop(
646        &mut self,
647        oop: OverflowOp,
648        ty: Ty<'tcx>,
649        lhs: Self::Value,
650        rhs: Self::Value,
651    ) -> (Self::Value, Self::Value) {
652        let (size, signed) = ty.int_size_and_signed(self.tcx);
653        let width = size.bits();
654
655        if !signed {
656            match oop {
657                OverflowOp::Sub => {
658                    // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these
659                    // to be the canonical form. It will attempt to reform llvm.usub.with.overflow
660                    // in the backend if profitable.
661                    let sub = self.sub(lhs, rhs);
662                    let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs);
663                    return (sub, cmp);
664                }
665                OverflowOp::Add => {
666                    // Like with sub above, using icmp is the preferred form. See
667                    // <https://rust-lang.zulipchat.com/#narrow/channel/187780-t-compiler.2Fllvm/topic/.60uadd.2Ewith.2Eoverflow.60.20.28again.29/near/533041085>
668                    let add = self.add(lhs, rhs);
669                    let cmp = self.icmp(IntPredicate::IntULT, add, lhs);
670                    return (add, cmp);
671                }
672                OverflowOp::Mul => {}
673            }
674        }
675
676        let oop_str = match oop {
677            OverflowOp::Add => "add",
678            OverflowOp::Sub => "sub",
679            OverflowOp::Mul => "mul",
680        };
681
682        let name = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("llvm.{0}{1}.with.overflow",
                if signed { 's' } else { 'u' }, oop_str))
    })format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });
683
684        let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
685        (self.extract_value(res, 0), self.extract_value(res, 1))
686    }
687
688    fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
689        if self.cx().val_ty(val) == self.cx().type_i1() {
690            self.zext(val, self.cx().type_i8())
691        } else {
692            val
693        }
694    }
695
696    fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value {
697        if scalar.is_bool() {
698            return self.unchecked_utrunc(val, self.cx().type_i1());
699        }
700        val
701    }
702
703    fn alloca(&mut self, size: Size, align: Align) -> &'ll Value {
704        let mut bx = Builder::with_cx(self.cx);
705        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
706        let ty = self.cx().type_array(self.cx().type_i8(), size.bytes());
707        unsafe {
708            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder(), ty, UNNAMED);
709            llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
710            // Cast to default addrspace if necessary
711            llvm::LLVMBuildPointerCast(bx.llbuilder(), alloca, self.cx().type_ptr(), UNNAMED)
712        }
713    }
714
715    fn alloca_with_ty(&mut self, layout: TyAndLayout<'tcx>) -> Self::Value {
716        let mut bx = Builder::with_cx(self.cx);
717        bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
718        let scalable_vector_ty = layout.llvm_type(self.cx);
719
720        unsafe {
721            let alloca = llvm::LLVMBuildAlloca(bx.llbuilder(), scalable_vector_ty, UNNAMED);
722            llvm::LLVMSetAlignment(alloca, layout.align.bytes() as c_uint);
723            alloca
724        }
725    }
726
727    fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
728        unsafe {
729            let load = llvm::LLVMBuildLoad2(self.llbuilder(), ty, ptr, UNNAMED);
730            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
731            llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
732            load
733        }
734    }
735
736    fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
737        unsafe {
738            let load = self.load(ty, ptr, align);
739            llvm::LLVMSetVolatile(load, llvm::TRUE);
740            load
741        }
742    }
743
744    fn atomic_load(
745        &mut self,
746        ty: &'ll Type,
747        ptr: &'ll Value,
748        order: rustc_middle::ty::AtomicOrdering,
749        volatile: bool,
750        size: Size,
751    ) -> &'ll Value {
752        unsafe {
753            let load = llvm::LLVMBuildLoad2(self.llbuilder(), ty, ptr, UNNAMED);
754            // Set atomic ordering
755            llvm::LLVMSetOrdering(load, AtomicOrdering::from_generic(order));
756            if volatile {
757                llvm::LLVMSetVolatile(load, llvm::TRUE);
758            }
759            // LLVM requires the alignment of atomic loads to be at least the size of the type.
760            llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
761            load
762        }
763    }
764
765    {}
#[allow(clippy :: suspicious_else_formatting)]
{
    let __tracing_attr_span;
    let __tracing_attr_guard;
    if ::tracing::Level::TRACE <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::TRACE <=
                    ::tracing::level_filters::LevelFilter::current() ||
            { false } {
        __tracing_attr_span =
            {
                use ::tracing::__macro_support::Callsite as _;
                static __CALLSITE: ::tracing::callsite::DefaultCallsite =
                    {
                        static META: ::tracing::Metadata<'static> =
                            {
                                ::tracing_core::metadata::Metadata::new("load_operand",
                                    "rustc_codegen_llvm::builder", ::tracing::Level::TRACE,
                                    ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                                    ::tracing_core::__macro_support::Option::Some(765u32),
                                    ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                                    ::tracing_core::field::FieldSet::new(&[{
                                                        const NAME:
                                                            ::tracing::__macro_support::FieldName<{
                                                                ::tracing::__macro_support::FieldName::len("place")
                                                            }> =
                                                            ::tracing::__macro_support::FieldName::new("place");
                                                        NAME.as_str()
                                                    }], ::tracing_core::callsite::Identifier(&__CALLSITE)),
                                    ::tracing::metadata::Kind::SPAN)
                            };
                        ::tracing::callsite::DefaultCallsite::new(&META)
                    };
                let mut interest = ::tracing::subscriber::Interest::never();
                if ::tracing::Level::TRACE <=
                                    ::tracing::level_filters::STATIC_MAX_LEVEL &&
                                ::tracing::Level::TRACE <=
                                    ::tracing::level_filters::LevelFilter::current() &&
                            { interest = __CALLSITE.interest(); !interest.is_never() }
                        &&
                        ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                            interest) {
                    let meta = __CALLSITE.metadata();
                    ::tracing::Span::new(meta,
                        &{
                                #[allow(unused_imports)]
                                use ::tracing::field::{debug, display, Value};
                                meta.fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&place)
                                                            as &dyn ::tracing::field::Value))])
                            })
                } else {
                    let span =
                        ::tracing::__macro_support::__disabled_span(__CALLSITE.metadata());
                    {};
                    span
                }
            };
        __tracing_attr_guard = __tracing_attr_span.enter();
    }

    #[warn(clippy :: suspicious_else_formatting)]
    {

        #[allow(unknown_lints, unreachable_code, clippy ::
        diverging_sub_expression, clippy :: empty_loop, clippy ::
        let_unit_value, clippy :: let_with_type_underscore, clippy ::
        needless_return, clippy :: unreachable)]
        if false {
            let __tracing_attr_fake_return: OperandRef<'tcx, &'ll Value> =
                loop {};
            return __tracing_attr_fake_return;
        }
        {
            if place.layout.is_unsized() {
                let tail =
                    self.tcx.struct_tail_for_codegen(place.layout.ty,
                        self.typing_env());
                if #[allow(non_exhaustive_omitted_patterns)] match tail.kind()
                        {
                        ty::Foreign(..) => true,
                        _ => false,
                    } {
                    {
                        ::core::panicking::panic_fmt(format_args!("unsized locals must not be `extern` types"));
                    };
                }
            }
            {
                match (&place.val.llextra.is_some(),
                        &place.layout.is_unsized()) {
                    (left_val, right_val) => {
                        if !(*left_val == *right_val) {
                            let kind = ::core::panicking::AssertKind::Eq;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
            if place.layout.is_zst() {
                return OperandRef::zero_sized(place.layout);
            }
            fn scalar_load_metadata<'a, 'll,
                'tcx>(bx: &mut Builder<'a, 'll, 'tcx>, load: &'ll Value,
                scalar: abi::Scalar, layout: TyAndLayout<'tcx>,
                offset: Size) {
                {}

                #[allow(clippy :: suspicious_else_formatting)]
                {
                    let __tracing_attr_span;
                    let __tracing_attr_guard;
                    if ::tracing::Level::TRACE <=
                                    ::tracing::level_filters::STATIC_MAX_LEVEL &&
                                ::tracing::Level::TRACE <=
                                    ::tracing::level_filters::LevelFilter::current() ||
                            { false } {
                        __tracing_attr_span =
                            {
                                use ::tracing::__macro_support::Callsite as _;
                                static __CALLSITE: ::tracing::callsite::DefaultCallsite =
                                    {
                                        static META: ::tracing::Metadata<'static> =
                                            {
                                                ::tracing_core::metadata::Metadata::new("scalar_load_metadata",
                                                    "rustc_codegen_llvm::builder", ::tracing::Level::TRACE,
                                                    ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                                                    ::tracing_core::__macro_support::Option::Some(782u32),
                                                    ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                                                    ::tracing_core::field::FieldSet::new(&[{
                                                                        const NAME:
                                                                            ::tracing::__macro_support::FieldName<{
                                                                                ::tracing::__macro_support::FieldName::len("load")
                                                                            }> =
                                                                            ::tracing::__macro_support::FieldName::new("load");
                                                                        NAME.as_str()
                                                                    },
                                                                    {
                                                                        const NAME:
                                                                            ::tracing::__macro_support::FieldName<{
                                                                                ::tracing::__macro_support::FieldName::len("scalar")
                                                                            }> =
                                                                            ::tracing::__macro_support::FieldName::new("scalar");
                                                                        NAME.as_str()
                                                                    },
                                                                    {
                                                                        const NAME:
                                                                            ::tracing::__macro_support::FieldName<{
                                                                                ::tracing::__macro_support::FieldName::len("layout")
                                                                            }> =
                                                                            ::tracing::__macro_support::FieldName::new("layout");
                                                                        NAME.as_str()
                                                                    },
                                                                    {
                                                                        const NAME:
                                                                            ::tracing::__macro_support::FieldName<{
                                                                                ::tracing::__macro_support::FieldName::len("offset")
                                                                            }> =
                                                                            ::tracing::__macro_support::FieldName::new("offset");
                                                                        NAME.as_str()
                                                                    }], ::tracing_core::callsite::Identifier(&__CALLSITE)),
                                                    ::tracing::metadata::Kind::SPAN)
                                            };
                                        ::tracing::callsite::DefaultCallsite::new(&META)
                                    };
                                let mut interest = ::tracing::subscriber::Interest::never();
                                if ::tracing::Level::TRACE <=
                                                    ::tracing::level_filters::STATIC_MAX_LEVEL &&
                                                ::tracing::Level::TRACE <=
                                                    ::tracing::level_filters::LevelFilter::current() &&
                                            { interest = __CALLSITE.interest(); !interest.is_never() }
                                        &&
                                        ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                                            interest) {
                                    let meta = __CALLSITE.metadata();
                                    ::tracing::Span::new(meta,
                                        &{
                                                #[allow(unused_imports)]
                                                use ::tracing::field::{debug, display, Value};
                                                meta.fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&load)
                                                                            as &dyn ::tracing::field::Value)),
                                                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&scalar)
                                                                            as &dyn ::tracing::field::Value)),
                                                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&layout)
                                                                            as &dyn ::tracing::field::Value)),
                                                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&offset)
                                                                            as &dyn ::tracing::field::Value))])
                                            })
                                } else {
                                    let span =
                                        ::tracing::__macro_support::__disabled_span(__CALLSITE.metadata());
                                    {};
                                    span
                                }
                            };
                        __tracing_attr_guard = __tracing_attr_span.enter();
                    }

                    #[warn(clippy :: suspicious_else_formatting)]
                    {

                        #[allow(unknown_lints, unreachable_code, clippy ::
                        diverging_sub_expression, clippy :: empty_loop, clippy ::
                        let_unit_value, clippy :: let_with_type_underscore, clippy
                        :: needless_return, clippy :: unreachable)]
                        if false {
                            let __tracing_attr_fake_return: () = loop {};
                            return __tracing_attr_fake_return;
                        }
                        {
                            if bx.cx.sess().opts.optimize == OptLevel::No { return; }
                            if !scalar.is_uninit_valid() { bx.noundef_metadata(load); }
                            match scalar.primitive() {
                                abi::Primitive::Int(..) => {
                                    if !scalar.is_always_valid(bx) {
                                        bx.range_metadata(load, scalar.valid_range(bx));
                                    }
                                }
                                abi::Primitive::Pointer(_) => {
                                    if !scalar.valid_range(bx).contains(0) {
                                        bx.nonnull_metadata(load);
                                    }
                                    if let Some(pointee) = layout.pointee_info_at(bx, offset) &&
                                            pointee.align > Align::ONE {
                                        bx.align_metadata(load, pointee.align);
                                    }
                                }
                                abi::Primitive::Float(_) => {}
                            }
                        }
                    }
                }
            }
            let val =
                if let Some(_) = place.val.llextra {
                    OperandValue::Ref(place.val)
                } else if place.layout.backend_repr.is_scalar_or_simd() {
                    let mut const_llval = None;
                    let llty = place.layout.llvm_type(self);
                    if let Some(global) =
                            llvm::LLVMIsAGlobalVariable(place.val.llval) {
                        if llvm::LLVMIsGlobalConstant(global).is_true() {
                            if let Some(init) = llvm::LLVMGetInitializer(global) {
                                if self.val_ty(init) == llty { const_llval = Some(init); }
                            }
                        }
                    }
                    let llval =
                        const_llval.unwrap_or_else(||
                                {
                                    let load =
                                        self.load(llty, place.val.llval, place.val.align);
                                    if let abi::BackendRepr::Scalar(scalar) =
                                            place.layout.backend_repr {
                                        scalar_load_metadata(self, load, scalar, place.layout,
                                            Size::ZERO);
                                        self.to_immediate_scalar(load, scalar)
                                    } else { load }
                                });
                    OperandValue::Immediate(llval)
                } else if let abi::BackendRepr::ScalarPair { a, b, b_offset }
                        = place.layout.backend_repr {
                    let mut load =
                        |i, scalar: abi::Scalar, layout, align, offset|
                            {
                                let llptr =
                                    if i == 0 {
                                        place.val.llval
                                    } else {
                                        self.inbounds_ptradd(place.val.llval,
                                            self.const_usize(b_offset.bytes()))
                                    };
                                let llty =
                                    place.layout.scalar_pair_element_llvm_type(self, i, false);
                                let load = self.load(llty, llptr, align);
                                scalar_load_metadata(self, load, scalar, layout, offset);
                                self.to_immediate_scalar(load, scalar)
                            };
                    OperandValue::Pair(load(0, a, place.layout, place.val.align,
                            Size::ZERO),
                        load(1, b, place.layout,
                            place.val.align.restrict_for_offset(b_offset), b_offset))
                } else { OperandValue::Ref(place.val) };
            OperandRef { val, layout: place.layout, move_annotation: None }
        }
    }
}#[instrument(level = "trace", skip(self))]
766    fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
767        if place.layout.is_unsized() {
768            let tail = self.tcx.struct_tail_for_codegen(place.layout.ty, self.typing_env());
769            if matches!(tail.kind(), ty::Foreign(..)) {
770                // Unsized locals and, at least conceptually, even unsized arguments must be copied
771                // around, which requires dynamically determining their size. Therefore, we cannot
772                // allow `extern` types here. Consult t-opsem before removing this check.
773                panic!("unsized locals must not be `extern` types");
774            }
775        }
776        assert_eq!(place.val.llextra.is_some(), place.layout.is_unsized());
777
778        if place.layout.is_zst() {
779            return OperandRef::zero_sized(place.layout);
780        }
781
782        #[instrument(level = "trace", skip(bx))]
783        fn scalar_load_metadata<'a, 'll, 'tcx>(
784            bx: &mut Builder<'a, 'll, 'tcx>,
785            load: &'ll Value,
786            scalar: abi::Scalar,
787            layout: TyAndLayout<'tcx>,
788            offset: Size,
789        ) {
790            if bx.cx.sess().opts.optimize == OptLevel::No {
791                // Don't emit metadata we're not going to use
792                return;
793            }
794
795            if !scalar.is_uninit_valid() {
796                bx.noundef_metadata(load);
797            }
798
799            match scalar.primitive() {
800                abi::Primitive::Int(..) => {
801                    if !scalar.is_always_valid(bx) {
802                        bx.range_metadata(load, scalar.valid_range(bx));
803                    }
804                }
805                abi::Primitive::Pointer(_) => {
806                    if !scalar.valid_range(bx).contains(0) {
807                        bx.nonnull_metadata(load);
808                    }
809
810                    if let Some(pointee) = layout.pointee_info_at(bx, offset)
811                        && pointee.align > Align::ONE
812                    {
813                        bx.align_metadata(load, pointee.align);
814                    }
815                }
816                abi::Primitive::Float(_) => {}
817            }
818        }
819
820        let val = if let Some(_) = place.val.llextra {
821            // FIXME: Merge with the `else` below?
822            OperandValue::Ref(place.val)
823        } else if place.layout.backend_repr.is_scalar_or_simd() {
824            let mut const_llval = None;
825            let llty = place.layout.llvm_type(self);
826            if let Some(global) = llvm::LLVMIsAGlobalVariable(place.val.llval) {
827                if llvm::LLVMIsGlobalConstant(global).is_true() {
828                    if let Some(init) = llvm::LLVMGetInitializer(global) {
829                        if self.val_ty(init) == llty {
830                            const_llval = Some(init);
831                        }
832                    }
833                }
834            }
835
836            let llval = const_llval.unwrap_or_else(|| {
837                let load = self.load(llty, place.val.llval, place.val.align);
838                if let abi::BackendRepr::Scalar(scalar) = place.layout.backend_repr {
839                    scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO);
840                    self.to_immediate_scalar(load, scalar)
841                } else {
842                    load
843                }
844            });
845            OperandValue::Immediate(llval)
846        } else if let abi::BackendRepr::ScalarPair { a, b, b_offset } = place.layout.backend_repr {
847            let mut load = |i, scalar: abi::Scalar, layout, align, offset| {
848                let llptr = if i == 0 {
849                    place.val.llval
850                } else {
851                    self.inbounds_ptradd(place.val.llval, self.const_usize(b_offset.bytes()))
852                };
853                let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
854                let load = self.load(llty, llptr, align);
855                scalar_load_metadata(self, load, scalar, layout, offset);
856                self.to_immediate_scalar(load, scalar)
857            };
858
859            OperandValue::Pair(
860                load(0, a, place.layout, place.val.align, Size::ZERO),
861                load(1, b, place.layout, place.val.align.restrict_for_offset(b_offset), b_offset),
862            )
863        } else {
864            OperandValue::Ref(place.val)
865        };
866
867        OperandRef { val, layout: place.layout, move_annotation: None }
868    }
869
870    fn write_operand_repeatedly(
871        &mut self,
872        cg_elem: OperandRef<'tcx, &'ll Value>,
873        count: u64,
874        dest: PlaceRef<'tcx, &'ll Value>,
875    ) {
876        if self.cx.sess().opts.optimize == OptLevel::No {
877            // To let debuggers single-step over lines like
878            //
879            //     let foo = ["bar"; 42];
880            //
881            // we need the debugger-friendly LLVM IR that `_unoptimized()`
882            // provides. The `_optimized()` version generates trickier LLVM IR.
883            // See PR #148058 for a failed attempt at handling that.
884            self.write_operand_repeatedly_unoptimized(cg_elem, count, dest);
885        } else {
886            self.write_operand_repeatedly_optimized(cg_elem, count, dest);
887        }
888    }
889
890    fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) {
891        if self.cx.sess().opts.optimize == OptLevel::No {
892            // Don't emit metadata we're not going to use
893            return;
894        }
895
896        let llty = self.cx.val_ty(load);
897        let md = [
898            llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.start)),
899            llvm::LLVMValueAsMetadata(self.cx.const_uint_big(llty, range.end.wrapping_add(1))),
900        ];
901        self.set_metadata_node(load, llvm::MD_range, &md);
902    }
903
904    fn nonnull_metadata(&mut self, load: &'ll Value) {
905        self.set_metadata_node(load, llvm::MD_nonnull, &[]);
906    }
907
908    fn store_with_flags(
909        &mut self,
910        val: &'ll Value,
911        ptr: &'ll Value,
912        align: Align,
913        flags: MemFlags,
914    ) {
915        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:915",
                        "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                        ::tracing_core::__macro_support::Option::Some(915u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Store {0:?} -> {1:?} ({2:?})",
                                                    val, ptr, flags) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
916        {
    match (&self.cx.type_kind(self.cx.val_ty(ptr)), &TypeKind::Pointer) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
917        unsafe {
918            let store = llvm::LLVMBuildStore(self.llbuilder(), val, ptr);
919            let align = align.min(self.cx().tcx.sess.target.max_reliable_alignment());
920            let align = align.bytes() as c_uint;
921            llvm::LLVMSetAlignment(store, align);
922            if flags.contains(MemFlags::VOLATILE) {
923                llvm::LLVMSetVolatile(store, llvm::TRUE);
924            }
925            if flags.contains(MemFlags::NONTEMPORAL) {
926                // Make sure that the current target architectures supports "sane" non-temporal
927                // stores, i.e., non-temporal stores that are equivalent to regular stores except
928                // for performance. LLVM doesn't seem to care about this, and will happily treat
929                // `!nontemporal` stores as-if they were normal stores (for reordering optimizations
930                // etc) even on x86, despite later lowering them to MOVNT which do *not* behave like
931                // regular stores but require special fences. So we keep a list of architectures
932                // where `!nontemporal` is known to be truly just a hint, and use regular stores
933                // everywhere else. (In the future, we could alternatively ensure that an sfence
934                // gets emitted after a sequence of movnt before any kind of synchronizing
935                // operation. But it's not clear how to do that with LLVM.)
936                // For more context, see <https://github.com/rust-lang/rust/issues/114582> and
937                // <https://github.com/llvm/llvm-project/issues/64521>.
938                let use_nontemporal = #[allow(non_exhaustive_omitted_patterns)] match self.cx.tcx.sess.target.arch {
    Arch::AArch64 | Arch::Arm | Arch::RiscV32 | Arch::RiscV64 => true,
    _ => false,
}matches!(
939                    self.cx.tcx.sess.target.arch,
940                    Arch::AArch64 | Arch::Arm | Arch::RiscV32 | Arch::RiscV64
941                );
942                if use_nontemporal {
943                    // According to LLVM [1] building a nontemporal store must
944                    // *always* point to a metadata value of the integer 1.
945                    //
946                    // [1]: https://llvm.org/docs/LangRef.html#store-instruction
947                    let one = llvm::LLVMValueAsMetadata(self.cx.const_i32(1));
948                    self.set_metadata_node(store, llvm::MD_nontemporal, &[one]);
949                }
950            }
951            if flags.contains(MemFlags::CAPTURES_READ_ONLY) {
952                if !(self.type_kind(self.val_ty(val)) == TypeKind::Pointer) {
    {
        ::core::panicking::panic_fmt(format_args!("CAPTURED_READ_ONLY is only supported on pointer stores"));
    }
};assert!(
953                    self.type_kind(self.val_ty(val)) == TypeKind::Pointer,
954                    "CAPTURED_READ_ONLY is only supported on pointer stores"
955                );
956                let args = [
957                    self.cx.create_metadata(b"address"),
958                    self.cx.create_metadata(b"read_provenance"),
959                ];
960                // FIXME: Switch this to use MD_captures once LLVM 22 is the minimum.
961                let id = self.get_md_kind_id("captures");
962                let md = llvm::LLVMMDNodeInContext2(self.cx.llcx, args.as_ptr(), args.len());
963                self.set_metadata(store, id, md);
964            }
965        }
966    }
967
968    fn atomic_store(
969        &mut self,
970        val: &'ll Value,
971        ptr: &'ll Value,
972        order: rustc_middle::ty::AtomicOrdering,
973        volatile: bool,
974        size: Size,
975    ) {
976        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:976",
                        "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                        ::tracing_core::__macro_support::Option::Some(976u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("Store {0:?} -> {1:?}",
                                                    val, ptr) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("Store {:?} -> {:?}", val, ptr);
977        {
    match (&self.cx.type_kind(self.cx.val_ty(ptr)), &TypeKind::Pointer) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
978        unsafe {
979            let store = llvm::LLVMBuildStore(self.llbuilder(), val, ptr);
980            // Set atomic ordering
981            llvm::LLVMSetOrdering(store, AtomicOrdering::from_generic(order));
982            if volatile {
983                llvm::LLVMSetVolatile(store, llvm::TRUE);
984            }
985            // LLVM requires the alignment of atomic stores to be at least the size of the type.
986            llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
987        }
988    }
989
990    fn gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
991        self.build_gep(stride, ptr, index, GEPNoWrapFlags::default())
992    }
993
994    fn inbounds_gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
995        self.build_gep(stride, ptr, index, GEPNoWrapFlags::InBounds)
996    }
997
998    fn inbounds_nuw_gep(&mut self, stride: Size, ptr: &'ll Value, index: &'ll Value) -> &'ll Value {
999        self.build_gep(stride, ptr, index, GEPNoWrapFlags::InBounds | GEPNoWrapFlags::NUW)
1000    }
1001
1002    /* Casts */
1003    fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1004        unsafe { llvm::LLVMBuildTrunc(self.llbuilder(), val, dest_ty, UNNAMED) }
1005    }
1006
1007    fn unchecked_utrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1008        if true {
    {
        match (&self.val_ty(val), &dest_ty) {
            (left_val, right_val) => {
                if *left_val == *right_val {
                    let kind = ::core::panicking::AssertKind::Ne;
                    ::core::panicking::assert_failed(kind, &*left_val,
                        &*right_val, ::core::option::Option::None);
                }
            }
        }
    };
};debug_assert_ne!(self.val_ty(val), dest_ty);
1009
1010        let trunc = self.trunc(val, dest_ty);
1011        unsafe {
1012            if llvm::LLVMIsAInstruction(trunc).is_some() {
1013                llvm::LLVMSetNUW(trunc, TRUE);
1014            }
1015        }
1016        trunc
1017    }
1018
1019    fn unchecked_strunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1020        if true {
    {
        match (&self.val_ty(val), &dest_ty) {
            (left_val, right_val) => {
                if *left_val == *right_val {
                    let kind = ::core::panicking::AssertKind::Ne;
                    ::core::panicking::assert_failed(kind, &*left_val,
                        &*right_val, ::core::option::Option::None);
                }
            }
        }
    };
};debug_assert_ne!(self.val_ty(val), dest_ty);
1021
1022        let trunc = self.trunc(val, dest_ty);
1023        unsafe {
1024            if llvm::LLVMIsAInstruction(trunc).is_some() {
1025                llvm::LLVMSetNSW(trunc, TRUE);
1026            }
1027        }
1028        trunc
1029    }
1030
1031    fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1032        unsafe { llvm::LLVMBuildSExt(self.llbuilder(), val, dest_ty, UNNAMED) }
1033    }
1034
1035    fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1036        self.call_intrinsic("llvm.fptoui.sat", &[dest_ty, self.val_ty(val)], &[val])
1037    }
1038
1039    fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1040        self.call_intrinsic("llvm.fptosi.sat", &[dest_ty, self.val_ty(val)], &[val])
1041    }
1042
1043    fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1044        // On WebAssembly the `fptoui` and `fptosi` instructions currently have
1045        // poor codegen. The reason for this is that the corresponding wasm
1046        // instructions, `i32.trunc_f32_s` for example, will trap when the float
1047        // is out-of-bounds, infinity, or nan. This means that LLVM
1048        // automatically inserts control flow around `fptoui` and `fptosi`
1049        // because the LLVM instruction `fptoui` is defined as producing a
1050        // poison value, not having UB on out-of-bounds values.
1051        //
1052        // This method, however, is only used with non-saturating casts that
1053        // have UB on out-of-bounds values. This means that it's ok if we use
1054        // the raw wasm instruction since out-of-bounds values can do whatever
1055        // we like. To ensure that LLVM picks the right instruction we choose
1056        // the raw wasm intrinsic functions which avoid LLVM inserting all the
1057        // other control flow automatically.
1058        if self.sess().target.is_like_wasm {
1059            let src_ty = self.cx.val_ty(val);
1060            if let Some(float_type) = self.cx.float_type(src_ty) {
1061                let int_width = self.cx.int_width(dest_ty).expect("dest was not an integer");
1062                if #[allow(non_exhaustive_omitted_patterns)] match (int_width, float_type) {
    (32 | 64, Float::F32 | Float::F64) => true,
    _ => false,
}matches!((int_width, float_type), (32 | 64, Float::F32 | Float::F64)) {
1063                    return self.call_intrinsic(
1064                        "llvm.wasm.trunc.unsigned",
1065                        &[dest_ty, src_ty],
1066                        &[val],
1067                    );
1068                }
1069            }
1070        }
1071        unsafe { llvm::LLVMBuildFPToUI(self.llbuilder(), val, dest_ty, UNNAMED) }
1072    }
1073
1074    fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1075        // see `fptoui` above for why wasm is different here
1076        if self.sess().target.is_like_wasm {
1077            let src_ty = self.cx.val_ty(val);
1078            if let Some(float_type) = self.cx.float_type(src_ty) {
1079                let int_width = self.cx.int_width(dest_ty).expect("dest ty was not an integer");
1080                if #[allow(non_exhaustive_omitted_patterns)] match (int_width, float_type) {
    (32 | 64, Float::F32 | Float::F64) => true,
    _ => false,
}matches!((int_width, float_type), (32 | 64, Float::F32 | Float::F64)) {
1081                    return self.call_intrinsic(
1082                        "llvm.wasm.trunc.signed",
1083                        &[dest_ty, src_ty],
1084                        &[val],
1085                    );
1086                }
1087            }
1088        }
1089        unsafe { llvm::LLVMBuildFPToSI(self.llbuilder(), val, dest_ty, UNNAMED) }
1090    }
1091
1092    fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1093        unsafe { llvm::LLVMBuildUIToFP(self.llbuilder(), val, dest_ty, UNNAMED) }
1094    }
1095
1096    fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1097        unsafe { llvm::LLVMBuildSIToFP(self.llbuilder(), val, dest_ty, UNNAMED) }
1098    }
1099
1100    fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1101        unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder(), val, dest_ty, UNNAMED) }
1102    }
1103
1104    fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1105        unsafe { llvm::LLVMBuildFPExt(self.llbuilder(), val, dest_ty, UNNAMED) }
1106    }
1107
1108    fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1109        unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder(), val, dest_ty, UNNAMED) }
1110    }
1111
1112    fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1113        unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder(), val, dest_ty, UNNAMED) }
1114    }
1115
1116    fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1117        unsafe { llvm::LLVMBuildBitCast(self.llbuilder(), val, dest_ty, UNNAMED) }
1118    }
1119
1120    fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
1121        unsafe {
1122            llvm::LLVMBuildIntCast2(
1123                self.llbuilder(),
1124                val,
1125                dest_ty,
1126                is_signed.to_llvm_bool(),
1127                UNNAMED,
1128            )
1129        }
1130    }
1131
1132    fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1133        unsafe { llvm::LLVMBuildPointerCast(self.llbuilder(), val, dest_ty, UNNAMED) }
1134    }
1135
1136    /* Comparisons */
1137    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1138        let op = llvm::IntPredicate::from_generic(op);
1139        unsafe { llvm::LLVMBuildICmp(self.llbuilder(), op as c_uint, lhs, rhs, UNNAMED) }
1140    }
1141
1142    fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1143        let op = llvm::RealPredicate::from_generic(op);
1144        unsafe { llvm::LLVMBuildFCmp(self.llbuilder(), op as c_uint, lhs, rhs, UNNAMED) }
1145    }
1146
1147    fn three_way_compare(
1148        &mut self,
1149        ty: Ty<'tcx>,
1150        lhs: Self::Value,
1151        rhs: Self::Value,
1152    ) -> Self::Value {
1153        let size = ty.primitive_size(self.tcx);
1154        let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };
1155
1156        self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])
1157    }
1158
1159    /* Miscellaneous instructions */
1160    fn memcpy(
1161        &mut self,
1162        dst: &'ll Value,
1163        dst_align: Align,
1164        src: &'ll Value,
1165        src_align: Align,
1166        size: &'ll Value,
1167        flags: MemFlags,
1168        tt: Option<FncTree>,
1169    ) {
1170        if !!flags.contains(MemFlags::NONTEMPORAL) {
    {
        ::core::panicking::panic_fmt(format_args!("non-temporal memcpy not supported"));
    }
};assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
1171        let size = self.intcast(size, self.type_isize(), false);
1172        let is_volatile = flags.contains(MemFlags::VOLATILE);
1173        let memcpy = unsafe {
1174            llvm::LLVMRustBuildMemCpy(
1175                self.llbuilder(),
1176                dst,
1177                dst_align.bytes() as c_uint,
1178                src,
1179                src_align.bytes() as c_uint,
1180                size,
1181                is_volatile,
1182            )
1183        };
1184
1185        // TypeTree metadata for memcpy is especially important: when Enzyme encounters
1186        // a memcpy during autodiff, it needs to know the structure of the data being
1187        // copied to properly track derivatives. For example, copying an array of floats
1188        // vs. copying a struct with mixed types requires different derivative handling.
1189        // The TypeTree tells Enzyme exactly what memory layout to expect.
1190        if let Some(tt) = tt {
1191            crate::typetree::add_tt(self, memcpy, tt);
1192        }
1193    }
1194
1195    fn memmove(
1196        &mut self,
1197        dst: &'ll Value,
1198        dst_align: Align,
1199        src: &'ll Value,
1200        src_align: Align,
1201        size: &'ll Value,
1202        flags: MemFlags,
1203    ) {
1204        if !!flags.contains(MemFlags::NONTEMPORAL) {
    {
        ::core::panicking::panic_fmt(format_args!("non-temporal memmove not supported"));
    }
};assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
1205        let size = self.intcast(size, self.type_isize(), false);
1206        let is_volatile = flags.contains(MemFlags::VOLATILE);
1207        unsafe {
1208            llvm::LLVMRustBuildMemMove(
1209                self.llbuilder(),
1210                dst,
1211                dst_align.bytes() as c_uint,
1212                src,
1213                src_align.bytes() as c_uint,
1214                size,
1215                is_volatile,
1216            );
1217        }
1218    }
1219
1220    fn memset(
1221        &mut self,
1222        ptr: &'ll Value,
1223        fill_byte: &'ll Value,
1224        size: &'ll Value,
1225        align: Align,
1226        flags: MemFlags,
1227    ) {
1228        if !!flags.contains(MemFlags::NONTEMPORAL) {
    {
        ::core::panicking::panic_fmt(format_args!("non-temporal memset not supported"));
    }
};assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memset not supported");
1229        let is_volatile = flags.contains(MemFlags::VOLATILE);
1230        unsafe {
1231            llvm::LLVMRustBuildMemSet(
1232                self.llbuilder(),
1233                ptr,
1234                align.bytes() as c_uint,
1235                fill_byte,
1236                size,
1237                is_volatile,
1238            );
1239        }
1240    }
1241
1242    fn vscale(&mut self, ty: &'ll Type) -> &'ll Value {
1243        unsafe { llvm::LLVMRustBuildVScale(self.llbuilder(), ty) }
1244    }
1245
1246    fn select(
1247        &mut self,
1248        cond: &'ll Value,
1249        then_val: &'ll Value,
1250        else_val: &'ll Value,
1251    ) -> &'ll Value {
1252        unsafe { llvm::LLVMBuildSelect(self.llbuilder(), cond, then_val, else_val, UNNAMED) }
1253    }
1254
1255    fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1256        unsafe { llvm::LLVMBuildVAArg(self.llbuilder(), list, ty, UNNAMED) }
1257    }
1258
1259    fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
1260        unsafe { llvm::LLVMBuildExtractElement(self.llbuilder(), vec, idx, UNNAMED) }
1261    }
1262
1263    fn vector_splat(&mut self, num_elts: u64, elt: &'ll Value) -> &'ll Value {
1264        unsafe {
1265            let elt_ty = self.cx.val_ty(elt);
1266            let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts));
1267            let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
1268            let vec_i32_ty = self.type_vector(self.type_i32(), num_elts);
1269            self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
1270        }
1271    }
1272
1273    fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
1274        {
    match (&(idx as c_uint as u64), &idx) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(idx as c_uint as u64, idx);
1275        unsafe { llvm::LLVMBuildExtractValue(self.llbuilder(), agg_val, idx as c_uint, UNNAMED) }
1276    }
1277
1278    fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
1279        {
    match (&(idx as c_uint as u64), &idx) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(idx as c_uint as u64, idx);
1280        unsafe {
1281            llvm::LLVMBuildInsertValue(self.llbuilder(), agg_val, elt, idx as c_uint, UNNAMED)
1282        }
1283    }
1284
1285    fn set_personality_fn(&mut self, personality: &'ll Value) {
1286        unsafe {
1287            llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1288        }
1289    }
1290
1291    fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) {
1292        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()]);
1293        let landing_pad = self.landing_pad(ty, pers_fn, 0);
1294        unsafe {
1295            llvm::LLVMSetCleanup(landing_pad, llvm::TRUE);
1296        }
1297        (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1))
1298    }
1299
1300    fn filter_landing_pad(&mut self, pers_fn: &'ll Value) {
1301        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()]);
1302        let landing_pad = self.landing_pad(ty, pers_fn, 1);
1303        self.add_clause(landing_pad, self.const_array(self.type_ptr(), &[]));
1304    }
1305
1306    fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) {
1307        let ty = self.type_struct(&[self.type_ptr(), self.type_i32()]);
1308        let mut exn = self.const_poison(ty);
1309        exn = self.insert_value(exn, exn0, 0);
1310        exn = self.insert_value(exn, exn1, 1);
1311        unsafe {
1312            llvm::LLVMBuildResume(self.llbuilder(), exn);
1313        }
1314    }
1315
1316    fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
1317        let ret = unsafe {
1318            llvm::LLVMBuildCleanupPad(
1319                self.llbuilder(),
1320                parent,
1321                args.as_ptr(),
1322                args.len() as c_uint,
1323                c"cleanuppad".as_ptr(),
1324            )
1325        };
1326        Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
1327    }
1328
1329    fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) {
1330        unsafe {
1331            llvm::LLVMBuildCleanupRet(self.llbuilder(), funclet.cleanuppad(), unwind)
1332                .expect("LLVM does not have support for cleanupret");
1333        }
1334    }
1335
1336    fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
1337        let ret = unsafe {
1338            llvm::LLVMBuildCatchPad(
1339                self.llbuilder(),
1340                parent,
1341                args.as_ptr(),
1342                args.len() as c_uint,
1343                c"catchpad".as_ptr(),
1344            )
1345        };
1346        Funclet::new(ret.expect("LLVM does not have support for catchpad"))
1347    }
1348
1349    fn catch_switch(
1350        &mut self,
1351        parent: Option<&'ll Value>,
1352        unwind: Option<&'ll BasicBlock>,
1353        handlers: &[&'ll BasicBlock],
1354    ) -> &'ll Value {
1355        let ret = unsafe {
1356            llvm::LLVMBuildCatchSwitch(
1357                self.llbuilder(),
1358                parent,
1359                unwind,
1360                handlers.len() as c_uint,
1361                c"catchswitch".as_ptr(),
1362            )
1363        };
1364        let ret = ret.expect("LLVM does not have support for catchswitch");
1365        for handler in handlers {
1366            unsafe {
1367                llvm::LLVMAddHandler(ret, handler);
1368            }
1369        }
1370        ret
1371    }
1372
1373    fn get_funclet_cleanuppad(&self, funclet: &Funclet<'ll>) -> &'ll Value {
1374        funclet.cleanuppad()
1375    }
1376
1377    // Atomic Operations
1378    fn atomic_cmpxchg(
1379        &mut self,
1380        dst: &'ll Value,
1381        cmp: &'ll Value,
1382        src: &'ll Value,
1383        order: rustc_middle::ty::AtomicOrdering,
1384        failure_order: rustc_middle::ty::AtomicOrdering,
1385        weak: bool,
1386    ) -> (&'ll Value, &'ll Value) {
1387        unsafe {
1388            let value = llvm::LLVMBuildAtomicCmpXchg(
1389                self.llbuilder(),
1390                dst,
1391                cmp,
1392                src,
1393                AtomicOrdering::from_generic(order),
1394                AtomicOrdering::from_generic(failure_order),
1395                llvm::FALSE, // SingleThreaded
1396            );
1397            llvm::LLVMSetWeak(value, weak.to_llvm_bool());
1398            let val = self.extract_value(value, 0);
1399            let success = self.extract_value(value, 1);
1400            (val, success)
1401        }
1402    }
1403
1404    fn atomic_rmw(
1405        &mut self,
1406        op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1407        dst: &'ll Value,
1408        src: &'ll Value,
1409        order: rustc_middle::ty::AtomicOrdering,
1410        ret_ptr: bool,
1411    ) -> &'ll Value {
1412        // FIXME: If `ret_ptr` is true and `src` is not a pointer, we *should* tell LLVM that the
1413        // LHS is a pointer and the operation should be provenance-preserving, but LLVM does not
1414        // currently support that (https://github.com/llvm/llvm-project/issues/120837).
1415        let mut res = unsafe {
1416            llvm::LLVMBuildAtomicRMW(
1417                self.llbuilder(),
1418                AtomicRmwBinOp::from_generic(op),
1419                dst,
1420                src,
1421                AtomicOrdering::from_generic(order),
1422                llvm::FALSE, // SingleThreaded
1423            )
1424        };
1425        if ret_ptr && self.val_ty(res) != self.type_ptr() {
1426            res = self.inttoptr(res, self.type_ptr());
1427        }
1428        res
1429    }
1430
1431    fn atomic_fence(
1432        &mut self,
1433        order: rustc_middle::ty::AtomicOrdering,
1434        scope: SynchronizationScope,
1435    ) {
1436        let single_threaded = match scope {
1437            SynchronizationScope::SingleThread => true,
1438            SynchronizationScope::CrossThread => false,
1439        };
1440        unsafe {
1441            llvm::LLVMBuildFence(
1442                self.llbuilder(),
1443                AtomicOrdering::from_generic(order),
1444                single_threaded.to_llvm_bool(),
1445                UNNAMED,
1446            );
1447        }
1448    }
1449
1450    fn set_invariant_load(&mut self, load: &'ll Value) {
1451        self.set_metadata_node(load, llvm::MD_invariant_load, &[]);
1452    }
1453
1454    fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1455        self.call_lifetime_intrinsic("llvm.lifetime.start", ptr, size);
1456    }
1457
1458    fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1459        self.call_lifetime_intrinsic("llvm.lifetime.end", ptr, size);
1460    }
1461
1462    fn call(
1463        &mut self,
1464        llty: &'ll Type,
1465        caller_attrs: Option<&CodegenFnAttrs>,
1466        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1467        llfn: &'ll Value,
1468        return_slot: ReturnSlot<&'ll Value>,
1469        args: &[&'ll Value],
1470        funclet: Option<&Funclet<'ll>>,
1471        callee_instance: Option<Instance<'tcx>>,
1472    ) -> &'ll Value {
1473        // If this function returns indirectly (`PassMode::Indirect`),
1474        // the `return_slot` should be the first argument.
1475        let args = match return_slot {
1476            ReturnSlot::Direct => args.to_vec(),
1477            ReturnSlot::Indirect(sret_ptr) => {
1478                let mut args = args.to_vec();
1479                args.insert(0, sret_ptr);
1480                args
1481            }
1482        };
1483        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:1483",
                        "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                        ::tracing_core::__macro_support::Option::Some(1483u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("call {0:?} with args ({1:?})",
                                                    llfn, args) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("call {:?} with args ({:?})", llfn, args);
1484        let args = self.check_call("call", llty, llfn, &args);
1485        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1486        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1487        if let Some(funclet_bundle) = funclet_bundle {
1488            bundles.push(funclet_bundle);
1489        }
1490
1491        // Emit CFI pointer type membership test
1492        self.cfi_type_test(caller_attrs, fn_abi, callee_instance, llfn);
1493
1494        // Emit KCFI operand bundle
1495        let kcfi_bundle = self.kcfi_operand_bundle(caller_attrs, fn_abi, callee_instance, llfn);
1496        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|b| b.as_ref()) {
1497            bundles.push(kcfi_bundle);
1498        }
1499
1500        let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
1501        if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
1502            bundles.push(p);
1503        }
1504
1505        let call = unsafe {
1506            llvm::LLVMBuildCallWithOperandBundles(
1507                self.llbuilder(),
1508                llty,
1509                llfn,
1510                args.as_ptr() as *const &llvm::Value,
1511                args.len() as c_uint,
1512                bundles.as_ptr(),
1513                bundles.len() as c_uint,
1514                c"".as_ptr(),
1515            )
1516        };
1517
1518        if let Some(fn_abi) = fn_abi {
1519            fn_abi.apply_attrs_callsite(self, call);
1520        }
1521        call
1522    }
1523
1524    fn tail_call(
1525        &mut self,
1526        llty: Self::Type,
1527        caller_attrs: Option<&CodegenFnAttrs>,
1528        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
1529        llfn: Self::Value,
1530        return_slot: ReturnSlot<Self::Value>,
1531        args: &[Self::Value],
1532        funclet: Option<&Self::Funclet>,
1533        callee_instance: Option<Instance<'tcx>>,
1534    ) {
1535        let call = self.call(
1536            llty,
1537            caller_attrs,
1538            Some(fn_abi),
1539            llfn,
1540            return_slot,
1541            args,
1542            funclet,
1543            callee_instance,
1544        );
1545        llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail);
1546
1547        match &fn_abi.ret.mode {
1548            PassMode::Ignore | PassMode::Indirect { .. } => self.ret_void(),
1549            PassMode::Direct(_) | PassMode::Pair { .. } | PassMode::Cast { .. } => self.ret(call),
1550            PassMode::IndirectUnsized { .. } => ::rustc_span::macros::bug_impl(None,
    format_args!("unsized returns are not supported"), Location::caller())bug!("unsized returns are not supported"),
1551        }
1552    }
1553
1554    fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1555        unsafe { llvm::LLVMBuildZExt(self.llbuilder(), val, dest_ty, UNNAMED) }
1556    }
1557
1558    fn apply_attrs_to_cleanup_callsite(&mut self, llret: &'ll Value) {
1559        // Cleanup is always the cold path.
1560        let cold_inline = llvm::AttributeKind::Cold.create_attr(self.llcx);
1561        attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[cold_inline]);
1562    }
1563}
1564
1565impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
1566    fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1567        // Forward to the `get_static` method of `CodegenCx`
1568        let global = self.cx().get_static(def_id);
1569        if self.cx().tcx.is_thread_local_static(def_id) {
1570            let pointer =
1571                self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
1572            // Cast to default address space if globals are in a different addrspace
1573            self.pointercast(pointer, self.type_ptr())
1574        } else {
1575            // Cast to default address space if globals are in a different addrspace
1576            self.cx().const_pointercast(global, self.type_ptr())
1577        }
1578    }
1579}
1580
1581impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1582    pub(crate) fn llfn(&self) -> &'ll Value {
1583        unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1584    }
1585
1586    fn generate_ubsan_cfi_diag_data(
1587        &mut self,
1588        span: rustc_span::Span,
1589        expected_ty: String,
1590        check_kind: u8,
1591    ) -> &'ll Value {
1592        let cx = self.cx();
1593        let tcx = cx.tcx;
1594
1595        let loc = tcx.sess.source_map().lookup_char_pos(span.lo());
1596
1597        let filename_str = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}\0",
                loc.file.name.prefer_local_unconditionally()))
    })format!("{}\0", loc.file.name.prefer_local_unconditionally());
1598        let filename_val = cx.const_bytes(filename_str.as_bytes());
1599        let filename_ptr = cx.static_addr_of_impl(filename_val, Align::ONE, None);
1600
1601        // SourceLocation UBSan struct: { const char *filename, uint32_t line, uint32_t column }
1602        let source_location = cx.const_struct(
1603            &[
1604                filename_ptr,
1605                cx.const_u32(loc.line as u32),
1606                // UBSan columns are 1-based
1607                cx.const_u32(loc.col.0 as u32 + 1),
1608            ],
1609            false, // packed = false
1610        );
1611
1612        let ty_name = ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("{0}\0", expected_ty))
    })format!("{}\0", expected_ty);
1613        let ty_name_val = cx.const_bytes(ty_name.as_bytes());
1614
1615        // TypeDescriptor UBSan struct: { uint16_t TypeKind, uint16_t TypeInfo, const char *TypeName }
1616        let type_descriptor =
1617            cx.const_struct(&[cx.const_i16(0xffffu16 as i16), cx.const_i16(0), ty_name_val], false);
1618
1619        let type_descriptor_ptr =
1620            cx.static_addr_of_impl(type_descriptor, Align::from_bytes(2).unwrap(), None);
1621
1622        // CFICheckFailData UBSan struct: { uint8_t CheckKind, SourceLocation Loc, TypeDescriptor *Type }
1623        let cfi_check_fail_data = cx
1624            .const_struct(&[cx.const_u8(check_kind), source_location, type_descriptor_ptr], false);
1625        let align = tcx.data_layout.aggregate_align;
1626
1627        // Returns the final opaque pointer to the struct to be passed to __ubsan_handle_cfi_check_fail
1628        cx.static_addr_of_mut(cfi_check_fail_data, align, Some("__ubsan_cfi_check_fail_data"))
1629    }
1630}
1631
1632impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1633    fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1634        unsafe {
1635            llvm::LLVMRustPositionBuilderAtStart(self.llbuilder(), llbb);
1636        }
1637    }
1638}
1639impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1640    fn align_metadata(&mut self, load: &'ll Value, align: Align) {
1641        let md = [llvm::LLVMValueAsMetadata(self.cx.const_u64(align.bytes()))];
1642        self.set_metadata_node(load, llvm::MD_align, &md);
1643    }
1644
1645    fn noundef_metadata(&mut self, load: &'ll Value) {
1646        self.set_metadata_node(load, llvm::MD_noundef, &[]);
1647    }
1648
1649    pub(crate) fn set_unpredictable(&mut self, inst: &'ll Value) {
1650        self.set_metadata_node(inst, llvm::MD_unpredictable, &[]);
1651    }
1652
1653    fn write_operand_repeatedly_optimized(
1654        &mut self,
1655        cg_elem: OperandRef<'tcx, &'ll Value>,
1656        count: u64,
1657        dest: PlaceRef<'tcx, &'ll Value>,
1658    ) {
1659        let zero = self.const_usize(0);
1660        let count = self.const_usize(count);
1661
1662        let header_bb = self.append_sibling_block("repeat_loop_header");
1663        let body_bb = self.append_sibling_block("repeat_loop_body");
1664        let next_bb = self.append_sibling_block("repeat_loop_next");
1665
1666        self.br(header_bb);
1667
1668        let mut header_bx = Self::build(self.cx, header_bb);
1669        let i = header_bx.phi(self.val_ty(zero), &[zero], &[self.llbb()]);
1670
1671        let keep_going = header_bx.icmp(IntPredicate::IntULT, i, count);
1672        header_bx.cond_br(keep_going, body_bb, next_bb);
1673
1674        let mut body_bx = Self::build(self.cx, body_bb);
1675        let dest_elem = dest.project_index(&mut body_bx, i);
1676        cg_elem.val.store(&mut body_bx, dest_elem);
1677
1678        let next = body_bx.unchecked_uadd(i, self.const_usize(1));
1679        body_bx.br(header_bb);
1680        header_bx.add_incoming_to_phi(i, next, body_bb);
1681
1682        *self = Self::build(self.cx, next_bb);
1683    }
1684
1685    fn write_operand_repeatedly_unoptimized(
1686        &mut self,
1687        cg_elem: OperandRef<'tcx, &'ll Value>,
1688        count: u64,
1689        dest: PlaceRef<'tcx, &'ll Value>,
1690    ) {
1691        let zero = self.const_usize(0);
1692        let count = self.const_usize(count);
1693        let start = dest.project_index(self, zero).val.llval;
1694        let end = dest.project_index(self, count).val.llval;
1695
1696        let header_bb = self.append_sibling_block("repeat_loop_header");
1697        let body_bb = self.append_sibling_block("repeat_loop_body");
1698        let next_bb = self.append_sibling_block("repeat_loop_next");
1699
1700        self.br(header_bb);
1701
1702        let mut header_bx = Self::build(self.cx, header_bb);
1703        let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]);
1704
1705        let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end);
1706        header_bx.cond_br(keep_going, body_bb, next_bb);
1707
1708        let mut body_bx = Self::build(self.cx, body_bb);
1709        let align = dest.val.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
1710        cg_elem
1711            .val
1712            .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align));
1713
1714        let next = body_bx.inbounds_ptradd(current, self.const_usize(cg_elem.layout.size.bytes()));
1715        body_bx.br(header_bb);
1716        header_bx.add_incoming_to_phi(current, next, body_bb);
1717
1718        *self = Self::build(self.cx, next_bb);
1719    }
1720
1721    pub(crate) fn minimum_number_nsz(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1722        let call = self.call_intrinsic("llvm.minimumnum", &[self.val_ty(lhs)], &[lhs, rhs]);
1723        unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
1724        call
1725    }
1726
1727    pub(crate) fn maximum_number_nsz(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1728        let call = self.call_intrinsic("llvm.maximumnum", &[self.val_ty(lhs)], &[lhs, rhs]);
1729        unsafe { llvm::LLVMRustSetNoSignedZeros(call) };
1730        call
1731    }
1732
1733    pub(crate) fn insert_element(
1734        &mut self,
1735        vec: &'ll Value,
1736        elt: &'ll Value,
1737        idx: &'ll Value,
1738    ) -> &'ll Value {
1739        unsafe { llvm::LLVMBuildInsertElement(self.llbuilder(), vec, elt, idx, UNNAMED) }
1740    }
1741
1742    pub(crate) fn shuffle_vector(
1743        &mut self,
1744        v1: &'ll Value,
1745        v2: &'ll Value,
1746        mask: &'ll Value,
1747    ) -> &'ll Value {
1748        unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder(), v1, v2, mask, UNNAMED) }
1749    }
1750
1751    pub(crate) fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1752        self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src])
1753    }
1754    pub(crate) fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1755        self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src])
1756    }
1757    pub(crate) fn vector_reduce_fadd_reassoc(
1758        &mut self,
1759        acc: &'ll Value,
1760        src: &'ll Value,
1761    ) -> &'ll Value {
1762        unsafe {
1763            let instr =
1764                self.call_intrinsic("llvm.vector.reduce.fadd", &[self.val_ty(src)], &[acc, src]);
1765            llvm::LLVMRustSetAllowReassoc(instr);
1766            instr
1767        }
1768    }
1769    pub(crate) fn vector_reduce_fmul_reassoc(
1770        &mut self,
1771        acc: &'ll Value,
1772        src: &'ll Value,
1773    ) -> &'ll Value {
1774        unsafe {
1775            let instr =
1776                self.call_intrinsic("llvm.vector.reduce.fmul", &[self.val_ty(src)], &[acc, src]);
1777            llvm::LLVMRustSetAllowReassoc(instr);
1778            instr
1779        }
1780    }
1781    pub(crate) fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1782        self.call_intrinsic("llvm.vector.reduce.add", &[self.val_ty(src)], &[src])
1783    }
1784    pub(crate) fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1785        self.call_intrinsic("llvm.vector.reduce.mul", &[self.val_ty(src)], &[src])
1786    }
1787    pub(crate) fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1788        self.call_intrinsic("llvm.vector.reduce.and", &[self.val_ty(src)], &[src])
1789    }
1790    pub(crate) fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1791        self.call_intrinsic("llvm.vector.reduce.or", &[self.val_ty(src)], &[src])
1792    }
1793    pub(crate) fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1794        self.call_intrinsic("llvm.vector.reduce.xor", &[self.val_ty(src)], &[src])
1795    }
1796    pub(crate) fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1797        self.call_intrinsic(
1798            if is_signed { "llvm.vector.reduce.smin" } else { "llvm.vector.reduce.umin" },
1799            &[self.val_ty(src)],
1800            &[src],
1801        )
1802    }
1803    pub(crate) fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1804        self.call_intrinsic(
1805            if is_signed { "llvm.vector.reduce.smax" } else { "llvm.vector.reduce.umax" },
1806            &[self.val_ty(src)],
1807            &[src],
1808        )
1809    }
1810}
1811impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1812    pub(crate) fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1813        unsafe {
1814            llvm::LLVMAddClause(landing_pad, clause);
1815        }
1816    }
1817
1818    pub(crate) fn catch_ret(
1819        &mut self,
1820        funclet: &Funclet<'ll>,
1821        unwind: &'ll BasicBlock,
1822    ) -> &'ll Value {
1823        let ret =
1824            unsafe { llvm::LLVMBuildCatchRet(self.llbuilder(), funclet.cleanuppad(), unwind) };
1825        ret.expect("LLVM does not have support for catchret")
1826    }
1827
1828    pub(crate) fn check_call<'b>(
1829        &mut self,
1830        typ: &str,
1831        fn_ty: &'ll Type,
1832        llfn: &'ll Value,
1833        args: &'b [&'ll Value],
1834    ) -> Cow<'b, [&'ll Value]> {
1835        if !(self.cx.type_kind(fn_ty) == TypeKind::Function) {
    {
        ::core::panicking::panic_fmt(format_args!("builder::{0} not passed a function, but {1:?}",
                typ, fn_ty));
    }
};assert!(
1836            self.cx.type_kind(fn_ty) == TypeKind::Function,
1837            "builder::{typ} not passed a function, but {fn_ty:?}"
1838        );
1839
1840        let param_tys = self.cx.func_params_types(fn_ty);
1841
1842        let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.cx.val_ty(v)))
1843            .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1844
1845        if all_args_match {
1846            return Cow::Borrowed(args);
1847        }
1848
1849        let casted_args: Vec<_> = iter::zip(param_tys, args)
1850            .enumerate()
1851            .map(|(i, (expected_ty, &actual_val))| {
1852                let actual_ty = self.cx.val_ty(actual_val);
1853                if expected_ty != actual_ty {
1854                    {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:1854",
                        "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                        ::tracing_core::__macro_support::Option::Some(1854u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("type mismatch in function call of {0:?}. Expected {1:?} for param {2}, got {3:?}; injecting bitcast",
                                                    llfn, expected_ty, i, actual_ty) as
                                            &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!(
1855                        "type mismatch in function call of {:?}. \
1856                            Expected {:?} for param {}, got {:?}; injecting bitcast",
1857                        llfn, expected_ty, i, actual_ty
1858                    );
1859                    self.bitcast(actual_val, expected_ty)
1860                } else {
1861                    actual_val
1862                }
1863            })
1864            .collect();
1865
1866        Cow::Owned(casted_args)
1867    }
1868
1869    pub(crate) fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1870        unsafe { llvm::LLVMBuildVAArg(self.llbuilder(), list, ty, UNNAMED) }
1871    }
1872}
1873
1874impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1875    pub(crate) fn call_intrinsic(
1876        &mut self,
1877        base_name: impl Into<Cow<'static, str>>,
1878        type_params: &[&'ll Type],
1879        args: &[&'ll Value],
1880    ) -> &'ll Value {
1881        let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
1882        // No LLVM intrinsic returns its data indirectly (via `sret`).
1883        self.call(ty, None, None, f, ReturnSlot::Direct, args, None, None)
1884    }
1885
1886    fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
1887        let size = size.bytes();
1888        if size == 0 {
1889            return;
1890        }
1891
1892        if !self.cx().sess().emit_lifetime_markers() {
1893            return;
1894        }
1895
1896        // LLVM 22 requires the lifetime intrinsic to act directly on the alloca,
1897        // there can't be an addrspacecast in between.
1898        let ptr = unsafe { llvm::LLVMRustStripPointerCasts(ptr) };
1899        self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]);
1900    }
1901}
1902impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
1903    pub(crate) fn phi(
1904        &mut self,
1905        ty: &'ll Type,
1906        vals: &[&'ll Value],
1907        bbs: &[&'ll BasicBlock],
1908    ) -> &'ll Value {
1909        {
    match (&vals.len(), &bbs.len()) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(vals.len(), bbs.len());
1910        let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder(), ty, UNNAMED) };
1911        unsafe {
1912            llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1913            phi
1914        }
1915    }
1916
1917    fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1918        unsafe {
1919            llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1920        }
1921    }
1922}
1923impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
1924    pub(crate) fn landing_pad(
1925        &mut self,
1926        ty: &'ll Type,
1927        pers_fn: &'ll Value,
1928        num_clauses: usize,
1929    ) -> &'ll Value {
1930        // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
1931        // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
1932        // personality lives on the parent function anyway.
1933        self.set_personality_fn(pers_fn);
1934        unsafe {
1935            llvm::LLVMBuildLandingPad(self.llbuilder(), ty, None, num_clauses as c_uint, UNNAMED)
1936        }
1937    }
1938
1939    pub(crate) fn callbr(
1940        &mut self,
1941        llty: &'ll Type,
1942        fn_attrs: Option<&CodegenFnAttrs>,
1943        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
1944        llfn: &'ll Value,
1945        args: &[&'ll Value],
1946        default_dest: &'ll BasicBlock,
1947        indirect_dest: &[&'ll BasicBlock],
1948        funclet: Option<&Funclet<'ll>>,
1949        instance: Option<Instance<'tcx>>,
1950    ) -> &'ll Value {
1951        {
    use ::tracing::__macro_support::Callsite as _;
    static __CALLSITE: ::tracing::callsite::DefaultCallsite =
        {
            static META: ::tracing::Metadata<'static> =
                {
                    ::tracing_core::metadata::Metadata::new("event /rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs:1951",
                        "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                        ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                        ::tracing_core::__macro_support::Option::Some(1951u32),
                        ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                        ::tracing_core::field::FieldSet::new(&["message"],
                            ::tracing_core::callsite::Identifier(&__CALLSITE)),
                        ::tracing::metadata::Kind::EVENT)
                };
            ::tracing::callsite::DefaultCallsite::new(&META)
        };
    let enabled =
        ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() &&
            {
                let interest = __CALLSITE.interest();
                !interest.is_never() &&
                    ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                        interest)
            };
    if enabled {
        (|value_set: ::tracing::field::ValueSet|
                    {
                        let meta = __CALLSITE.metadata();
                        ::tracing::Event::dispatch(meta, &value_set);
                        ;
                    })({
                #[allow(unused_imports)]
                use ::tracing::field::{debug, display, Value};
                __CALLSITE.metadata().fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&format_args!("invoke {0:?} with args ({1:?})",
                                                    llfn, args) as &dyn ::tracing::field::Value))])
            });
    } else { ; }
};debug!("invoke {:?} with args ({:?})", llfn, args);
1952
1953        let args = self.check_call("callbr", llty, llfn, args);
1954        let funclet_bundle = funclet.map(|funclet| funclet.bundle());
1955        let mut bundles: SmallVec<[_; 2]> = SmallVec::new();
1956        if let Some(funclet_bundle) = funclet_bundle {
1957            bundles.push(funclet_bundle);
1958        }
1959
1960        // Emit CFI pointer type membership test
1961        self.cfi_type_test(fn_attrs, fn_abi, instance, llfn);
1962
1963        // Emit KCFI operand bundle
1964        let kcfi_bundle = self.kcfi_operand_bundle(fn_attrs, fn_abi, instance, llfn);
1965        if let Some(kcfi_bundle) = kcfi_bundle.as_ref().map(|bundle| bundle.as_ref()) {
1966            bundles.push(kcfi_bundle);
1967        }
1968
1969        let pauth = self.ptrauth_operand_bundle(llfn, fn_abi);
1970        if let Some(p) = pauth.as_ref().map(|b| b.as_ref()) {
1971            bundles.push(p);
1972        }
1973
1974        let callbr = unsafe {
1975            llvm::LLVMBuildCallBr(
1976                self.llbuilder(),
1977                llty,
1978                llfn,
1979                default_dest,
1980                indirect_dest.as_ptr(),
1981                indirect_dest.len() as c_uint,
1982                args.as_ptr(),
1983                args.len() as c_uint,
1984                bundles.as_ptr(),
1985                bundles.len() as c_uint,
1986                UNNAMED,
1987            )
1988        };
1989        if let Some(fn_abi) = fn_abi {
1990            fn_abi.apply_attrs_callsite(self, callbr);
1991        }
1992        callbr
1993    }
1994
1995    // Emits CFI pointer type membership tests.
1996    fn cfi_type_test(
1997        &mut self,
1998        fn_attrs: Option<&CodegenFnAttrs>,
1999        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
2000        instance: Option<Instance<'tcx>>,
2001        llfn: &'ll Value,
2002    ) {
2003        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
2004        if self.tcx.sess.is_sanitizer_cfi_enabled()
2005            && let Some(fn_abi) = fn_abi
2006            && is_indirect_call
2007        {
2008            if let Some(fn_attrs) = fn_attrs
2009                && fn_attrs.sanitizers.disabled.contains(SanitizerSet::CFI)
2010            {
2011                return;
2012            }
2013            if crate::llvm::HasStringAttribute(self.llfn(), "no-sanitize-cfi") {
2014                return;
2015            }
2016
2017            let mut options = cfi::TypeIdOptions::empty();
2018            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
2019                options.insert(cfi::TypeIdOptions::GENERALIZE_POINTERS);
2020            }
2021            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
2022                options.insert(cfi::TypeIdOptions::NORMALIZE_INTEGERS);
2023            }
2024
2025            if self.cx.is_sanitizer_type_ignored(c"cfi", fn_abi) {
2026                return;
2027            }
2028
2029            let typeid = if let Some(instance) = instance {
2030                cfi::typeid_for_instance(self.tcx, instance, options)
2031            } else {
2032                cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
2033            };
2034            let typeid_metadata = self.cx.create_metadata(typeid.as_bytes());
2035            let dbg_loc = self.get_dbg_loc();
2036
2037            // Test whether the function pointer is associated with the type identifier using the
2038            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
2039            // calls to this intrinsic with code to test type membership.
2040            let typeid = self.get_metadata_value(typeid_metadata);
2041            let cond = self.call_intrinsic("llvm.type.test", &[], &[llfn, typeid]);
2042            let bb_pass = self.append_sibling_block("type_test.pass");
2043            let bb_fail = self.append_sibling_block("type_test.fail");
2044            self.cond_br(cond, bb_pass, bb_fail);
2045
2046            self.switch_to_block(bb_fail);
2047            if let Some(dbg_loc) = dbg_loc {
2048                self.set_dbg_loc(dbg_loc);
2049            }
2050
2051            let is_diag = self.tcx.sess.opts.unstable_opts.sanitizer_cfi_diag.unwrap_or(false);
2052            let is_recover =
2053                self.tcx.sess.opts.unstable_opts.sanitizer_cfi_recover.unwrap_or(false);
2054            let is_minimal =
2055                self.tcx.sess.opts.unstable_opts.sanitizer_cfi_minimal_runtime.unwrap_or(false);
2056
2057            if is_diag || is_recover {
2058                let fty = if is_minimal {
2059                    self.cx.type_func(&[], self.cx.type_void())
2060                } else {
2061                    self.cx.type_func(
2062                        &[self.cx.type_ptr(), self.cx.type_isize(), self.cx.type_isize()],
2063                        self.cx.type_void(),
2064                    )
2065                };
2066                let ubsan_handler = self.declare_cfn(
2067                    match (is_minimal, is_recover) {
2068                        (true, true) => "__ubsan_handle_cfi_check_fail_minimal",
2069                        (true, false) => "__ubsan_handle_cfi_check_fail_minimal_abort",
2070                        (false, true) => "__ubsan_handle_cfi_check_fail",
2071                        (false, false) => "__ubsan_handle_cfi_check_fail_abort",
2072                    },
2073                    llvm::UnnamedAddr::Global,
2074                    fty,
2075                );
2076
2077                let mut expected_ty = String::from("fn(");
2078                {
    let _guard = NoTrimmedGuard::new();
    {
        for (i, arg) in fn_abi.args.iter().enumerate() {
            if i > 0 { expected_ty.push_str(", "); }
            use std::fmt::Write;
            (&mut expected_ty).write_fmt(format_args!("{0}",
                        arg.layout.ty)).unwrap();
        }
        expected_ty.push(')');
        if !fn_abi.ret.layout.ty.is_unit() {
            use std::fmt::Write;
            (&mut expected_ty).write_fmt(format_args!(" -> {0}",
                        fn_abi.ret.layout.ty)).unwrap();
        }
    }
};ty::print::with_no_trimmed_paths!({
2079                    for (i, arg) in fn_abi.args.iter().enumerate() {
2080                        if i > 0 {
2081                            expected_ty.push_str(", ");
2082                        }
2083                        use std::fmt::Write;
2084                        write!(&mut expected_ty, "{}", arg.layout.ty).unwrap();
2085                    }
2086                    expected_ty.push(')');
2087                    if !fn_abi.ret.layout.ty.is_unit() {
2088                        use std::fmt::Write;
2089                        write!(&mut expected_ty, " -> {}", fn_abi.ret.layout.ty).unwrap();
2090                    }
2091                });
2092
2093                // 4 for cfi-icall (indirect call)
2094                let check_kind = 4;
2095                let diag_data =
2096                    self.generate_ubsan_cfi_diag_data(self.span, expected_ty, check_kind);
2097
2098                let function_address = self.ptrtoint(llfn, self.cx.type_isize());
2099                let arguments: &[_] = if is_minimal {
2100                    &[]
2101                } else {
2102                    &[diag_data, function_address, self.const_usize(0)]
2103                };
2104                self.call(
2105                    fty,
2106                    None,
2107                    None,
2108                    ubsan_handler,
2109                    ReturnSlot::Direct,
2110                    arguments,
2111                    None,
2112                    None,
2113                );
2114                if is_recover {
2115                    self.br(bb_pass);
2116                } else {
2117                    self.unreachable();
2118                }
2119            } else {
2120                self.abort_immediate();
2121                self.unreachable();
2122            }
2123
2124            self.switch_to_block(bb_pass);
2125            if let Some(dbg_loc) = dbg_loc {
2126                self.set_dbg_loc(dbg_loc);
2127            }
2128        }
2129    }
2130
2131    // Emits KCFI operand bundles.
2132    fn kcfi_operand_bundle(
2133        &mut self,
2134        fn_attrs: Option<&CodegenFnAttrs>,
2135        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
2136        instance: Option<Instance<'tcx>>,
2137        llfn: &'ll Value,
2138    ) -> Option<llvm::OperandBundleBox<'ll>> {
2139        let is_indirect_call = unsafe { llvm::LLVMRustIsNonGVFunctionPointerTy(llfn) };
2140        let kcfi_bundle = if self.tcx.sess.is_sanitizer_kcfi_enabled()
2141            && let Some(fn_abi) = fn_abi
2142            && is_indirect_call
2143        {
2144            if let Some(fn_attrs) = fn_attrs
2145                && fn_attrs.sanitizers.disabled.contains(SanitizerSet::KCFI)
2146            {
2147                return None;
2148            }
2149            if crate::llvm::HasStringAttribute(self.llfn(), "no-sanitize-kcfi") {
2150                return None;
2151            }
2152
2153            let mut options = kcfi::TypeIdOptions::empty();
2154            if self.tcx.sess.is_sanitizer_cfi_generalize_pointers_enabled() {
2155                options.insert(kcfi::TypeIdOptions::GENERALIZE_POINTERS);
2156            }
2157            if self.tcx.sess.is_sanitizer_cfi_normalize_integers_enabled() {
2158                options.insert(kcfi::TypeIdOptions::NORMALIZE_INTEGERS);
2159            }
2160
2161            if self.cx.is_sanitizer_type_ignored(c"kcfi", fn_abi) {
2162                return None;
2163            }
2164
2165            let kcfi_typeid = if let Some(instance) = instance {
2166                kcfi::typeid_for_instance(self.tcx, instance, options)
2167            } else {
2168                kcfi::typeid_for_fnabi(self.tcx, fn_abi, options)
2169            };
2170
2171            Some(llvm::OperandBundleBox::new("kcfi", &[self.const_u32(kcfi_typeid)]))
2172        } else {
2173            None
2174        };
2175        kcfi_bundle
2176    }
2177
2178    // Emits pauth operand bundle.
2179    fn ptrauth_operand_bundle(
2180        &mut self,
2181        llfn: &'ll Value,
2182        fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>,
2183    ) -> Option<llvm::OperandBundleBox<'ll>> {
2184        let schema = self.sess().pointer_authentication_functions()?;
2185
2186        // Pointer authentication support is currently limited to extern "C" calls; filter out other
2187        // ABIs.
2188        if fn_abi?.conv != CanonAbi::C {
2189            return None;
2190        }
2191        // Filter out LLVM intrinsics.
2192        if llvm::get_value_name(llfn).starts_with(b"llvm.") {
2193            return None;
2194        }
2195
2196        // FIXME(jchlanda) Operand bundles should only be attached to indirect function calls.
2197        // However, function pointer signing is currently performed in `get_fn_addr`, which causes
2198        // the logic to be applied too broadly, including to function values (not just pointers).
2199        // As a result, direct calls using signed function values must also receive operand
2200        // bundles.
2201        // Once this is resolved, we should analyze each call and skip direct calls. See the
2202        // discussion in the rust-lang issue: <https://github.com/rust-lang/rust/issues/152532>
2203        Some(llvm::OperandBundleBox::new(
2204            "ptrauth",
2205            &[
2206                self.const_u32(schema.key as u32),
2207                self.const_u64(schema.constant_discriminator as u64),
2208            ],
2209        ))
2210    }
2211
2212    /// Emits a call to `llvm.instrprof.increment`. Used by coverage instrumentation.
2213    {}
#[allow(clippy :: suspicious_else_formatting)]
{
    let __tracing_attr_span;
    let __tracing_attr_guard;
    if ::tracing::Level::DEBUG <= ::tracing::level_filters::STATIC_MAX_LEVEL
                &&
                ::tracing::Level::DEBUG <=
                    ::tracing::level_filters::LevelFilter::current() ||
            { false } {
        __tracing_attr_span =
            {
                use ::tracing::__macro_support::Callsite as _;
                static __CALLSITE: ::tracing::callsite::DefaultCallsite =
                    {
                        static META: ::tracing::Metadata<'static> =
                            {
                                ::tracing_core::metadata::Metadata::new("instrprof_increment",
                                    "rustc_codegen_llvm::builder", ::tracing::Level::DEBUG,
                                    ::tracing_core::__macro_support::Option::Some("/rustc-dev/63931fb6b6bc05c38a5111ed7b32fe7648d22b1c/compiler/rustc_codegen_llvm/src/builder.rs"),
                                    ::tracing_core::__macro_support::Option::Some(2213u32),
                                    ::tracing_core::__macro_support::Option::Some("rustc_codegen_llvm::builder"),
                                    ::tracing_core::field::FieldSet::new(&[{
                                                        const NAME:
                                                            ::tracing::__macro_support::FieldName<{
                                                                ::tracing::__macro_support::FieldName::len("fn_name")
                                                            }> =
                                                            ::tracing::__macro_support::FieldName::new("fn_name");
                                                        NAME.as_str()
                                                    },
                                                    {
                                                        const NAME:
                                                            ::tracing::__macro_support::FieldName<{
                                                                ::tracing::__macro_support::FieldName::len("hash")
                                                            }> =
                                                            ::tracing::__macro_support::FieldName::new("hash");
                                                        NAME.as_str()
                                                    },
                                                    {
                                                        const NAME:
                                                            ::tracing::__macro_support::FieldName<{
                                                                ::tracing::__macro_support::FieldName::len("num_counters")
                                                            }> =
                                                            ::tracing::__macro_support::FieldName::new("num_counters");
                                                        NAME.as_str()
                                                    },
                                                    {
                                                        const NAME:
                                                            ::tracing::__macro_support::FieldName<{
                                                                ::tracing::__macro_support::FieldName::len("index")
                                                            }> =
                                                            ::tracing::__macro_support::FieldName::new("index");
                                                        NAME.as_str()
                                                    }], ::tracing_core::callsite::Identifier(&__CALLSITE)),
                                    ::tracing::metadata::Kind::SPAN)
                            };
                        ::tracing::callsite::DefaultCallsite::new(&META)
                    };
                let mut interest = ::tracing::subscriber::Interest::never();
                if ::tracing::Level::DEBUG <=
                                    ::tracing::level_filters::STATIC_MAX_LEVEL &&
                                ::tracing::Level::DEBUG <=
                                    ::tracing::level_filters::LevelFilter::current() &&
                            { interest = __CALLSITE.interest(); !interest.is_never() }
                        &&
                        ::tracing::__macro_support::__is_enabled(__CALLSITE.metadata(),
                            interest) {
                    let meta = __CALLSITE.metadata();
                    ::tracing::Span::new(meta,
                        &{
                                #[allow(unused_imports)]
                                use ::tracing::field::{debug, display, Value};
                                meta.fields().value_set_all(&[(::tracing::__macro_support::Option::Some(&::tracing::field::debug(&fn_name)
                                                            as &dyn ::tracing::field::Value)),
                                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&hash)
                                                            as &dyn ::tracing::field::Value)),
                                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&num_counters)
                                                            as &dyn ::tracing::field::Value)),
                                                (::tracing::__macro_support::Option::Some(&::tracing::field::debug(&index)
                                                            as &dyn ::tracing::field::Value))])
                            })
                } else {
                    let span =
                        ::tracing::__macro_support::__disabled_span(__CALLSITE.metadata());
                    {};
                    span
                }
            };
        __tracing_attr_guard = __tracing_attr_span.enter();
    }

    #[warn(clippy :: suspicious_else_formatting)]
    {

        #[allow(unknown_lints, unreachable_code, clippy ::
        diverging_sub_expression, clippy :: empty_loop, clippy ::
        let_unit_value, clippy :: let_with_type_underscore, clippy ::
        needless_return, clippy :: unreachable)]
        if false {
            let __tracing_attr_fake_return: () = loop {};
            return __tracing_attr_fake_return;
        }
        {
            self.call_intrinsic("llvm.instrprof.increment", &[],
                &[fn_name, hash, num_counters, index]);
        }
    }
}#[instrument(level = "debug", skip(self))]
2214    pub(crate) fn instrprof_increment(
2215        &mut self,
2216        fn_name: &'ll Value,
2217        hash: &'ll Value,
2218        num_counters: &'ll Value,
2219        index: &'ll Value,
2220    ) {
2221        self.call_intrinsic("llvm.instrprof.increment", &[], &[fn_name, hash, num_counters, index]);
2222    }
2223}