rustc_codegen_llvm/

consts.rs

use std::ops::Range;

use rustc_abi::{Align, ExternAbi, HasDataLayout, Primitive, Scalar, Size, WrappingRange};
use rustc_attr_ir::Linkage;
use rustc_attr_ir::lang_items::LangItem;
use rustc_codegen_ssa::common;
use rustc_codegen_ssa::traits::*;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::{DefId, LOCAL_CRATE};
use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
use rustc_middle::mir::interpret::{
    Allocation, ConstAllocation, ErrorHandled, InitChunk, Pointer, Scalar as InterpScalar,
    read_target_uint,
};
use rustc_middle::mono::MonoItem;
use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
use rustc_middle::ty::{self, Instance};
use rustc_span::{Symbol, bug, span_bug};
use rustc_target::spec::Arch;
use tracing::{debug, instrument, trace};

use crate::common::CodegenCx;
use crate::diagnostics::SymbolAlreadyDefined;
use crate::llvm::{self, Type, Value, const_ptr_auth};
use crate::type_of::LayoutLlvmExt;
use crate::{base, debuginfo};

/// Indicates whether a value originates from a `static`.
pub(crate) enum IsStatic {
    Yes,
    No,
}
/// Indicates whether a symbol is part of `.init_array` or `.fini_array`.
#[derive(PartialEq)]
pub(crate) enum IsInitOrFini {
    Yes,
    No,
}
pub(crate) fn const_alloc_to_llvm<'ll>(
    cx: &CodegenCx<'ll, '_>,
    alloc: &Allocation,
    is_static: IsStatic,
    is_init_fini: IsInitOrFini,
) -> &'ll Value {
    // We expect that callers of const_alloc_to_llvm will instead directly codegen a pointer or
    // integer for any &ZST where the ZST is a constant (i.e. not a static). We should never be
    // producing empty LLVM allocations as they're just adding noise to binaries and forcing less
    // optimal codegen.
    //
    // Statics have a guaranteed meaningful address so it's less clear that we want to do
    // something like this; it's also harder.
    if matches!(is_static, IsStatic::No) {
        assert!(alloc.len() != 0);
    }
    let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1);
    let dl = cx.data_layout();
    let pointer_size = dl.pointer_size();
    let pointer_size_bytes = pointer_size.bytes() as usize;

    // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, so `range`
    // must be within the bounds of `alloc` and not contain or overlap a pointer provenance.
    fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>(
        llvals: &mut Vec<&'ll Value>,
        cx: &'a CodegenCx<'ll, 'b>,
        alloc: &'a Allocation,
        range: Range<usize>,
    ) {
        let chunks = alloc.init_mask().range_as_init_chunks(range.clone().into());

        let chunk_to_llval = move |chunk| match chunk {
            InitChunk::Init(range) => {
                let range = (range.start.bytes() as usize)..(range.end.bytes() as usize);
                let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range);
                cx.const_bytes(bytes)
            }
            InitChunk::Uninit(range) => {
                let len = range.end.bytes() - range.start.bytes();
                cx.const_undef(cx.type_array(cx.type_i8(), len))
            }
        };

        // Generating partially-uninit consts is limited to small numbers of chunks,
        // to avoid the cost of generating large complex const expressions.
        // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, and
        // would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`.
        let max = cx.sess().opts.unstable_opts.uninit_const_chunk_threshold;
        let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max;

        if allow_uninit_chunks {
            llvals.extend(chunks.map(chunk_to_llval));
        } else {
            // If this allocation contains any uninit bytes, codegen as if it was initialized
            // (using some arbitrary value for uninit bytes).
            let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range);
            llvals.push(cx.const_bytes(bytes));
        }
    }

    let mut next_offset = 0;
    for &(offset, prov) in alloc.provenance().ptrs().iter() {
        let offset = offset.bytes();
        assert_eq!(offset as usize as u64, offset);
        let offset = offset as usize;
        if offset > next_offset {
            // This `inspect` is okay since we have checked that there is no provenance, it
            // is within the bounds of the allocation, and it doesn't affect interpreter execution
            // (we inspect the result after interpreter execution).
            append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset);
        }
        let ptr_offset = read_target_uint(
            dl.endian,
            // This `inspect` is okay since it is within the bounds of the allocation, it doesn't
            // affect interpreter execution (we inspect the result after interpreter execution),
            // and we properly interpret the provenance as a relocation pointer offset.
            alloc.inspect_with_uninit_and_ptr_outside_interpreter(
                offset..(offset + pointer_size_bytes),
            ),
        )
        .expect("const_alloc_to_llvm: could not read relocation pointer")
            as u64;

        let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx);
        let schema = if cx.sess().pointer_authentication() {
            match is_init_fini {
                IsInitOrFini::Yes => cx.sess().pointer_authentication_init_fini(),
                IsInitOrFini::No => cx.sess().pointer_authentication_functions(),
            }
        } else {
            None
        };
        llvals.push(cx.scalar_to_backend_with_pac(
            InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx),
            Scalar::Initialized {
                value: Primitive::Pointer(address_space),
                valid_range: WrappingRange::full(pointer_size),
            },
            cx.type_ptr_ext(address_space),
            schema,
        ));
        next_offset = offset + pointer_size_bytes;
    }
    if alloc.len() >= next_offset {
        let range = next_offset..alloc.len();
        // This `inspect` is okay since we have check that it is after all provenance, it is
        // within the bounds of the allocation, and it doesn't affect interpreter execution (we
        // inspect the result after interpreter execution).
        append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range);
    }

    // Avoid wrapping in a struct if there is only a single value. This ensures
    // that LLVM is able to perform the string merging optimization if the constant
    // is a valid C string. LLVM only considers bare arrays for this optimization,
    // not arrays wrapped in a struct. LLVM handles this at:
    // https://github.com/rust-lang/llvm-project/blob/acaea3d2bb8f351b740db7ebce7d7a40b9e21488/llvm/lib/Target/TargetLoweringObjectFile.cpp#L249-L280
    if let &[data] = &*llvals { data } else { cx.const_struct(&llvals, true) }
}

fn codegen_static_initializer<'ll, 'tcx>(
    cx: &CodegenCx<'ll, 'tcx>,
    def_id: DefId,
) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> {
    let alloc = cx.tcx.eval_static_initializer(def_id)?;
    let attrs = cx.tcx.codegen_fn_attrs(def_id);
    // FIXME(jchlanda) Decide if this could be better served by `ctor` crate. See the discussion
    // here: <https://github.com/rust-lang/rust/pull/155722#discussion_r3320477047>
    let is_in_init_fini: IsInitOrFini = attrs
        .link_section
        .map(|link_section| {
            let s = link_section.as_str();
            if s.starts_with(".init_array") || s.starts_with(".fini_array") {
                IsInitOrFini::Yes
            } else {
                IsInitOrFini::No
            }
        })
        .unwrap_or(IsInitOrFini::No);
    Ok((const_alloc_to_llvm(cx, alloc.inner(), IsStatic::Yes, is_in_init_fini), alloc))
}

fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) {
    // The target may require greater alignment for globals than the type does.
    // Note: GCC and Clang also allow `__attribute__((aligned))` on variables,
    // which can force it to be smaller. Rust doesn't support this yet.
    if let Some(min_global) = cx.sess().target.min_global_align {
        align = Ord::max(align, min_global);
    }
    llvm::set_alignment(gv, align);
}

fn check_and_apply_linkage<'ll, 'tcx>(
    cx: &CodegenCx<'ll, 'tcx>,
    attrs: &CodegenFnAttrs,
    llty: &'ll Type,
    sym: &str,
    def_id: DefId,
) -> &'ll Value {
    if let Some(linkage) = attrs.import_linkage {
        debug!("get_static: sym={} linkage={:?}", sym, linkage);

        let mut should_sign = false;
        // Declare a symbol `foo`. If `foo` is an extern_weak symbol, we declare
        // an extern_weak function, otherwise a global with the desired linkage.
        let g1 = if matches!(attrs.import_linkage, Some(Linkage::ExternalWeak)) {
            // An `extern_weak` function is represented as an `Option<unsafe extern ...>`,
            // we extract the function signature and declare it as an extern_weak function
            // instead of an extern_weak i8.
            let instance = Instance::mono(cx.tcx, def_id);
            if let ty::Adt(struct_def, args) = instance.ty(cx.tcx, cx.typing_env()).kind()
                && cx.tcx.is_lang_item(struct_def.did(), LangItem::Option)
                && let ty::FnPtr(sig, header) = args.type_at(0).kind()
            {
                let fn_sig = sig.with(*header);
                let fn_abi = cx.fn_abi_of_fn_ptr(fn_sig, ty::List::empty());
                // Decide if the initializer needs to be signed
                if cx.sess().pointer_authentication_functions().is_some()
                    && matches!(fn_sig.abi(), ExternAbi::C { .. } | ExternAbi::System { .. })
                {
                    should_sign = true;
                }
                cx.declare_fn(sym, &fn_abi, None)
            } else {
                cx.declare_global(sym, cx.type_i8())
            }
        } else {
            cx.declare_global(sym, cx.type_i8())
        };
        llvm::set_linkage(g1, base::linkage_to_llvm(linkage));

        // Normally this is done in `get_static_inner`, but when as we generate an internal global,
        // it will apply the dso_local to the internal global instead, so do it here, too.
        cx.assume_dso_local(g1, true);

        // Declare an internal global `extern_with_linkage_foo` which
        // is initialized with the address of `foo`. If `foo` is
        // discarded during linking (for example, if `foo` has weak
        // linkage and there are no definitions), then
        // `extern_with_linkage_foo` will instead be initialized to
        // zero.
        let real_name =
            format!("_rust_extern_with_linkage_{:016x}_{sym}", cx.tcx.stable_crate_id(LOCAL_CRATE));
        let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| {
            cx.sess().dcx().emit_fatal(SymbolAlreadyDefined {
                span: cx.tcx.def_span(def_id),
                symbol_name: sym,
            })
        });
        llvm::set_linkage(g2, llvm::Linkage::InternalLinkage);
        llvm::set_unnamed_address(g2, llvm::UnnamedAddr::Global);

        // Sign the function pointer that is used to initialize the global
        let initializer = if should_sign {
            let schema = cx
                .tcx
                .sess
                .pointer_authentication_functions()
                .expect("pointer authentication functions must be available when signing");

            const_ptr_auth(
                cx.const_bitcast(g1, llty),
                schema.key as u32,
                schema.constant_discriminator as u64,
                None, /* address_diversity */
            )
        } else {
            g1
        };

        llvm::set_initializer(g2, initializer);

        g2
    } else if cx.tcx.sess.target.arch == Arch::X86
        && common::is_using_dlltool(&cx.tcx.sess.target)
        && let Some(dllimport) = crate::common::get_dllimport(cx.tcx, def_id, sym)
    {
        cx.declare_global(&common::i686_decorated_name(dllimport, true, true, false), llty)
    } else {
        // Generate an external declaration.
        // FIXME(nagisa): investigate whether it can be changed into define_global
        cx.declare_global(sym, llty)
    }
}

impl<'ll> CodegenCx<'ll, '_> {
    pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value {
        unsafe { llvm::LLVMConstBitCast(val, ty) }
    }

    pub(crate) fn const_pointercast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value {
        unsafe { llvm::LLVMConstPointerCast(val, ty) }
    }

    /// Create a global variable.
    ///
    /// The returned global variable is a pointer in the default address space for globals.
    /// Fails if a symbol with the given name already exists.
    pub(crate) fn static_addr_of_mut(
        &self,
        cv: &'ll Value,
        align: Align,
        kind: Option<&str>,
    ) -> &'ll Value {
        let gv = match kind {
            Some(kind) if !self.tcx.sess.fewer_names() => {
                let name = self.generate_local_symbol_name(kind);
                let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| {
                    bug!("symbol `{}` is already defined", name);
                });
                gv
            }
            _ => self.define_global("", self.val_ty(cv)).unwrap_or_else(|| {
                bug!("anonymous global symbol is already defined");
            }),
        };
        llvm::set_linkage(gv, llvm::Linkage::PrivateLinkage);
        llvm::set_initializer(gv, cv);
        set_global_alignment(self, gv, align);
        llvm::set_unnamed_address(gv, llvm::UnnamedAddr::Global);
        gv
    }

    /// Create a global constant.
    ///
    /// The returned global variable is a pointer in the default address space for globals.
    pub(crate) fn static_addr_of_impl(
        &self,
        cv: &'ll Value,
        align: Align,
        kind: Option<&str>,
    ) -> &'ll Value {
        if let Some(&gv) = self.const_globals.borrow().get(&cv) {
            unsafe {
                // Upgrade the alignment in cases where the same constant is used with different
                // alignment requirements
                let llalign = align.bytes() as u32;
                if llalign > llvm::LLVMGetAlignment(gv) {
                    llvm::LLVMSetAlignment(gv, llalign);
                }
            }
            return gv;
        }
        let gv = self.static_addr_of_mut(cv, align, kind);
        llvm::set_global_constant(gv, true);

        self.const_globals.borrow_mut().insert(cv, gv);
        gv
    }

    #[instrument(level = "debug", skip(self))]
    pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value {
        let instance = Instance::mono(self.tcx, def_id);
        trace!(?instance);

        let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() };
        // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure
        // out the llvm type from the actual evaluated initializer.
        let llty = if nested {
            self.type_i8()
        } else {
            let ty = instance.ty(self.tcx, self.typing_env());
            trace!(?ty);
            self.layout_of(ty).llvm_type(self)
        };
        self.get_static_inner(def_id, llty)
    }

    #[instrument(level = "debug", skip(self, llty))]
    fn get_static_inner(&self, def_id: DefId, llty: &'ll Type) -> &'ll Value {
        let instance = Instance::mono(self.tcx, def_id);
        if let Some(&g) = self.instances.borrow().get(&instance) {
            trace!("used cached value");
            return g;
        }

        let defined_in_current_codegen_unit =
            self.codegen_unit.items().contains_key(&MonoItem::Static(def_id));
        assert!(
            !defined_in_current_codegen_unit,
            "consts::get_static() should always hit the cache for \
                 statics defined in the same CGU, but did not for `{def_id:?}`"
        );

        let sym = self.tcx.symbol_name(instance).name;
        let fn_attrs = self.tcx.codegen_fn_attrs(def_id);

        debug!(?sym, ?fn_attrs);

        let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) {
            if let Some(g) = self.get_declared_value(sym) {
                if self.val_ty(g) != self.type_ptr() {
                    span_bug!(self.tcx.def_span(def_id), "Conflicting types for static");
                }
            }

            let g = self.declare_global(sym, llty);

            if !self.tcx.is_reachable_non_generic(def_id) {
                llvm::set_visibility(g, llvm::Visibility::Hidden);
            }

            g
        } else if let Some(classname) = fn_attrs.objc_class {
            self.get_objc_classref(classname)
        } else if let Some(methname) = fn_attrs.objc_selector {
            self.get_objc_selref(methname)
        } else {
            check_and_apply_linkage(self, fn_attrs, llty, sym, def_id)
        };

        // Thread-local statics in some other crate need to *always* be linked
        // against in a thread-local fashion, so we need to be sure to apply the
        // thread-local attribute locally if it was present remotely. If we
        // don't do this then linker errors can be generated where the linker
        // complains that one object files has a thread local version of the
        // symbol and another one doesn't.
        if fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
            llvm::set_thread_local_mode(g, self.tls_model);
        }

        let dso_local = self.assume_dso_local(g, true);

        if !def_id.is_local() {
            let is_eii = fn_attrs.flags.contains(CodegenFnAttrFlags::EXTERNALLY_IMPLEMENTABLE_ITEM);
            let needs_dll_storage_attr = self.use_dll_storage_attrs
                // EII static declarations are encoded as foreign items, but their symbols are
                // resolved by Rust crates, not native libraries.
                && (!self.tcx.is_foreign_item(def_id) || is_eii)
                // Local definitions can never be imported, so we must not apply
                // the DLLImport annotation.
                && !dso_local
                // Linker plugin ThinLTO doesn't create the self-dllimport Rust uses for rlibs
                // as the code generation happens out of process. Instead we assume static linkage
                // and disallow dynamic linking when linker plugin based LTO is enabled.
                // Regular in-process ThinLTO doesn't need this workaround.
                && !self.tcx.sess.opts.cg.linker_plugin_lto.enabled();

            // If this assertion triggers, there's something wrong with commandline
            // argument validation.
            assert!(
                !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled()
                    && self.tcx.sess.target.is_like_windows
                    && self.tcx.sess.opts.cg.prefer_dynamic)
            );

            if needs_dll_storage_attr {
                // This item is external but not foreign, i.e., it originates from an external Rust
                // crate. EII static declarations are handled the same way, even though they are
                // represented as foreign items. Since we don't know whether this crate will be
                // linked dynamically or statically in the final application, we always mark such
                // symbols as 'dllimport'. If final linkage happens to be static, we rely on
                // compiler-emitted __imp_ stubs to make things work.
                //
                // However, in some scenarios we defer emission of statics to downstream
                // crates, so there are cases where a static with an upstream DefId
                // is actually present in the current crate. We can find out via the
                // is_codegened_item query.
                if !self.tcx.is_codegened_item(def_id) {
                    llvm::set_dllimport_storage_class(g);
                }
            }
        }

        if self.use_dll_storage_attrs
            && let Some(library) = self.tcx.native_library(def_id)
            && library.kind.is_dllimport()
        {
            // For foreign (native) libs we know the exact storage type to use.
            llvm::set_dllimport_storage_class(g);
        }

        self.instances.borrow_mut().insert(instance, g);
        g
    }

    fn codegen_static_item(&mut self, def_id: DefId) {
        assert!(
            llvm::LLVMGetInitializer(
                self.instances.borrow().get(&Instance::mono(self.tcx, def_id)).unwrap()
            )
            .is_none()
        );
        let attrs = self.tcx.codegen_fn_attrs(def_id);

        let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else {
            // Error has already been reported
            return;
        };
        let alloc = alloc.inner();

        let val_llty = self.val_ty(v);

        let g = self.get_static_inner(def_id, val_llty);
        let llty = self.get_type_of_global(g);

        let g = if val_llty == llty {
            g
        } else {
            // codegen_static_initializer creates the global value just from the
            // `Allocation` data by generating one big struct value that is just
            // all the bytes and pointers after each other. This will almost never
            // match the type that the static was declared with. Unfortunately
            // we can't just LLVMConstBitCast our way out of it because that has very
            // specific rules on what can be cast. So instead of adding a new way to
            // generate static initializers that match the static's type, we picked
            // the easier option and retroactively change the type of the static item itself.
            let name = String::from_utf8(llvm::get_value_name(g))
                .expect("we declare our statics with a utf8-valid name");
            llvm::set_value_name(g, b"");

            let linkage = llvm::get_linkage(g);
            let visibility = llvm::get_visibility(g);

            let new_g = self.declare_global(&name, val_llty);

            llvm::set_linkage(new_g, linkage);
            llvm::set_visibility(new_g, visibility);

            // The old global has had its name removed but is returned by
            // get_static since it is in the instance cache. Provide an
            // alternative lookup that points to the new global so that
            // global_asm! can compute the correct mangled symbol name
            // for the global.
            self.renamed_statics.borrow_mut().insert(def_id, new_g);

            // To avoid breaking any invariants, we leave around the old
            // global for the moment; we'll replace all references to it
            // with the new global later. (See base::codegen_backend.)
            self.statics_to_rauw.borrow_mut().push((g, new_g));
            new_g
        };

        // NOTE: Alignment from attributes has already been applied to the allocation.
        set_global_alignment(self, g, alloc.align);
        llvm::set_initializer(g, v);

        self.assume_dso_local(g, true);

        // Forward the allocation's mutability (picked by the const interner) to LLVM.
        if alloc.mutability.is_not() {
            llvm::set_global_constant(g, true);
        }

        debuginfo::build_global_var_di_node(self, def_id, g);

        if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
            llvm::set_thread_local_mode(g, self.tls_model);
        }

        // Wasm statics with custom link sections get special treatment as they
        // also go into custom sections of the wasm executable. The exception to
        // this is the `.init_array` section for which we can't emit a custom
        // section as it contains relocations.
        if self.tcx.sess.target.is_like_wasm
            && attrs
                .link_section
                .map(|link_section| !link_section.as_str().starts_with(".init_array"))
                .unwrap_or(true)
        {
            if let Some(section) = attrs.link_section {
                let section = self.create_metadata(section.as_str().as_bytes());
                assert!(alloc.provenance().ptrs().is_empty());

                // The `inspect` method is okay here because we checked for provenance, and
                // because we are doing this access to inspect the final interpreter state (not
                // as part of the interpreter execution).
                let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len());
                let alloc = self.create_metadata(bytes);
                let data = [section, alloc];
                self.module_add_named_metadata_node(self.llmod(), c"wasm.custom_sections", &data);
            }
        }

        base::set_link_section(g, attrs);
        base::set_variable_sanitizer_attrs(g, attrs);

        if let Some(ignorelist) = &self.sanitizer_ignorelist {
            let instance = ty::Instance::mono(self.tcx, def_id);
            let sym_name = self.tcx.symbol_name(instance).name;
            let span = self.tcx.def_span(def_id);
            let source_map = self.tcx.sess.source_map();
            let filename =
                source_map.span_to_filename(span).prefer_local_unconditionally().to_string();
            let ty_name = rustc_middle::ty::print::with_no_trimmed_paths!(
                self.tcx.type_of(def_id).skip_binder().to_string()
            );
            let mainfile = self
                .tcx
                .sess
                .local_crate_source_file()
                .and_then(|path| path.local_path().map(|p| p.display().to_string()))
                .unwrap_or_default();

            let demangled =
                rustc_middle::ty::print::with_no_trimmed_paths!(self.tcx.def_path_str(def_id));

            let global_blame = |section| -> (
                rustc_sanitizers::ignorelist::Blame,
                rustc_sanitizers::ignorelist::Blame,
            ) {
                let mut no_san = rustc_sanitizers::ignorelist::Blame::NONE;
                let mut san = rustc_sanitizers::ignorelist::Blame::NONE;
                let mut update = |prefix, query| {
                    let (ns, s) = ignorelist.in_section_blame(section, prefix, query);
                    no_san = no_san.max(ns);
                    san = san.max(s);
                };
                update(c"global", sym_name);
                update(c"global", &demangled);
                update(c"src", &filename);
                if !mainfile.is_empty() {
                    update(c"mainfile", &mainfile);
                }
                update(c"type", &ty_name);
                (no_san, san)
            };

            let sanitizers = self.tcx.sess.sanitizers();
            let (address_nosan, address_san) = global_blame(c"address");
            let (kaddress_nosan, kaddress_san) = global_blame(c"kernel-address");
            let (hwaddress_nosan, hwaddress_san) = global_blame(c"hwaddress");
            let (khwaddress_nosan, khwaddress_san) = global_blame(c"kernel-hwaddress");

            let ignore_address =
                rustc_sanitizers::ignorelist::is_blame_ignored(address_nosan, address_san);
            let ignore_kernel_address = rustc_sanitizers::ignorelist::is_blame_ignored(
                address_nosan.max(kaddress_nosan),
                address_san.max(kaddress_san),
            );
            let ignore_hwaddress =
                rustc_sanitizers::ignorelist::is_blame_ignored(hwaddress_nosan, hwaddress_san);
            let ignore_kernel_hwaddress = rustc_sanitizers::ignorelist::is_blame_ignored(
                hwaddress_nosan.max(khwaddress_nosan),
                hwaddress_san.max(khwaddress_san),
            );

            if (sanitizers.contains(rustc_target::spec::SanitizerSet::ADDRESS) && ignore_address)
                || (sanitizers.contains(rustc_target::spec::SanitizerSet::KERNELADDRESS)
                    && ignore_kernel_address)
            {
                unsafe { llvm::LLVMRustSetNoSanitizeAddress(g) };
            }
            if (sanitizers.contains(rustc_target::spec::SanitizerSet::HWADDRESS)
                && ignore_hwaddress)
                || (sanitizers.contains(rustc_target::spec::SanitizerSet::KERNELHWADDRESS)
                    && ignore_kernel_hwaddress)
            {
                unsafe { llvm::LLVMRustSetNoSanitizeHWAddress(g) };
            }
        }

        if attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER) {
            // `USED` and `USED_LINKER` can't be used together.
            assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER));

            // The semantics of #[used] in Rust only require the symbol to make it into the
            // object file. It is explicitly allowed for the linker to strip the symbol if it
            // is dead, which means we are allowed to use `llvm.compiler.used` instead of
            // `llvm.used` here.
            //
            // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique
            // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs
            // in the handling of `.init_array` (the static constructor list) in versions of
            // the gold linker (prior to the one released with binutils 2.36).
            //
            // That said, we only ever emit these when `#[used(compiler)]` is explicitly
            // requested. This is to avoid similar breakage on other targets, in particular
            // MachO targets have *their* static constructor lists broken if `llvm.compiler.used`
            // is emitted rather than `llvm.used`. However, that check happens when assigning
            // the `CodegenFnAttrFlags` in the `codegen_fn_attrs` query, so we don't need to
            // take care of it here.
            self.add_compiler_used_global(g);
        }
        if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) {
            // `USED` and `USED_LINKER` can't be used together.
            assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER));

            self.add_used_global(g);
        }
    }

    /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr.
    pub(crate) fn add_used_global(&mut self, global: &'ll Value) {
        self.used_statics.push(global);
    }

    /// Add a global value to a list to be stored in the `llvm.compiler.used` variable,
    /// an array of ptr.
    pub(crate) fn add_compiler_used_global(&self, global: &'ll Value) {
        self.compiler_used_statics.borrow_mut().push(global);
    }

    // We do our best here to match what Clang does when compiling Objective-C natively.
    // See Clang's `CGObjCCommonMac::CreateCStringLiteral`:
    // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L4134
    fn define_objc_classname(&self, classname: &str) -> &'ll Value {
        assert_eq!(self.objc_abi_version(), 1);

        let llval = self.null_terminate_const_bytes(classname.as_bytes());
        let llty = self.val_ty(llval);
        let sym = self.generate_local_symbol_name("OBJC_CLASS_NAME_");
        let g = self.define_global(&sym, llty).unwrap_or_else(|| {
            bug!("symbol `{}` is already defined", sym);
        });
        set_global_alignment(self, g, self.tcx.data_layout.i8_align);
        llvm::set_initializer(g, llval);
        llvm::set_linkage(g, llvm::Linkage::PrivateLinkage);
        llvm::set_section(g, c"__TEXT,__cstring,cstring_literals");
        llvm::LLVMSetGlobalConstant(g, llvm::TRUE);
        llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global);
        self.add_compiler_used_global(g);

        g
    }

    // We do our best here to match what Clang does when compiling Objective-C natively.
    // See Clang's `ObjCNonFragileABITypesHelper`:
    // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L6052
    fn get_objc_class_t(&self) -> &'ll Type {
        if let Some(class_t) = self.objc_class_t.get() {
            return class_t;
        }

        assert_eq!(self.objc_abi_version(), 2);

        // struct _class_t {
        //     struct _class_t* isa;
        //     struct _class_t* const superclass;
        //     void* cache;
        //     IMP* vtable;
        //     struct class_ro_t* ro;
        // }

        let class_t = self.type_named_struct("struct._class_t");
        let els = [self.type_ptr(); 5];
        let packed = false;
        self.set_struct_body(class_t, &els, packed);

        self.objc_class_t.set(Some(class_t));
        class_t
    }

    // We do our best here to match what Clang does when compiling Objective-C natively. We
    // deduplicate references within a CGU, but we need a reference definition in each referencing
    // CGU. All attempts at using external references to a single reference definition result in
    // linker errors.
    fn get_objc_classref(&self, classname: Symbol) -> &'ll Value {
        let mut classrefs = self.objc_classrefs.borrow_mut();
        if let Some(classref) = classrefs.get(&classname).copied() {
            return classref;
        }

        let g = match self.objc_abi_version() {
            1 => {
                // See Clang's `CGObjCMac::EmitClassRefFromId`:
                // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5205
                let llval = self.define_objc_classname(classname.as_str());
                let llty = self.type_ptr();
                let sym = self.generate_local_symbol_name("OBJC_CLASS_REFERENCES_");
                let g = self.define_global(&sym, llty).unwrap_or_else(|| {
                    bug!("symbol `{}` is already defined", sym);
                });
                set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi);
                llvm::set_initializer(g, llval);
                llvm::set_linkage(g, llvm::Linkage::PrivateLinkage);
                llvm::set_section(g, c"__OBJC,__cls_refs,literal_pointers,no_dead_strip");
                self.add_compiler_used_global(g);
                g
            }
            2 => {
                // See Clang's `CGObjCNonFragileABIMac::EmitClassRefFromId`:
                // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L7423
                let llval = {
                    let extern_sym = format!("OBJC_CLASS_$_{}", classname.as_str());
                    let extern_llty = self.get_objc_class_t();
                    self.declare_global(&extern_sym, extern_llty)
                };
                let llty = self.type_ptr();
                let sym = self.generate_local_symbol_name("OBJC_CLASSLIST_REFERENCES_$_");
                let g = self.define_global(&sym, llty).unwrap_or_else(|| {
                    bug!("symbol `{}` is already defined", sym);
                });
                set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi);
                llvm::set_initializer(g, llval);
                llvm::set_linkage(g, llvm::Linkage::InternalLinkage);
                llvm::set_section(g, c"__DATA,__objc_classrefs,regular,no_dead_strip");
                self.add_compiler_used_global(g);
                g
            }
            _ => unreachable!(),
        };

        classrefs.insert(classname, g);
        g
    }

    // We do our best here to match what Clang does when compiling Objective-C natively. We
    // deduplicate references within a CGU, but we need a reference definition in each referencing
    // CGU. All attempts at using external references to a single reference definition result in
    // linker errors.
    //
    // Newer versions of Apple Clang generate calls to `@"objc_msgSend$methname"` selector stub
    // functions. We don't currently do that. The code we generate is closer to what Apple Clang
    // generates with the `-fno-objc-msgsend-selector-stubs` option.
    fn get_objc_selref(&self, methname: Symbol) -> &'ll Value {
        let mut selrefs = self.objc_selrefs.borrow_mut();
        if let Some(selref) = selrefs.get(&methname).copied() {
            return selref;
        }

        let abi_version = self.objc_abi_version();

        // See Clang's `CGObjCCommonMac::CreateCStringLiteral`:
        // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L4134
        let methname_llval = self.null_terminate_const_bytes(methname.as_str().as_bytes());
        let methname_llty = self.val_ty(methname_llval);
        let methname_sym = self.generate_local_symbol_name("OBJC_METH_VAR_NAME_");
        let methname_g = self.define_global(&methname_sym, methname_llty).unwrap_or_else(|| {
            bug!("symbol `{}` is already defined", methname_sym);
        });
        set_global_alignment(self, methname_g, self.tcx.data_layout.i8_align);
        llvm::set_initializer(methname_g, methname_llval);
        llvm::set_linkage(methname_g, llvm::Linkage::PrivateLinkage);
        llvm::set_section(
            methname_g,
            match abi_version {
                1 => c"__TEXT,__cstring,cstring_literals",
                2 => c"__TEXT,__objc_methname,cstring_literals",
                _ => unreachable!(),
            },
        );
        llvm::LLVMSetGlobalConstant(methname_g, llvm::TRUE);
        llvm::LLVMSetUnnamedAddress(methname_g, llvm::UnnamedAddr::Global);
        self.add_compiler_used_global(methname_g);

        // See Clang's `CGObjCMac::EmitSelectorAddr`:
        // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5243
        // And Clang's `CGObjCNonFragileABIMac::EmitSelectorAddr`:
        // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L7586
        let selref_llval = methname_g;
        let selref_llty = self.type_ptr();
        let selref_sym = self.generate_local_symbol_name("OBJC_SELECTOR_REFERENCES_");
        let selref_g = self.define_global(&selref_sym, selref_llty).unwrap_or_else(|| {
            bug!("symbol `{}` is already defined", selref_sym);
        });
        set_global_alignment(self, selref_g, self.tcx.data_layout.pointer_align().abi);
        llvm::set_initializer(selref_g, selref_llval);
        llvm::set_externally_initialized(selref_g, true);
        llvm::set_linkage(
            selref_g,
            match abi_version {
                1 => llvm::Linkage::PrivateLinkage,
                2 => llvm::Linkage::InternalLinkage,
                _ => unreachable!(),
            },
        );
        llvm::set_section(
            selref_g,
            match abi_version {
                1 => c"__OBJC,__message_refs,literal_pointers,no_dead_strip",
                2 => c"__DATA,__objc_selrefs,literal_pointers,no_dead_strip",
                _ => unreachable!(),
            },
        );
        self.add_compiler_used_global(selref_g);

        selrefs.insert(methname, selref_g);
        selref_g
    }

    // We do our best here to match what Clang does when compiling Objective-C natively.
    // See Clang's `ObjCTypesHelper`:
    // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5936
    // And Clang's `CGObjCMac::EmitModuleInfo`:
    // https://github.com/llvm/llvm-project/blob/llvmorg-20.1.8/clang/lib/CodeGen/CGObjCMac.cpp#L5151
    pub(crate) fn define_objc_module_info(&mut self) {
        assert_eq!(self.objc_abi_version(), 1);

        // struct _objc_module {
        //     long version;                // Hardcoded to 7 in Clang.
        //     long size;                   // sizeof(struct _objc_module)
        //     char* name;                  // Hardcoded to classname "" in Clang.
        //     struct _objc_symtab* symtab; // Null without class or category definitions.
        //  }

        let llty = self.type_named_struct("struct._objc_module");
        let i32_llty = self.type_i32();
        let ptr_llty = self.type_ptr();
        let packed = false;
        self.set_struct_body(llty, &[i32_llty, i32_llty, ptr_llty, ptr_llty], packed);

        let version = self.const_uint(i32_llty, 7);
        let size = self.const_uint(i32_llty, 16);
        let name = self.define_objc_classname("");
        let symtab = self.const_null(ptr_llty);
        let llval = crate::common::named_struct(llty, &[version, size, name, symtab]);

        let sym = "OBJC_MODULES";
        let g = self.define_global(&sym, llty).unwrap_or_else(|| {
            bug!("symbol `{}` is already defined", sym);
        });
        set_global_alignment(self, g, self.tcx.data_layout.pointer_align().abi);
        llvm::set_initializer(g, llval);
        llvm::set_linkage(g, llvm::Linkage::PrivateLinkage);
        llvm::set_section(g, c"__OBJC,__module_info,regular,no_dead_strip");

        self.add_compiler_used_global(g);
    }
}

impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> {
    /// Get a pointer to a global variable.
    ///
    /// The pointer will always be in the default address space. If global variables default to a
    /// different address space, an addrspacecast is inserted.
    fn static_addr_of(&self, alloc: ConstAllocation<'_>, kind: Option<&str>) -> &'ll Value {
        // FIXME: should we cache `const_alloc_to_llvm` to avoid repeating this for the
        // same `ConstAllocation`?
        let cv = const_alloc_to_llvm(self, alloc.inner(), IsStatic::No, IsInitOrFini::No);

        let gv = self.static_addr_of_impl(cv, alloc.inner().align, kind);
        // static_addr_of_impl returns the bare global variable, which might not be in the default
        // address space. Cast to the default address space if necessary.
        self.const_pointercast(gv, self.type_ptr())
    }

    fn codegen_static(&mut self, def_id: DefId) {
        self.codegen_static_item(def_id)
    }
}