rustc_codegen_ssa/back/

symbol_export.rs

use std::collections::hash_map::Entry::*;

use object::pe;
use rustc_abi::{CanonAbi, X86Call};
use rustc_ast::expand::allocator::{AllocatorKind, NO_ALLOC_SHIM_IS_UNSTABLE, global_fn_name};
use rustc_crate_store::CrateDepKind;
use rustc_data_structures::fx::FxHashSet;
use rustc_data_structures::unord::UnordMap;
use rustc_hir as hir;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LOCAL_CRATE, LocalDefId};
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::middle::exported_symbols::{
    ExportedSymbol, SymbolExportInfo, SymbolExportKind, SymbolExportLevel,
};
use rustc_middle::query::LocalCrate;
use rustc_middle::ty::{
    self, GenericArgKind, GenericArgsRef, Instance, ShimKind, SymbolName, Ty, TyCtxt,
};
use rustc_middle::util::Providers;
use rustc_span::{Span, bug};
use rustc_structures::CrateType;
use rustc_symbol_mangling::{is_offload_kernel, mangle_internal_symbol};
use rustc_target::spec::{Arch, Os, TlsModel};
use tracing::debug;

use crate::SymbolExport;
use crate::back::symbol_export;
use crate::base::allocator_shim_contents;

fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel {
    crates_export_threshold(tcx.crate_types())
}

fn crate_export_threshold(crate_type: CrateType) -> SymbolExportLevel {
    match crate_type {
        CrateType::Executable | CrateType::StaticLib | CrateType::ProcMacro | CrateType::Cdylib => {
            SymbolExportLevel::C
        }
        CrateType::Rlib | CrateType::Dylib | CrateType::Sdylib => SymbolExportLevel::Rust,
    }
}

pub fn crates_export_threshold(crate_types: &[CrateType]) -> SymbolExportLevel {
    if crate_types
        .iter()
        .any(|&crate_type| crate_export_threshold(crate_type) == SymbolExportLevel::Rust)
    {
        SymbolExportLevel::Rust
    } else {
        SymbolExportLevel::C
    }
}

fn reachable_non_generics_provider(tcx: TyCtxt<'_>, _: LocalCrate) -> DefIdMap<SymbolExportInfo> {
    if !tcx.sess.opts.output_types.should_codegen() && !tcx.is_sdylib_interface_build() {
        return Default::default();
    }

    reachable_non_generics_helper(tcx)
}

/// Exposed separately *without* the "should codegen" check so Miri can access it.
pub fn reachable_non_generics_helper(tcx: TyCtxt<'_>) -> DefIdMap<SymbolExportInfo> {
    let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE);

    let mut reachable_non_generics: DefIdMap<_> = tcx
        .reachable_set(())
        .items()
        .filter_map(|&def_id| {
            // We want to ignore some FFI functions that are not exposed from
            // this crate. Reachable FFI functions can be lumped into two
            // categories:
            //
            // 1. Those that are included statically via a static library
            // 2. Those included otherwise (e.g., dynamically or via a framework)
            //
            // Although our LLVM module is not literally emitting code for the
            // statically included symbols, it's an export of our library which
            // needs to be passed on to the linker and encoded in the metadata.
            //
            // As a result, if this id is an FFI item (foreign item) then we only
            // let it through if it's included statically.
            if let Some(parent_id) = tcx.opt_local_parent(def_id)
                && let DefKind::ForeignMod = tcx.def_kind(parent_id)
            {
                let library = tcx.native_library(def_id)?;
                return library.kind.is_statically_included().then_some(def_id);
            }

            // Only consider nodes that actually have exported symbols.
            match tcx.def_kind(def_id) {
                DefKind::Fn | DefKind::AssocFn
                    if tcx.constness(def_id) == hir::Constness::Const { always: true } =>
                {
                    return None;
                }
                DefKind::Fn | DefKind::Static { .. } => {}
                DefKind::AssocFn if tcx.impl_of_assoc(def_id.to_def_id()).is_some() => {}
                _ => return None,
            };

            let generics = tcx.generics_of(def_id);
            if generics.requires_monomorphization(tcx) {
                return None;
            }

            let instance = Instance::mono(tcx, def_id.into());

            if tcx.instantiate_and_check_impossible_clauses((instance.def_id(), instance.args)) {
                return None;
            }

            if instance.def.requires_inline(tcx) {
                return None;
            }

            if tcx.cross_crate_inlinable(def_id) { None } else { Some(def_id) }
        })
        .map(|def_id| {
            let export_level = if is_compiler_builtins {
                // We don't want to export compiler-builtins symbols from any
                // dylibs, even rust dylibs. Unlike all other crates it gets
                // duplicated in every linker invocation and it may otherwise
                // unintentionally override definitions of these symbols by
                // libgcc or compiler-rt for C code.
                SymbolExportLevel::Rust
            } else {
                symbol_export_level(tcx, def_id.to_def_id())
            };
            let codegen_attrs = tcx.codegen_fn_attrs(def_id.to_def_id());
            debug!(
                "EXPORTED SYMBOL (local): {} ({:?})",
                tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())),
                export_level
            );
            let info = SymbolExportInfo {
                level: export_level,
                kind: if tcx.is_static(def_id.to_def_id()) {
                    if codegen_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) {
                        SymbolExportKind::Tls
                    } else {
                        SymbolExportKind::Data
                    }
                } else {
                    SymbolExportKind::Text
                },
                used: codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER)
                    || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER),
                rustc_std_internal_symbol: codegen_attrs
                    .flags
                    .contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL)
                    || codegen_attrs
                        .flags
                        .contains(CodegenFnAttrFlags::EXTERNALLY_IMPLEMENTABLE_ITEM),
            };
            (def_id.to_def_id(), info)
        })
        .into();

    if let Some(id) = tcx.proc_macro_decls_static(()) {
        reachable_non_generics.insert(
            id.to_def_id(),
            SymbolExportInfo {
                level: SymbolExportLevel::C,
                kind: SymbolExportKind::Data,
                used: false,
                rustc_std_internal_symbol: false,
            },
        );
    }

    reachable_non_generics
}

fn is_reachable_non_generic_provider_local(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
    let export_threshold = threshold(tcx);

    if let Some(&info) = tcx.reachable_non_generics(LOCAL_CRATE).get(&def_id.to_def_id()) {
        info.level.is_below_threshold(export_threshold)
    } else {
        false
    }
}

fn is_reachable_non_generic_provider_extern(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
    tcx.reachable_non_generics(def_id.krate).contains_key(&def_id)
}

fn exported_non_generic_symbols_provider_local<'tcx>(
    tcx: TyCtxt<'tcx>,
    _: LocalCrate,
) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
    if !tcx.sess.opts.output_types.should_codegen() && !tcx.is_sdylib_interface_build() {
        return &[];
    }

    exported_non_generic_symbols_helper(tcx)
}

/// Exposed separately *without* the "should codegen" check so Miri can access it.
pub fn exported_non_generic_symbols_helper<'tcx>(
    tcx: TyCtxt<'tcx>,
) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
    // FIXME: Sorting this is unnecessary since we are sorting later anyway.
    //        Can we skip the later sorting?
    let sorted = tcx.with_stable_hashing_context(|mut hcx| {
        tcx.reachable_non_generics(LOCAL_CRATE).to_sorted(&mut hcx, true)
    });

    let mut symbols: Vec<_> =
        sorted.iter().map(|&(&def_id, &info)| (ExportedSymbol::NonGeneric(def_id), info)).collect();

    // Export TLS shims
    if !tcx.sess.target.dll_tls_export {
        symbols.extend(sorted.iter().filter_map(|&(&def_id, &info)| {
            tcx.needs_thread_local_shim(def_id).then(|| {
                (
                    ExportedSymbol::ThreadLocalShim(def_id),
                    SymbolExportInfo {
                        level: info.level,
                        kind: SymbolExportKind::Text,
                        used: info.used,
                        rustc_std_internal_symbol: info.rustc_std_internal_symbol,
                    },
                )
            })
        }))
    }

    symbols.extend(sorted.iter().flat_map(|&(&def_id, &info)| {
        tcx.codegen_fn_attrs(def_id).foreign_item_symbol_aliases.iter().map(
            move |&(foreign_item, _linkage, _visibility)| {
                (ExportedSymbol::NonGeneric(foreign_item), info)
            },
        )
    }));

    if tcx.entry_fn(()).is_some() {
        let exported_symbol =
            ExportedSymbol::NoDefId(SymbolName::new(tcx, tcx.sess.target.entry_name.as_ref()));

        symbols.push((
            exported_symbol,
            SymbolExportInfo {
                level: SymbolExportLevel::C,
                kind: SymbolExportKind::Text,
                used: false,
                rustc_std_internal_symbol: false,
            },
        ));
    }

    let is_device_offload = tcx
        .sess
        .opts
        .unstable_opts
        .offload
        .iter()
        .any(|o| matches!(o, rustc_session::config::Offload::Device(_)));
    if is_device_offload {
        let crate_items = tcx.hir_crate_items(());
        let mut seen: rustc_data_structures::fx::FxHashSet<DefId> = symbols
            .iter()
            .filter_map(|(s, _)| match s {
                ExportedSymbol::NonGeneric(d) => Some(*d),
                _ => None,
            })
            .collect();

        let mut try_emit_offload_kernel = |def_id: DefId, seen: &mut FxHashSet<DefId>| {
            if !matches!(tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) {
                return;
            }
            if !tcx.generics_of(def_id).requires_monomorphization(tcx)
                && is_offload_kernel(tcx.codegen_fn_attrs(def_id))
                && seen.insert(def_id)
            {
                symbols.push((
                    ExportedSymbol::NonGeneric(def_id),
                    SymbolExportInfo {
                        level: SymbolExportLevel::C,
                        kind: SymbolExportKind::Text,
                        used: false,
                        rustc_std_internal_symbol: false,
                    },
                ));
            }
        };

        for id in crate_items.free_items() {
            try_emit_offload_kernel(id.owner_id.to_def_id(), &mut seen);
        }
        for id in crate_items.impl_items() {
            try_emit_offload_kernel(id.owner_id.to_def_id(), &mut seen);
        }
    }

    // Sort so we get a stable incr. comp. hash.
    symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx));

    tcx.arena.alloc_from_iter(symbols)
}

fn exported_generic_symbols_provider_local<'tcx>(
    tcx: TyCtxt<'tcx>,
    _: LocalCrate,
) -> &'tcx [(ExportedSymbol<'tcx>, SymbolExportInfo)] {
    if !tcx.sess.opts.output_types.should_codegen() && !tcx.is_sdylib_interface_build() {
        return &[];
    }

    let mut symbols: Vec<_> = vec![];

    let export_generics = tcx.local_crate_exports_generics();
    let is_device_offload = tcx
        .sess
        .opts
        .unstable_opts
        .offload
        .iter()
        .any(|o| matches!(o, rustc_session::config::Offload::Device(_)));

    if export_generics || is_device_offload {
        use rustc_attr_ir::{InlineAttr, Linkage};
        use rustc_middle::mono::{MonoItem, Visibility};
        use rustc_middle::ty::InstanceKind;

        // Normally, we require that shared monomorphizations are not hidden,
        // because if we want to re-use a monomorphization from a Rust dylib, it
        // needs to be exported.
        // However, on platforms that don't allow for Rust dylibs, having
        // external linkage is enough for monomorphization to be linked to.
        let need_visibility = tcx.sess.target.dynamic_linking && !tcx.sess.target.only_cdylib;

        let cgus = tcx.collect_and_partition_mono_items(()).codegen_units;

        // Do not export symbols that cannot be instantiated by downstream crates.
        let reachable_set = tcx.reachable_set(());
        let is_local_to_current_crate = |ty: Ty<'_>| {
            let no_refs = ty.peel_refs();
            let root_def_id = match no_refs.kind() {
                ty::Closure(closure, _) => *closure,
                ty::FnDef(def_id, _) => *def_id,
                ty::Coroutine(def_id, _) => *def_id,
                ty::CoroutineClosure(def_id, _) => *def_id,
                ty::CoroutineWitness(def_id, _) => *def_id,
                _ => return false,
            };
            let Some(root_def_id) = root_def_id.as_local() else {
                return false;
            };

            let is_local = !reachable_set.contains(&root_def_id);
            is_local
        };

        let is_instantiable_downstream =
            |did: Option<DefId>, generic_args: GenericArgsRef<'tcx>| {
                generic_args
                    .types()
                    .chain(did.into_iter().map(move |did| tcx.type_of(did).skip_binder()))
                    .all(move |arg| {
                        arg.walk()
                            .all(|ty| ty.as_type().is_none_or(|ty| !is_local_to_current_crate(ty)))
                    })
            };

        let is_offload_instance = |mono_item: &MonoItem<'tcx>| {
            if let MonoItem::Fn(instance) = mono_item {
                is_offload_kernel(tcx.codegen_fn_attrs(instance.def_id()))
            } else {
                false
            }
        };

        // The symbols created in this loop are sorted below it
        #[allow(rustc::potential_query_instability)]
        for (mono_item, data) in cgus.iter().flat_map(|cgu| cgu.items().iter()) {
            if data.linkage != Linkage::External {
                // We can only re-use things with external linkage, otherwise
                // we'll get a linker error
                continue;
            }

            if need_visibility && data.visibility == Visibility::Hidden {
                // If we potentially share things from Rust dylibs, they must
                // not be hidden
                continue;
            }

            let item_is_offload = is_offload_instance(mono_item);

            if !item_is_offload && !tcx.sess.opts.share_generics() {
                if tcx.codegen_fn_attrs(mono_item.def_id()).inline == InlineAttr::Never {
                    // this is OK, we explicitly allow sharing inline(never) across crates even
                    // without share-generics.
                } else {
                    continue;
                }
            }

            // Note: These all set rustc_std_internal_symbol to false as generic functions must not
            // be marked with this attribute and we are only handling generic functions here.
            match *mono_item {
                MonoItem::Fn(Instance { def: InstanceKind::Item(def), args }) => {
                    let has_generics = args.non_erasable_generics().next().is_some();

                    let should_export = if item_is_offload {
                        has_generics
                    } else {
                        has_generics && is_instantiable_downstream(Some(def), &args)
                    };

                    if should_export {
                        let symbol = ExportedSymbol::Generic(def, args);
                        symbols.push((
                            symbol,
                            SymbolExportInfo {
                                level: if item_is_offload {
                                    SymbolExportLevel::C
                                } else {
                                    SymbolExportLevel::Rust
                                },
                                kind: SymbolExportKind::Text,
                                used: false,
                                rustc_std_internal_symbol: false,
                            },
                        ));
                    }
                }
                MonoItem::Fn(Instance {
                    def: InstanceKind::Shim(ShimKind::DropGlue(_, Some(ty))),
                    args,
                }) => {
                    // A little sanity-check
                    assert_eq!(args.non_erasable_generics().next(), Some(GenericArgKind::Type(ty)));

                    // Drop glue did is always going to be non-local outside of libcore, thus we don't need to check it's locality (which includes invoking `type_of` query).
                    let should_export = match ty.kind() {
                        ty::Adt(_, args) => is_instantiable_downstream(None, args),
                        ty::Closure(_, args) => is_instantiable_downstream(None, args),
                        _ => true,
                    };

                    if should_export {
                        symbols.push((
                            ExportedSymbol::DropGlue(ty),
                            SymbolExportInfo {
                                level: SymbolExportLevel::Rust,
                                kind: SymbolExportKind::Text,
                                used: false,
                                rustc_std_internal_symbol: false,
                            },
                        ));
                    }
                }
                MonoItem::Fn(Instance {
                    def: InstanceKind::Shim(ShimKind::AsyncDropGlueCtor(_, ty)),
                    args,
                }) => {
                    // A little sanity-check
                    assert_eq!(args.non_erasable_generics().next(), Some(GenericArgKind::Type(ty)));
                    symbols.push((
                        ExportedSymbol::AsyncDropGlueCtorShim(ty),
                        SymbolExportInfo {
                            level: SymbolExportLevel::Rust,
                            kind: SymbolExportKind::Text,
                            used: false,
                            rustc_std_internal_symbol: false,
                        },
                    ));
                }
                MonoItem::Fn(Instance {
                    def: InstanceKind::Shim(ShimKind::AsyncDropGlue(def, ty)),
                    args: _,
                }) => {
                    symbols.push((
                        ExportedSymbol::AsyncDropGlue(def, ty),
                        SymbolExportInfo {
                            level: SymbolExportLevel::Rust,
                            kind: SymbolExportKind::Text,
                            used: false,
                            rustc_std_internal_symbol: false,
                        },
                    ));
                }
                _ => {
                    // Any other symbols don't qualify for sharing
                }
            }
        }
    }

    // Sort so we get a stable incr. comp. hash.
    symbols.sort_by_cached_key(|s| s.0.symbol_name_for_local_instance(tcx));

    tcx.arena.alloc_from_iter(symbols)
}

fn upstream_monomorphizations_provider(
    tcx: TyCtxt<'_>,
    (): (),
) -> DefIdMap<UnordMap<GenericArgsRef<'_>, CrateNum>> {
    let cnums = tcx.crates(());

    let mut instances: DefIdMap<UnordMap<_, _>> = Default::default();

    let drop_glue_fn_def_id = tcx.lang_items().drop_glue_fn();
    let async_drop_in_place_fn_def_id = tcx.lang_items().async_drop_in_place_fn();

    for &cnum in cnums.iter() {
        // It should be possible to compile to build a crate against a conditional dependency then
        // later link that crate without the conditional dependency, so we cannot use exported
        // generics from conditional dependencies.
        // https://github.com/rust-lang/rust/issues/159682
        if tcx.crate_dep_kind(cnum) == CrateDepKind::Conditional {
            continue;
        }

        for (exported_symbol, _) in tcx.exported_generic_symbols(cnum).iter() {
            let (def_id, args) = match *exported_symbol {
                ExportedSymbol::Generic(def_id, args) => (def_id, args),
                ExportedSymbol::DropGlue(ty) => {
                    if let Some(drop_in_place_fn_def_id) = drop_glue_fn_def_id {
                        (drop_in_place_fn_def_id, tcx.mk_args(&[ty.into()]))
                    } else {
                        // `drop_glue` does not exist, don't try to use it.
                        continue;
                    }
                }
                ExportedSymbol::AsyncDropGlueCtorShim(ty) => {
                    if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id {
                        (async_drop_in_place_fn_def_id, tcx.mk_args(&[ty.into()]))
                    } else {
                        continue;
                    }
                }
                ExportedSymbol::AsyncDropGlue(def_id, ty) => (def_id, tcx.mk_args(&[ty.into()])),
                ExportedSymbol::NonGeneric(..)
                | ExportedSymbol::ThreadLocalShim(..)
                | ExportedSymbol::NoDefId(..) => unreachable!("{exported_symbol:?}"),
            };

            let args_map = instances.entry(def_id).or_default();

            match args_map.entry(args) {
                Occupied(mut e) => {
                    // If there are multiple monomorphizations available,
                    // we select one deterministically.
                    let other_cnum = *e.get();
                    if tcx.stable_crate_id(other_cnum) > tcx.stable_crate_id(cnum) {
                        e.insert(cnum);
                    }
                }
                Vacant(e) => {
                    e.insert(cnum);
                }
            }
        }
    }

    instances
}

fn upstream_monomorphizations_for_provider(
    tcx: TyCtxt<'_>,
    def_id: DefId,
) -> Option<&UnordMap<GenericArgsRef<'_>, CrateNum>> {
    assert!(!def_id.is_local());
    tcx.upstream_monomorphizations(()).get(&def_id)
}

fn upstream_drop_glue_for_provider<'tcx>(
    tcx: TyCtxt<'tcx>,
    args: GenericArgsRef<'tcx>,
) -> Option<CrateNum> {
    let def_id = tcx.lang_items().drop_glue_fn()?;
    tcx.upstream_monomorphizations_for(def_id)?.get(&args).cloned()
}

fn upstream_async_drop_glue_for_provider<'tcx>(
    tcx: TyCtxt<'tcx>,
    args: GenericArgsRef<'tcx>,
) -> Option<CrateNum> {
    let def_id = tcx.lang_items().async_drop_in_place_fn()?;
    tcx.upstream_monomorphizations_for(def_id)?.get(&args).cloned()
}

fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
    !tcx.reachable_set(()).contains(&def_id)
}

pub(crate) fn provide(providers: &mut Providers) {
    providers.queries.reachable_non_generics = reachable_non_generics_provider;
    providers.queries.is_reachable_non_generic = is_reachable_non_generic_provider_local;
    providers.queries.exported_non_generic_symbols = exported_non_generic_symbols_provider_local;
    providers.queries.exported_generic_symbols = exported_generic_symbols_provider_local;
    providers.queries.upstream_monomorphizations = upstream_monomorphizations_provider;
    providers.queries.is_unreachable_local_definition = is_unreachable_local_definition_provider;
    providers.queries.upstream_drop_glue_for = upstream_drop_glue_for_provider;
    providers.queries.upstream_async_drop_glue_for = upstream_async_drop_glue_for_provider;
    providers.queries.wasm_import_module_map = wasm_import_module_map;
    providers.extern_queries.is_reachable_non_generic = is_reachable_non_generic_provider_extern;
    providers.extern_queries.upstream_monomorphizations_for =
        upstream_monomorphizations_for_provider;
}

pub(crate) fn allocator_shim_symbols(
    tcx: TyCtxt<'_>,
    kind: AllocatorKind,
) -> impl Iterator<Item = (String, SymbolExportKind)> {
    allocator_shim_contents(tcx, kind)
        .into_iter()
        .map(move |method| mangle_internal_symbol(tcx, global_fn_name(method.name).as_str()))
        .chain([mangle_internal_symbol(tcx, NO_ALLOC_SHIM_IS_UNSTABLE)])
        .map(move |symbol_name| {
            let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name));

            (
                symbol_export::exporting_symbol_name_for_instance_in_crate(
                    tcx,
                    exported_symbol,
                    LOCAL_CRATE,
                ),
                SymbolExportKind::Text,
            )
        })
}

fn symbol_export_level(tcx: TyCtxt<'_>, sym_def_id: DefId) -> SymbolExportLevel {
    // We export anything that's not mangled at the "C" layer as it probably has
    // to do with ABI concerns. We do not, however, apply such treatment to
    // special symbols in the standard library for various plumbing between
    // core/std/allocators/etc. For example symbols used to hook up allocation
    // are not considered for export
    let codegen_fn_attrs = tcx.codegen_fn_attrs(sym_def_id);
    let is_extern = codegen_fn_attrs.contains_extern_indicator();
    let std_internal =
        codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::RUSTC_STD_INTERNAL_SYMBOL);
    let eii = codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::EXTERNALLY_IMPLEMENTABLE_ITEM);

    if is_extern && !std_internal && !eii {
        let target = &tcx.sess.target.llvm_target;
        // WebAssembly cannot export data symbols, so reduce their export level
        // FIXME(jdonszelmann) don't do a substring match here.
        if target.contains("emscripten") {
            if let DefKind::Static { .. } = tcx.def_kind(sym_def_id) {
                return SymbolExportLevel::Rust;
            }
        }

        SymbolExportLevel::C
    } else {
        SymbolExportLevel::Rust
    }
}

/// This is the symbol name of the given instance instantiated in a specific crate.
pub(crate) fn symbol_name_for_instance_in_crate<'tcx>(
    tcx: TyCtxt<'tcx>,
    symbol: ExportedSymbol<'tcx>,
    instantiating_crate: CrateNum,
) -> String {
    // If this is something instantiated in the local crate then we might
    // already have cached the name as a query result.
    if instantiating_crate == LOCAL_CRATE {
        return symbol.symbol_name_for_local_instance(tcx).to_string();
    }

    // This is something instantiated in an upstream crate, so we have to use
    // the slower (because uncached) version of computing the symbol name.
    match symbol {
        ExportedSymbol::NonGeneric(def_id) => {
            rustc_symbol_mangling::symbol_name_for_instance_in_crate(
                tcx,
                Instance::mono(tcx, def_id),
                instantiating_crate,
            )
        }
        ExportedSymbol::Generic(def_id, args) => {
            rustc_symbol_mangling::symbol_name_for_instance_in_crate(
                tcx,
                Instance::new_raw(def_id, args),
                instantiating_crate,
            )
        }
        ExportedSymbol::ThreadLocalShim(def_id) => {
            rustc_symbol_mangling::symbol_name_for_instance_in_crate(
                tcx,
                ty::Instance {
                    def: ty::InstanceKind::Shim(ty::ShimKind::ThreadLocal(def_id)),
                    args: ty::GenericArgs::empty(),
                },
                instantiating_crate,
            )
        }
        ExportedSymbol::DropGlue(ty) => rustc_symbol_mangling::symbol_name_for_instance_in_crate(
            tcx,
            Instance::resolve_drop_glue(tcx, ty),
            instantiating_crate,
        ),
        ExportedSymbol::AsyncDropGlueCtorShim(ty) => {
            rustc_symbol_mangling::symbol_name_for_instance_in_crate(
                tcx,
                Instance::resolve_async_drop_in_place(tcx, ty),
                instantiating_crate,
            )
        }
        ExportedSymbol::AsyncDropGlue(def_id, ty) => {
            rustc_symbol_mangling::symbol_name_for_instance_in_crate(
                tcx,
                Instance::resolve_async_drop_in_place_poll(tcx, def_id, ty),
                instantiating_crate,
            )
        }
        ExportedSymbol::NoDefId(symbol_name) => symbol_name.to_string(),
    }
}

fn calling_convention_for_symbol<'tcx>(
    tcx: TyCtxt<'tcx>,
    symbol: ExportedSymbol<'tcx>,
) -> (CanonAbi, &'tcx [rustc_target::callconv::ArgAbi<'tcx, Ty<'tcx>>]) {
    let instance = match symbol {
        ExportedSymbol::NonGeneric(def_id) | ExportedSymbol::Generic(def_id, _)
            if tcx.is_static(def_id) =>
        {
            None
        }
        ExportedSymbol::NonGeneric(def_id) => Some(Instance::mono(tcx, def_id)),
        ExportedSymbol::Generic(def_id, args) => Some(Instance::new_raw(def_id, args)),
        // DropGlue always use the Rust calling convention and thus follow the target's default
        // symbol decoration scheme.
        ExportedSymbol::DropGlue(..) => None,
        // AsyncDropGlueCtorShim always use the Rust calling convention and thus follow the
        // target's default symbol decoration scheme.
        ExportedSymbol::AsyncDropGlueCtorShim(..) => None,
        ExportedSymbol::AsyncDropGlue(..) => None,
        // NoDefId always follow the target's default symbol decoration scheme.
        ExportedSymbol::NoDefId(..) => None,
        // ThreadLocalShim always follow the target's default symbol decoration scheme.
        ExportedSymbol::ThreadLocalShim(..) => None,
    };

    instance
        .map(|i| {
            tcx.fn_abi_of_instance(
                ty::TypingEnv::fully_monomorphized().as_query_input((i, ty::List::empty())),
            )
            .unwrap_or_else(|_| bug!("fn_abi_of_instance({i:?}) failed"))
        })
        .map(|fnabi| (fnabi.conv, &fnabi.args[..]))
        // FIXME(workingjubilee): why don't we know the convention here?
        .unwrap_or((CanonAbi::Rust, &[]))
}

/// This is the symbol name of the given instance as seen by the linker.
///
/// On 32-bit Windows symbols are decorated according to their calling conventions.
pub(crate) fn linking_symbol_name_for_instance_in_crate<'tcx>(
    tcx: TyCtxt<'tcx>,
    symbol: ExportedSymbol<'tcx>,
    export_kind: SymbolExportKind,
    instantiating_crate: CrateNum,
) -> String {
    let mut undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate);

    // thread local will not be a function call,
    // so it is safe to return before windows symbol decoration check.
    if let Some(name) = maybe_emutls_symbol_name(tcx, symbol, &undecorated) {
        return name;
    }

    let target = &tcx.sess.target;
    if !target.is_like_windows {
        // Mach-O has a global "_" suffix and `object` crate will handle it.
        // ELF does not have any symbol decorations.
        return undecorated;
    }

    let prefix = match target.arch {
        Arch::X86 => Some('_'),
        Arch::X86_64 => None,
        // Only functions are decorated for arm64ec.
        Arch::Arm64EC if export_kind == SymbolExportKind::Text => Some('#'),
        // Only x86/64 and arm64ec use symbol decorations.
        _ => return undecorated,
    };

    let (callconv, args) = calling_convention_for_symbol(tcx, symbol);

    // Decorate symbols with prefixes, suffixes and total number of bytes of arguments.
    // Reference: https://docs.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170
    let (prefix, suffix) = match callconv {
        CanonAbi::X86(X86Call::Fastcall) => ("@", "@"),
        CanonAbi::X86(X86Call::Stdcall) => ("_", "@"),
        CanonAbi::X86(X86Call::Vectorcall) => ("", "@@"),
        _ => {
            if let Some(prefix) = prefix {
                undecorated.insert(0, prefix);
            }
            return undecorated;
        }
    };

    let args_in_bytes: u64 = args
        .iter()
        .map(|abi| abi.layout.size.bytes().next_multiple_of(target.pointer_width as u64 / 8))
        .sum();
    format!("{prefix}{undecorated}{suffix}{args_in_bytes}")
}

pub(crate) fn exporting_symbol_name_for_instance_in_crate<'tcx>(
    tcx: TyCtxt<'tcx>,
    symbol: ExportedSymbol<'tcx>,
    cnum: CrateNum,
) -> String {
    let undecorated = symbol_name_for_instance_in_crate(tcx, symbol, cnum);
    maybe_emutls_symbol_name(tcx, symbol, &undecorated).unwrap_or(undecorated)
}

fn strip_numeric_suffix<'a>(base: &'a str, suffix: impl AsRef<str>, fallback: &'a str) -> &'a str {
    if suffix.as_ref().parse::<u32>().is_ok() { base } else { fallback }
}

/// COFF `IMAGE_FILE_MACHINE_*` for an arch with decorated symbols, or `None`.
pub(super) fn coff_machine_for_arch(arch: &Arch) -> Option<u16> {
    match *arch {
        Arch::X86 => Some(pe::IMAGE_FILE_MACHINE_I386),
        Arch::X86_64 => Some(pe::IMAGE_FILE_MACHINE_AMD64),
        Arch::Arm64EC => Some(pe::IMAGE_FILE_MACHINE_ARM64EC),
        _ => None,
    }
}

/// Undecorated counterpart of [`linking_symbol_name_for_instance_in_crate`].
pub(super) fn undecorate_coff_symbol<'a>(
    name: &'a str,
    machine: u16,
    kind: SymbolExportKind,
) -> &'a str {
    // MSVC C++ mangled names start with '?' and use a completely different
    // decorating scheme that includes '@@' as structural delimiters.
    // They must not be subjected to C calling-convention undecoration.
    if name.starts_with('?') {
        return name;
    }
    match machine {
        pe::IMAGE_FILE_MACHINE_I386 => {
            if let Some(rest) = name.strip_prefix('@') {
                // fastcall: @foo@N -> foo
                rest.rsplit_once('@')
                    .map(|(base, suffix)| strip_numeric_suffix(base, suffix, name))
                    .unwrap_or(name)
            } else if let Some(stripped) = name.strip_prefix('_') {
                if let Some((base, suffix)) = stripped.rsplit_once('@') {
                    // stdcall: _foo@N -> foo
                    strip_numeric_suffix(base, suffix, stripped)
                } else {
                    // cdecl: _foo -> foo
                    stripped
                }
            } else {
                // vectorcall: foo@@N -> foo
                name.rsplit_once("@@")
                    .map(|(base, suffix)| strip_numeric_suffix(base, suffix, name))
                    .unwrap_or(name)
            }
        }
        pe::IMAGE_FILE_MACHINE_AMD64 => {
            // vectorcall: foo@@N -> foo
            name.rsplit_once("@@")
                .map(|(base, suffix)| strip_numeric_suffix(base, suffix, name))
                .unwrap_or(name)
        }
        pe::IMAGE_FILE_MACHINE_ARM64EC if kind == SymbolExportKind::Text => {
            // `#` prefix distinguishes ARM64EC text symbols from x64 thunks.
            name.strip_prefix('#').unwrap_or(name)
        }
        _ => name,
    }
}

/// On amdhsa, `gpu-kernel` functions have an associated metadata object with a `.kd` suffix.
/// Add it to the symbols list for all kernel functions, so that it is exported in the linked
/// object.
pub(crate) fn extend_exported_symbols<'tcx>(
    symbols: &mut Vec<SymbolExport>,
    tcx: TyCtxt<'tcx>,
    symbol: ExportedSymbol<'tcx>,
    instantiating_crate: CrateNum,
) {
    let (callconv, _) = calling_convention_for_symbol(tcx, symbol);

    if callconv != CanonAbi::GpuKernel || tcx.sess.target.os != Os::AmdHsa {
        return;
    }

    let undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate);

    // Add the symbol for the kernel descriptor (with .kd suffix)
    // Per https://llvm.org/docs/AMDGPUUsage.html#symbols these will always be `STT_OBJECT` so
    // export as data.
    symbols.push(SymbolExport::new(format!("{undecorated}.kd"), SymbolExportKind::Data));
}

fn maybe_emutls_symbol_name<'tcx>(
    tcx: TyCtxt<'tcx>,
    symbol: ExportedSymbol<'tcx>,
    undecorated: &str,
) -> Option<String> {
    if matches!(tcx.sess.tls_model(), TlsModel::Emulated)
        && let ExportedSymbol::NonGeneric(def_id) = symbol
        && tcx.is_thread_local_static(def_id)
    {
        // When using emutls, LLVM will add the `__emutls_v.` prefix to thread local symbols,
        // and exported symbol name need to match this.
        Some(format!("__emutls_v.{undecorated}"))
    } else {
        None
    }
}

fn wasm_import_module_map(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<String> {
    // Build up a map from DefId to a `NativeLib` structure, where
    // `NativeLib` internally contains information about
    // `#[link(wasm_import_module = "...")]` for example.
    let native_libs = tcx.native_libraries(cnum);

    let def_id_to_native_lib = native_libs
        .iter()
        .filter_map(|lib| lib.foreign_module.map(|id| (id, lib)))
        .collect::<DefIdMap<_>>();

    let mut ret = DefIdMap::default();
    for (def_id, lib) in tcx.foreign_modules(cnum).iter() {
        let module = def_id_to_native_lib.get(def_id).and_then(|s| s.wasm_import_module());
        let Some(module) = module else { continue };
        ret.extend(lib.foreign_items.iter().map(|id| {
            assert_eq!(id.krate, cnum);
            (*id, module.to_string())
        }));
    }

    ret
}

pub fn escape_symbol_name(tcx: TyCtxt<'_>, symbol: &str, span: Span) -> String {
    // https://github.com/llvm/llvm-project/blob/a55fbab0cffc9b4af497b9e4f187b61143743e06/llvm/lib/MC/MCSymbol.cpp
    use rustc_target::spec::{Arch, BinaryFormat};
    if !symbol.is_empty()
        && symbol.chars().all(|c| matches!(c, '0'..='9' | 'A'..='Z' | 'a'..='z' | '_' | '$' | '.'))
    {
        return symbol.to_string();
    }
    if tcx.sess.target.binary_format == BinaryFormat::Xcoff {
        tcx.sess.dcx().span_fatal(
            span,
            format!(
                "symbol escaping is not supported for the binary format {}",
                tcx.sess.target.binary_format
            ),
        );
    }
    if tcx.sess.target.arch == Arch::Nvptx64 {
        tcx.sess.dcx().span_fatal(
            span,
            format!(
                "symbol escaping is not supported for the architecture {}",
                tcx.sess.target.arch
            ),
        );
    }
    let mut escaped_symbol = String::new();
    escaped_symbol.push('\"');
    for c in symbol.chars() {
        match c {
            '\n' => escaped_symbol.push_str("\\\n"),
            '"' => escaped_symbol.push_str("\\\""),
            '\\' => escaped_symbol.push_str("\\\\"),
            c => escaped_symbol.push(c),
        }
    }
    escaped_symbol.push('\"');
    escaped_symbol
}