rustc_codegen_llvm/

common.rs

//! Code that is useful in various codegen modules.

use std::borrow::Borrow;

use libc::c_uint;
use rustc_abi::Primitive::Pointer;
use rustc_abi::{self as abi, ExternAbi, HasDataLayout as _};
use rustc_ast::Mutability;
use rustc_codegen_ssa::traits::*;
use rustc_crate_store::DllImport;
use rustc_data_structures::stable_hash::{StableHash, StableHasher};
use rustc_hashes::Hash128;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::DefId;
use rustc_middle::mir::interpret::{GlobalAlloc, PointerArithmetic, Scalar};
use rustc_middle::ty::{Instance, TyCtxt};
use rustc_session::{PointerAuthAddressDiscriminator, PointerAuthSchema};
use rustc_span::bug;
use tracing::debug;

use crate::consts::{IsInitOrFini, IsStatic, const_alloc_to_llvm};
pub(crate) use crate::context::CodegenCx;
use crate::context::{GenericCx, SCx};
use crate::llvm::{
    self, BasicBlock, ConstantInt, FALSE, TRUE, ToLlvmBool, Type, TypeKind, Value, const_ptr_auth,
};

pub(crate) fn maybe_sign_fn_ptr<'ll, 'tcx>(
    cx: &CodegenCx<'ll, '_>,
    instance: Instance<'tcx>,
    llfn: llvm::Value<'ll>,
    schema: &PointerAuthSchema,
) -> llvm::Value<'ll> {
    if cx.tcx.sess.pointer_authentication_functions().is_none() {
        return llfn;
    }

    // Only free functions or methods
    let def_id = instance.def_id();
    if !matches!(cx.tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) {
        return llfn;
    }
    // Only C ABI
    let abi = cx.tcx.fn_sig(def_id).skip_binder().abi();
    if !matches!(abi, ExternAbi::C { .. } | ExternAbi::System { .. }) {
        return llfn;
    }
    // Ignore LLVM intrinsics
    if llvm::get_value_name(llfn).starts_with(b"llvm.") {
        return llfn;
    }
    if Some(def_id) == cx.tcx.lang_items().eh_personality() {
        return llfn;
    }

    let addr_diversity = match schema.is_address_discriminated {
        PointerAuthAddressDiscriminator::HardwareAddress(true) => Some(llfn),
        PointerAuthAddressDiscriminator::HardwareAddress(false) => None,
        PointerAuthAddressDiscriminator::Synthetic(val) => {
            let llval = cx.const_u64(val);
            let llty = cx.val_ty(llfn);
            Some(unsafe { llvm::LLVMConstIntToPtr(llval, llty) })
        }
    };
    const_ptr_auth(llfn, schema.key as u32, schema.constant_discriminator as u64, addr_diversity)
}

/*
 * A note on nomenclature of linking: "extern", "foreign", and "upcall".
 *
 * An "extern" is an LLVM symbol we wind up emitting an undefined external
 * reference to. This means "we don't have the thing in this compilation unit,
 * please make sure you link it in at runtime". This could be a reference to
 * C code found in a C library, or rust code found in a rust crate.
 *
 * Most "externs" are implicitly declared (automatically) as a result of a
 * user declaring an extern _module_ dependency; this causes the rust driver
 * to locate an extern crate, scan its compilation metadata, and emit extern
 * declarations for any symbols used by the declaring crate.
 *
 * A "foreign" is an extern that references C (or other non-rust ABI) code.
 * There is no metadata to scan for extern references so in these cases either
 * a header-digester like bindgen, or manual function prototypes, have to
 * serve as declarators. So these are usually given explicitly as prototype
 * declarations, in rust code, with ABI attributes on them noting which ABI to
 * link via.
 *
 * An "upcall" is a foreign call generated by the compiler (not corresponding
 * to any user-written call in the code) into the runtime library, to perform
 * some helper task such as bringing a task to life, allocating memory, etc.
 *
 */

/// A structure representing an active landing pad for the duration of a basic
/// block.
///
/// Each `Block` may contain an instance of this, indicating whether the block
/// is part of a landing pad or not. This is used to make decision about whether
/// to emit `invoke` instructions (e.g., in a landing pad we don't continue to
/// use `invoke`) and also about various function call metadata.
///
/// For GNU exceptions (`landingpad` + `resume` instructions) this structure is
/// just a bunch of `None` instances (not too interesting), but for MSVC
/// exceptions (`cleanuppad` + `cleanupret` instructions) this contains data.
/// When inside of a landing pad, each function call in LLVM IR needs to be
/// annotated with which landing pad it's a part of. This is accomplished via
/// the `OperandBundleDef` value created for MSVC landing pads.
pub(crate) struct Funclet<'ll> {
    cleanuppad: Value<'ll>,
    operand: llvm::OperandBundleBox<'ll>,
}

impl<'ll> Funclet<'ll> {
    pub(crate) fn new(cleanuppad: Value<'ll>) -> Self {
        Funclet { cleanuppad, operand: llvm::OperandBundleBox::new("funclet", &[cleanuppad]) }
    }

    pub(crate) fn cleanuppad(&self) -> Value<'ll> {
        self.cleanuppad
    }

    pub(crate) fn bundle(&self) -> &llvm::OperandBundle<'ll> {
        self.operand.as_ref()
    }
}

impl<'ll, CX: Borrow<SCx<'ll>>> BackendTypes for GenericCx<'ll, CX> {
    // FIXME(eddyb) replace this with a `Function` "subclass" of `Value`.
    type Function = Value<'ll>;
    type BasicBlock = &'ll BasicBlock;
    type Funclet = Funclet<'ll>;

    type Value = Value<'ll>;
    type Type = Type<'ll>;
    type FunctionSignature = Type<'ll>;

    type DIScope = &'ll llvm::debuginfo::DIScope;
    type DILocation = &'ll llvm::debuginfo::DILocation;
    type DIVariable = &'ll llvm::debuginfo::DIVariable;
}

impl<'ll, CX: Borrow<SCx<'ll>>> GenericCx<'ll, CX> {
    pub(crate) fn const_array(&self, ty: Type<'ll>, elts: &[Value<'ll>]) -> Value<'ll> {
        let len = u64::try_from(elts.len()).expect("LLVMConstArray2 elements len overflow");
        unsafe { llvm::LLVMConstArray2(ty, elts.as_ptr(), len) }
    }

    pub(crate) fn const_bytes(&self, bytes: &[u8]) -> Value<'ll> {
        bytes_in_context(self.llcx(), bytes)
    }

    pub(crate) fn null_terminate_const_bytes(&self, bytes: &[u8]) -> Value<'ll> {
        null_terminate_bytes_in_context(self.llcx(), bytes)
    }

    pub(crate) fn const_get_elt(&self, v: Value<'ll>, idx: u64) -> Value<'ll> {
        unsafe {
            let idx = c_uint::try_from(idx).expect("LLVMGetAggregateElement index overflow");
            let r = llvm::LLVMGetAggregateElement(v, idx).unwrap();

            debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r);

            r
        }
    }

    pub(crate) fn const_null(&self, t: Type<'ll>) -> Value<'ll> {
        unsafe { llvm::LLVMConstNull(t) }
    }

    pub(crate) fn const_struct(&self, elts: &[Value<'ll>], packed: bool) -> Value<'ll> {
        struct_in_context(self.llcx(), elts, packed)
    }
}

impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
    pub(crate) fn alloc_to_backend(
        &self,
        global_alloc: GlobalAlloc<'tcx>,
        need_symbol_name: bool,
        schema: Option<&PointerAuthSchema>,
    ) -> Result<Value<'ll>, u64> {
        let alloc = match global_alloc {
            GlobalAlloc::Function { instance, .. } => {
                return Ok(self.get_fn_addr(instance, schema));
            }
            GlobalAlloc::Static(def_id) => {
                assert!(self.tcx.is_static(def_id));
                assert!(!self.tcx.is_thread_local_static(def_id));
                return Ok(
                    // `alloc_to_backend` might be called by `global_asm!` codegen. In which case
                    // `global_asm!` would need to find the renamed statics to use for symbol name.
                    self.renamed_statics
                        .borrow()
                        .get(&def_id)
                        .copied()
                        .unwrap_or_else(|| self.get_static(def_id)),
                );
            }
            GlobalAlloc::TypeId { .. } => {
                // Drop the provenance, the offset contains the bytes of the hash, so
                // just return 0 as base address.
                return Err(0);
            }

            GlobalAlloc::Memory(alloc) => {
                if alloc.inner().len() == 0 {
                    // For ZSTs directly codegen an aligned pointer.
                    // This avoids generating a zero-sized constant value and actually needing a
                    // real address at runtime.
                    return Err(alloc.inner().align.bytes());
                }

                alloc
            }
            GlobalAlloc::VTable(ty, dyn_ty) => {
                self.tcx
                    .global_alloc(self.tcx.vtable_allocation((
                        ty,
                        dyn_ty.principal().map(|principal| {
                            self.tcx.instantiate_bound_regions_with_erased(principal)
                        }),
                    )))
                    .unwrap_memory()
            }
        };

        let init = const_alloc_to_llvm(self, alloc.inner(), IsStatic::No, IsInitOrFini::No);
        let alloc = alloc.inner();

        if need_symbol_name {
            // If a symbol name is needed, use `static_addr_of_mut` so we can give it unique symbol names.
            let value = self.static_addr_of_mut(init, alloc.align, None);
            if alloc.mutability.is_not() {
                llvm::set_global_constant(value, true);
            }

            // Even though we're generating with internal linkage, this symbol name still needs to
            // be globally unique. LTO can rename symbol names if there are duplicates, but the
            // names inserted into global asm as text cannot be updated.
            let name = self.generate_global_symbol_name();
            llvm::set_value_name(value, name.as_bytes());
            llvm::set_linkage(value, llvm::Linkage::InternalLinkage);
            return Ok(value);
        }

        let value = match alloc.mutability {
            Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
            _ => self.static_addr_of_impl(init, alloc.align, None),
        };
        if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty() {
            let hash = self.tcx.with_stable_hashing_context(|mut hcx| {
                let mut hasher = StableHasher::new();
                alloc.stable_hash(&mut hcx, &mut hasher);
                hasher.finish::<Hash128>()
            });
            llvm::set_value_name(value, format!("alloc_{hash:032x}").as_bytes());
        }

        Ok(value)
    }
}

impl<'ll, 'tcx> ConstCodegenMethods for CodegenCx<'ll, 'tcx> {
    fn const_null(&self, t: Type<'ll>) -> Value<'ll> {
        unsafe { llvm::LLVMConstNull(t) }
    }

    fn const_undef(&self, t: Type<'ll>) -> Value<'ll> {
        unsafe { llvm::LLVMGetUndef(t) }
    }

    fn const_poison(&self, t: Type<'ll>) -> Value<'ll> {
        unsafe { llvm::LLVMGetPoison(t) }
    }

    fn const_bool(&self, val: bool) -> Value<'ll> {
        self.const_uint(self.type_i1(), val as u64)
    }

    fn const_i8(&self, i: i8) -> Value<'ll> {
        self.const_int(self.type_i8(), i as i64)
    }

    fn const_i16(&self, i: i16) -> Value<'ll> {
        self.const_int(self.type_i16(), i as i64)
    }

    fn const_i32(&self, i: i32) -> Value<'ll> {
        self.const_int(self.type_i32(), i as i64)
    }

    fn const_i64(&self, i: i64) -> Value<'ll> {
        self.const_int(self.type_i64(), i)
    }

    fn const_int(&self, t: Type<'ll>, i: i64) -> Value<'ll> {
        debug_assert!(
            self.type_kind(t) == TypeKind::Integer,
            "only allows integer types in const_int"
        );
        unsafe { llvm::LLVMConstInt(t, i as u64, TRUE) }
    }

    fn const_u8(&self, i: u8) -> Value<'ll> {
        self.const_uint(self.type_i8(), i as u64)
    }

    fn const_u32(&self, i: u32) -> Value<'ll> {
        self.const_uint(self.type_i32(), i as u64)
    }

    fn const_u64(&self, i: u64) -> Value<'ll> {
        self.const_uint(self.type_i64(), i)
    }

    fn const_u128(&self, i: u128) -> Value<'ll> {
        self.const_uint_big(self.type_i128(), i)
    }

    fn const_usize(&self, i: u64) -> Value<'ll> {
        let bit_size = self.data_layout().pointer_size().bits();
        if bit_size < 64 {
            // make sure it doesn't overflow
            assert!(i < (1 << bit_size));
        }

        self.const_uint(self.isize_ty, i)
    }

    fn const_uint(&self, t: Type<'ll>, i: u64) -> Value<'ll> {
        debug_assert!(
            self.type_kind(t) == TypeKind::Integer,
            "only allows integer types in const_uint"
        );
        unsafe { llvm::LLVMConstInt(t, i, FALSE) }
    }

    fn const_uint_big(&self, t: Type<'ll>, u: u128) -> Value<'ll> {
        debug_assert!(
            self.type_kind(t) == TypeKind::Integer,
            "only allows integer types in const_uint_big"
        );
        unsafe {
            let words = [u as u64, (u >> 64) as u64];
            llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr())
        }
    }

    fn const_real(&self, t: Type<'ll>, val: f64) -> Value<'ll> {
        unsafe { llvm::LLVMConstReal(t, val) }
    }

    fn const_str(&self, s: &str) -> (Value<'ll>, Value<'ll>) {
        let mut const_str_cache = self.const_str_cache.borrow_mut();
        let str_global = const_str_cache.get(s).copied().unwrap_or_else(|| {
            let sc = self.const_bytes(s.as_bytes());
            let sym = self.generate_local_symbol_name("str");
            let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| {
                bug!("symbol `{}` is already defined", sym);
            });
            llvm::set_initializer(g, sc);

            llvm::set_global_constant(g, true);
            llvm::set_unnamed_address(g, llvm::UnnamedAddr::Global);

            llvm::set_linkage(g, llvm::Linkage::InternalLinkage);
            // Cast to default address space if globals are in a different addrspace
            let g = self.const_pointercast(g, self.type_ptr());
            const_str_cache.insert(s.to_owned(), g);
            g
        });
        let len = s.len();
        (str_global, self.const_usize(len as u64))
    }

    fn const_struct(&self, elts: &[Value<'ll>], packed: bool) -> Value<'ll> {
        struct_in_context(self.llcx, elts, packed)
    }

    fn const_vector(&self, elts: &[Value<'ll>]) -> Value<'ll> {
        let len = c_uint::try_from(elts.len()).expect("LLVMConstVector elements len overflow");
        unsafe { llvm::LLVMConstVector(elts.as_ptr(), len) }
    }

    fn const_to_opt_uint(&self, v: Value<'ll>) -> Option<u64> {
        try_as_const_integral(v).and_then(|v| unsafe {
            let mut i = 0u64;
            let success = llvm::LLVMRustConstIntGetZExtValue(v, &mut i);
            success.then_some(i)
        })
    }

    fn const_to_opt_u128(&self, v: Value<'ll>, sign_ext: bool) -> Option<u128> {
        try_as_const_integral(v).and_then(|v| unsafe {
            let (mut lo, mut hi) = (0u64, 0u64);
            let success = llvm::LLVMRustConstInt128Get(v, sign_ext, &mut hi, &mut lo);
            success.then_some(hi_lo_to_u128(lo, hi))
        })
    }

    fn scalar_to_backend_with_pac(
        &self,
        cv: Scalar,
        layout: abi::Scalar,
        llty: Type<'ll>,
        schema: Option<&PointerAuthSchema>,
    ) -> Value<'ll> {
        let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() };
        match cv {
            Scalar::Int(int) => {
                let data = int.to_bits(layout.size(self));
                let llval = self.const_uint_big(self.type_ix(bitsize), data);
                if matches!(layout.primitive(), Pointer(_)) {
                    unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
                } else {
                    self.const_bitcast(llval, llty)
                }
            }
            Scalar::Ptr(ptr, _size) => {
                let (prov, offset) = ptr.prov_and_relative_offset();
                let global_alloc = self.tcx.global_alloc(prov.alloc_id());
                let base_addr_space = global_alloc.address_space(self);
                let base_addr = match self.alloc_to_backend(global_alloc, false, schema) {
                    Ok(base_addr) => base_addr,
                    Err(base_addr) => {
                        let val = base_addr.wrapping_add(offset.bytes());
                        let llval = self.const_usize(self.tcx.truncate_to_target_usize(val));
                        return if matches!(layout.primitive(), Pointer(_)) {
                            unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
                        } else {
                            self.const_bitcast(llval, llty)
                        };
                    }
                };
                let llval = unsafe {
                    llvm::LLVMConstInBoundsGEP2(
                        self.type_i8(),
                        // Cast to the required address space if necessary
                        self.const_pointercast(base_addr, self.type_ptr_ext(base_addr_space)),
                        &self.const_usize(offset.bytes()),
                        1,
                    )
                };
                if !matches!(layout.primitive(), Pointer(_)) {
                    unsafe { llvm::LLVMConstPtrToInt(llval, llty) }
                } else {
                    self.const_bitcast(llval, llty)
                }
            }
        }
    }

    fn const_ptr_byte_offset(&self, base_addr: Self::Value, offset: abi::Size) -> Self::Value {
        unsafe {
            llvm::LLVMConstInBoundsGEP2(
                self.type_i8(),
                base_addr,
                &self.const_usize(offset.bytes()),
                1,
            )
        }
    }
}

/// Get the [LLVM type][Type] of a [`Value`].
pub(crate) fn val_ty<'ll>(v: Value<'ll>) -> Type<'ll> {
    unsafe { llvm::LLVMTypeOf(v) }
}

pub(crate) fn bytes_in_context<'ll>(llcx: llvm::Context<'ll>, bytes: &[u8]) -> Value<'ll> {
    unsafe { llvm::LLVMConstStringInContext2(llcx, bytes.as_ptr(), bytes.len(), TRUE) }
}

pub(crate) fn null_terminate_bytes_in_context<'ll>(
    llcx: llvm::Context<'ll>,
    bytes: &[u8],
) -> Value<'ll> {
    unsafe { llvm::LLVMConstStringInContext2(llcx, bytes.as_ptr(), bytes.len(), FALSE) }
}

pub(crate) fn named_struct<'ll>(ty: Type<'ll>, elts: &[Value<'ll>]) -> Value<'ll> {
    let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow");
    unsafe { llvm::LLVMConstNamedStruct(ty, elts.as_ptr(), len) }
}

fn struct_in_context<'ll>(
    llcx: llvm::Context<'ll>,
    elts: &[Value<'ll>],
    packed: bool,
) -> Value<'ll> {
    let len = c_uint::try_from(elts.len()).expect("LLVMConstStructInContext elements len overflow");
    unsafe { llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), len, packed.to_llvm_bool()) }
}

#[inline]
fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 {
    ((hi as u128) << 64) | (lo as u128)
}

fn try_as_const_integral<'ll>(v: Value<'ll>) -> Option<&'ll ConstantInt> {
    unsafe { llvm::LLVMIsAConstantInt(v) }
}

pub(crate) fn get_dllimport<'tcx>(
    tcx: TyCtxt<'tcx>,
    id: DefId,
    name: &str,
) -> Option<&'tcx DllImport> {
    tcx.native_library(id)
        .and_then(|lib| lib.dll_imports.iter().find(|di| di.name.as_str() == name))
}