rustc_codegen_ssa/mir/

mod.rs

use std::iter;

use rustc_index::IndexVec;
use rustc_index::bit_set::DenseBitSet;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir;
use rustc_middle::mir::{Body, Local, UnwindTerminateReason, traversal};
use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout};
use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
use rustc_span::{ErrorGuaranteed, bug, span_bug};
use rustc_target::callconv::{FnAbi, IndirectMode, PassMode};
use tracing::{debug, instrument};

use crate::base;
use crate::traits::*;

mod analyze;
mod block;
mod constant;
mod coverageinfo;
pub mod debuginfo;
mod intrinsic;
mod locals;
pub mod naked_asm;
pub mod operand;
pub mod place;
mod retag;
mod rvalue;
mod statement;

pub use self::block::store_cast;
use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo};
use self::operand::{OperandRef, OperandValue};
use self::place::PlaceRef;

// Used for tracking the state of generated basic blocks.
enum CachedLlbb<T> {
    /// Nothing created yet.
    None,

    /// Has been created.
    Some(T),

    /// Nothing created yet, and nothing should be.
    Skip,
}

type PerLocalVarDebugInfoIndexVec<'tcx, V> =
    IndexVec<mir::Local, Vec<PerLocalVarDebugInfo<'tcx, V>>>;

/// Master context for codegenning from MIR.
pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
    instance: Instance<'tcx>,

    mir: &'tcx mir::Body<'tcx>,

    debug_context: Option<FunctionDebugContext<'tcx, Bx::DIScope, Bx::DILocation>>,

    llfn: Bx::Function,

    cx: &'a Bx::CodegenCx,

    fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,

    /// When unwinding is initiated, we have to store this personality
    /// value somewhere so that we can load it and re-use it in the
    /// resume instruction. The personality is (afaik) some kind of
    /// value used for C++ unwinding, which must filter by type: we
    /// don't really care about it very much. Anyway, this value
    /// contains an alloca into which the personality is stored and
    /// then later loaded when generating the DIVERGE_BLOCK.
    personality_slot: Option<PlaceRef<'tcx, Bx::Value>>,

    /// A backend `BasicBlock` for each MIR `BasicBlock`, created lazily
    /// as-needed (e.g. RPO reaching it or another block branching to it).
    // FIXME(eddyb) rename `llbbs` and other `ll`-prefixed things to use a
    // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbbs`).
    cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>>,

    /// The funclet status of each basic block
    cleanup_kinds: Option<IndexVec<mir::BasicBlock, analyze::CleanupKind>>,

    /// When targeting MSVC, this stores the cleanup info for each funclet BB.
    /// This is initialized at the same time as the `landing_pads` entry for the
    /// funclets' head block, i.e. when needed by an unwind / `cleanup_ret` edge.
    funclets: IndexVec<mir::BasicBlock, Option<Bx::Funclet>>,

    /// This stores the cached landing/cleanup pad block for a given BB.
    // FIXME(eddyb) rename this to `eh_pads`.
    landing_pads: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>>,

    /// Cached unreachable block
    unreachable_block: Option<Bx::BasicBlock>,

    /// Cached terminate upon unwinding block and its reason. For non-wasm
    /// targets, there is at most one such block per function, stored at index
    /// `START_BLOCK`. For wasm targets, each funclet needs its own terminate
    /// block, indexed by the cleanup block that is the funclet's head.
    terminate_blocks: IndexVec<mir::BasicBlock, Option<(Bx::BasicBlock, UnwindTerminateReason)>>,

    /// A cold block is a block that is unlikely to be executed at runtime.
    cold_blocks: DenseBitSet<mir::BasicBlock>,

    nop_landing_pads: DenseBitSet<mir::BasicBlock>,

    /// The location where each MIR arg/var/tmp/ret is stored. This is
    /// usually an `PlaceRef` representing an alloca, but not always:
    /// sometimes we can skip the alloca and just store the value
    /// directly using an `OperandRef`, which makes for tighter LLVM
    /// IR. The conditions for using an `OperandRef` are as follows:
    ///
    /// - the type of the local must be judged "immediate" by `is_llvm_immediate`
    /// - the operand must never be referenced indirectly
    ///     - we should not take its address using the `&` operator
    ///     - nor should it appear in a place path like `tmp.a`
    /// - the operand must be defined by an rvalue that can generate immediate
    ///   values
    ///
    /// Avoiding allocs can also be important for certain intrinsics,
    /// notably `expect`.
    locals: locals::Locals<'tcx, Bx::Value>,

    /// All `VarDebugInfo` from the MIR body, partitioned by `Local`.
    /// This is `None` if no variable debuginfo/names are needed.
    per_local_var_debug_info: Option<PerLocalVarDebugInfoIndexVec<'tcx, Bx::DIVariable>>,

    /// Caller location propagated if this function has `#[track_caller]`.
    caller_location: Option<OperandRef<'tcx, Bx::Value>>,
}

impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
    pub fn monomorphize<T>(&self, value: T) -> T
    where
        T: Copy + TypeFoldable<TyCtxt<'tcx>>,
    {
        debug!("monomorphize: self.instance={:?}", self.instance);
        self.instance.instantiate_mir_and_normalize_erasing_regions(
            self.cx.tcx(),
            self.cx.typing_env(),
            ty::EarlyBinder::bind(self.cx.tcx(), value),
        )
    }
}

enum LocalRef<'tcx, V> {
    Place(PlaceRef<'tcx, V>),
    /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place).
    /// `*p` is the wide pointer that references the actual unsized place.
    ///
    /// MIR only supports unsized args, not dynamically-sized locals, so
    /// new unsized temps don't exist and we must reuse the referred-to place.
    ///
    /// FIXME: Since the removal of unsized locals in <https://github.com/rust-lang/rust/pull/142911>,
    /// can we maybe use `Place` here? Or refactor it in another way? There are quite a few
    /// `UnsizedPlace => bug` branches now.
    UnsizedPlace(PlaceRef<'tcx, V>),
    /// The backend [`OperandValue`] has already been generated.
    Operand(OperandRef<'tcx, V>),
    /// Will be a `Self::Operand` once we get to its definition.
    PendingOperand,
}

pub enum IntrinsicResult<'tcx, V> {
    /// This intrinsic created an operand without using the `result_place` argument.
    ///
    /// `codegen_call_terminator` will handle writing the result into the place,
    /// if doing so is needed.
    ///
    /// The vast majority of intrinsics can do this, see MCP#970
    Operand(OperandValue<V>),

    /// The intrinsic wrote its result into the `result_place` argument.
    ///
    /// Most things don't need to do this, but there are some: `volatile_load`
    /// of a non-scalar type, for example, has to.
    WroteIntoPlace,

    /// Another instance should be called instead. This is used to invoke intrinsic
    /// default bodies in case an intrinsic is not implemented by the backend.
    Fallback(ty::Instance<'tcx>),

    /// Arguably this shouldn't exist, per MCP#620, but a bunch do it.
    Err(ErrorGuaranteed),
}

impl<'tcx, V: CodegenObject> LocalRef<'tcx, V> {
    fn new_operand(layout: TyAndLayout<'tcx>) -> LocalRef<'tcx, V> {
        if layout.is_zst() {
            // Zero-size temporaries aren't always initialized, which
            // doesn't matter because they don't contain data, but
            // we need something sufficiently aligned in the operand.
            LocalRef::Operand(OperandRef::zero_sized(layout))
        } else {
            LocalRef::PendingOperand
        }
    }
}

///////////////////////////////////////////////////////////////////////////

#[instrument(level = "debug", skip(cx))]
pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
    cx: &'a Bx::CodegenCx,
    instance: Instance<'tcx>,
) {
    assert!(!instance.args.has_infer());

    let tcx = cx.tcx();
    let llfn = cx.get_fn(instance);

    let mut mir = tcx.instance_mir(instance.def);
    // Note that the ABI logic has deduced facts about the functions' parameters based on the MIR we
    // got here (`deduce_param_attrs`). That means we can *not* apply arbitrary further MIR
    // transforms as that may invalidate those deduced facts!

    let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
    debug!("fn_abi: {:?}", fn_abi);

    let nop_landing_pads = tcx.find_noop_landing_pads_for_instance(mir, instance, cx.typing_env());

    if tcx.features().ergonomic_clones() {
        let monomorphized_mir = instance.instantiate_mir_and_normalize_erasing_regions(
            tcx,
            cx.typing_env(),
            ty::EarlyBinder::bind(tcx, mir.clone()),
        );
        mir = tcx.arena.alloc(optimize_use_clone::<Bx>(cx, monomorphized_mir));
    }

    let start_llbb = Bx::append_block(cx, llfn, "start");
    let mut start_bx = Bx::build(cx, start_llbb);

    if mir::traversal::mono_reachable(&mir, tcx, instance).any(|(bb, block)| {
        (block.is_cleanup && !nop_landing_pads.contains(bb))
            || matches!(block.terminator().unwind(), Some(mir::UnwindAction::Terminate(_)))
    }) {
        start_bx.set_personality_fn(cx.eh_personality());
    }

    let cleanup_kinds = base::wants_new_eh_instructions(&tcx.sess.target)
        .then(|| analyze::cleanup_kinds(&mir, &nop_landing_pads));

    let cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>> =
        mir.basic_blocks
            .indices()
            .map(|bb| {
                if bb == mir::START_BLOCK { CachedLlbb::Some(start_llbb) } else { CachedLlbb::None }
            })
            .collect();

    let mut fx = FunctionCx {
        instance,
        mir,
        llfn,
        fn_abi,
        cx,
        personality_slot: None,
        cached_llbbs,
        unreachable_block: None,
        terminate_blocks: IndexVec::from_elem(None, &mir.basic_blocks),
        cleanup_kinds,
        landing_pads: IndexVec::from_elem(None, &mir.basic_blocks),
        funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()),
        cold_blocks: find_cold_blocks(tcx, mir),
        locals: locals::Locals::empty(),
        debug_context: None,
        per_local_var_debug_info: None,
        caller_location: None,
        nop_landing_pads,
    };

    // It may seem like we should iterate over `required_consts` to ensure they all successfully
    // evaluate; however, the `MirUsedCollector` already did that during the collection phase of
    // monomorphization, and if there is an error during collection then codegen never starts -- so
    // we don't have to do it again.

    fx.fill_function_debug_context(&mut start_bx);

    let (per_local_var_debug_info, consts_debug_info) =
        fx.compute_per_local_var_debug_info(&mut start_bx).unzip();
    fx.per_local_var_debug_info = per_local_var_debug_info;

    let mut traversal_order = traversal::mono_reachable_reverse_postorder(mir, tcx, instance);

    // Filter out blocks that won't be codegen'd because of nop_landing_pads optimization.
    // FIXME: We might want to integrate the nop_landing_pads analysis into mono reachability.
    {
        let mut reachable = DenseBitSet::new_empty(mir.basic_blocks.len());
        let mut to_visit = vec![mir::START_BLOCK];
        while let Some(next) = to_visit.pop() {
            if !reachable.insert(next) {
                continue;
            }

            let block = &mir.basic_blocks[next];
            let successors = block.mono_successors(tcx, instance);

            if let Some(mir::UnwindAction::Cleanup(target)) = block.terminator().unwind()
                && fx.nop_landing_pads.contains(*target)
            {
                // This edge will not be followed when we actually codegen, so skip generating it here.
                //
                // It's guaranteed that the cleanup block (`target`) occurs only in
                // UnwindAction::Cleanup(...) -- i.e., we can't incorrectly filter too much here --
                // because cleanup transitions must happen via UnwindAction::Cleanup.
                to_visit.extend(successors.filter(|s| s != target));
            } else {
                to_visit.extend(successors);
            }
        }

        traversal_order.retain(|bb| reachable.contains(*bb));
    }

    let memory_locals = analyze::non_ssa_locals(&fx, &traversal_order);

    // Allocate variable and temp allocas
    let local_values = {
        let args = arg_local_refs(&mut start_bx, &mut fx, &memory_locals);

        let mut allocate_local = |local: Local| {
            let decl = &mir.local_decls[local];
            let layout = start_bx.layout_of(fx.monomorphize(decl.ty));
            assert!(!layout.ty.has_erasable_regions());

            if local == mir::RETURN_PLACE {
                match fx.fn_abi.ret.mode {
                    PassMode::Indirect { .. } => {
                        debug!("alloc: {:?} (return place) -> place", local);
                        let llretptr = start_bx.get_param(0);
                        return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
                    }
                    PassMode::Cast { ref cast, .. } => {
                        debug!("alloc: {:?} (return place) -> place", local);
                        let size = cast.size(&start_bx).max(layout.size);
                        return LocalRef::Place(PlaceRef::alloca_size(&mut start_bx, size, layout));
                    }
                    PassMode::IndirectUnsized { .. } => bug!("unsized returns are not supported"),
                    _ => {}
                };
            }

            if memory_locals.contains(local) {
                debug!("alloc: {:?} -> place", local);
                if layout.is_unsized() {
                    LocalRef::UnsizedPlace(PlaceRef::alloca_unsized_indirect(&mut start_bx, layout))
                } else {
                    LocalRef::Place(PlaceRef::alloca(&mut start_bx, layout))
                }
            } else {
                debug!("alloc: {:?} -> operand", local);
                LocalRef::new_operand(layout)
            }
        };

        let retptr = allocate_local(mir::RETURN_PLACE);
        iter::once(retptr)
            .chain(args.into_iter())
            .chain(mir.vars_and_temps_iter().map(allocate_local))
            .collect()
    };
    fx.initialize_locals(local_values);

    // Apply debuginfo to the newly allocated locals.
    fx.debug_introduce_locals(&mut start_bx, consts_debug_info.unwrap_or_default());

    // The builders will be created separately for each basic block at `codegen_block`.
    // So drop the builder of `start_llbb` to avoid having two at the same time.
    drop(start_bx);

    let mut unreached_blocks = DenseBitSet::new_filled(mir.basic_blocks.len());
    // Codegen the body of each reachable block using our reverse postorder list.
    for bb in traversal_order {
        fx.codegen_block(bb);
        unreached_blocks.remove(bb);
    }

    // FIXME: These empty unreachable blocks are *mostly* a waste. They are occasionally
    // targets for a SwitchInt terminator, but the reimplementation of the mono-reachable
    // simplification in SwitchInt lowering sometimes misses cases that
    // mono_reachable_reverse_postorder manages to figure out.
    // The solution is to do something like post-mono GVN. But for now we have this hack.
    for bb in unreached_blocks.iter() {
        fx.codegen_block_as_unreachable(bb);
    }
}

/// Replace `clone` calls that come from `use` statements with direct copies if possible.
// FIXME: Move this function to mir::transform when post-mono MIR passes land.
fn optimize_use_clone<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
    cx: &'a Bx::CodegenCx,
    mut mir: Body<'tcx>,
) -> Body<'tcx> {
    let tcx = cx.tcx();

    if tcx.features().ergonomic_clones() {
        for bb in mir.basic_blocks.as_mut() {
            let mir::TerminatorKind::Call {
                args,
                destination,
                target,
                call_source: mir::CallSource::Use,
                ..
            } = &bb.terminator().kind
            else {
                continue;
            };

            // CallSource::Use calls always use 1 argument.
            assert_eq!(args.len(), 1);
            let arg = &args[0];

            // These types are easily available from locals, so check that before
            // doing DefId lookups to figure out what we're actually calling.
            let arg_ty = arg.node.ty(&mir.local_decls, tcx);

            let ty::Ref(_region, inner_ty, mir::Mutability::Not) = *arg_ty.kind() else { continue };

            if !tcx.type_is_copy_modulo_regions(cx.typing_env(), inner_ty) {
                continue;
            }

            let Some(arg_place) = arg.node.place() else { continue };

            let destination_block = target.unwrap();

            bb.statements.push(mir::Statement::new(
                bb.terminator().source_info,
                mir::StatementKind::Assign(Box::new((
                    *destination,
                    mir::Rvalue::Use(
                        mir::Operand::Copy(
                            arg_place.project_deeper(&[mir::ProjectionElem::Deref], tcx),
                        ),
                        mir::WithRetag::Yes,
                    ),
                ))),
            ));

            bb.terminator_mut().kind = mir::TerminatorKind::goto(destination_block);
        }
    }

    mir
}

/// Produces, for each argument, a `Value` pointing at the
/// argument's value. As arguments are places, these are always
/// indirect.
fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
    bx: &mut Bx,
    fx: &mut FunctionCx<'a, 'tcx, Bx>,
    memory_locals: &DenseBitSet<mir::Local>,
) -> Vec<LocalRef<'tcx, Bx::Value>> {
    let mir = fx.mir;
    let mut idx = 0;
    let mut llarg_idx = fx.fn_abi.ret.is_indirect() as usize;

    let mut num_untupled = None;

    let codegen_fn_attrs = bx.tcx().codegen_instance_attrs(fx.instance.def);
    if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) {
        return vec![];
    }

    let mut args = mir
        .args_iter()
        .enumerate()
        .map(|(arg_index, local)| {
            let arg_decl = &mir.local_decls[local];
            let arg_ty = fx.monomorphize(arg_decl.ty);

            // FIXME(splat): re-tuple splatted arguments that were un-tupled in the ABI
            if Some(local) == mir.spread_arg {
                // This argument (e.g., the last argument in the "rust-call" ABI)
                // is a tuple that was spread at the ABI level and now we have
                // to reconstruct it into a tuple local variable, from multiple
                // individual LLVM function arguments.
                let ty::Tuple(tupled_arg_tys) = arg_ty.kind() else {
                    bug!("spread argument isn't a tuple?!");
                };

                let layout = bx.layout_of(arg_ty);

                // FIXME: support unsized params in "rust-call" ABI
                if layout.is_unsized() {
                    span_bug!(
                        arg_decl.source_info.span,
                        "\"rust-call\" ABI does not support unsized params",
                    );
                }

                let place = PlaceRef::alloca(bx, layout);
                for i in 0..tupled_arg_tys.len() {
                    let arg = &fx.fn_abi.args[idx];
                    idx += 1;
                    if let PassMode::Cast { pad_i32_count, .. } = arg.mode {
                        llarg_idx += usize::from(pad_i32_count);
                    }
                    let pr_field = place.project_field(bx, i);
                    bx.store_fn_arg(arg, &mut llarg_idx, pr_field);
                }
                assert_eq!(
                    None,
                    num_untupled.replace(tupled_arg_tys.len()),
                    "Replaced existing num_untupled"
                );

                return LocalRef::Place(place);
            }

            if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() {
                let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty));

                // Explicitly start the lifetime of the `va_list`, improves LLVM codegen.
                bx.lifetime_start(va_list.val.llval, va_list.layout.size);

                bx.va_start(va_list.val.llval);

                return LocalRef::Place(va_list);
            }

            let arg = &fx.fn_abi.args[idx];
            idx += 1;
            if let PassMode::Cast { pad_i32_count, .. } = arg.mode {
                llarg_idx += usize::from(pad_i32_count);
            }

            if !memory_locals.contains(local) {
                // We don't have to cast or keep the argument in the alloca.
                // FIXME(eddyb): We should figure out how to use llvm.dbg.value instead
                // of putting everything in allocas just so we can use llvm.dbg.declare.
                let local = |op| LocalRef::Operand(op);
                match arg.mode {
                    PassMode::Ignore => {
                        return local(OperandRef::zero_sized(arg.layout));
                    }
                    PassMode::Direct(_) => {
                        let llarg = bx.get_param(llarg_idx);
                        llarg_idx += 1;
                        debug_assert!(arg.layout.backend_repr.is_scalar_or_simd());
                        return local(OperandRef {
                            val: OperandValue::Immediate(llarg),
                            layout: arg.layout,
                            move_annotation: None,
                        });
                    }
                    PassMode::Pair(..) => {
                        let (a, b) = (bx.get_param(llarg_idx), bx.get_param(llarg_idx + 1));
                        llarg_idx += 2;

                        return local(OperandRef {
                            val: OperandValue::Pair(a, b),
                            layout: arg.layout,
                            move_annotation: None,
                        });
                    }
                    _ => {}
                }
            }

            match arg.mode {
                // Sized indirect arguments
                PassMode::Indirect { attrs, address_space: _, mode } => {
                    // Don't copy an indirect argument to an alloca, the caller already put it
                    // in a temporary alloca and gave it up.
                    // AmdgpuKernelArg/byref arguments must not be modified, so always create a
                    // local alloca for them.
                    // If the argument is underaligned, then we need to copy it to a higher-aligned
                    // alloca.
                    // FIXME: lifetimes
                    let mut needs_alloca = mode == IndirectMode::AmdgpuKernelArg;
                    if let Some(pointee_align) = attrs.pointee_align
                        && pointee_align < arg.layout.align
                    {
                        needs_alloca = true;
                    }
                    if needs_alloca {
                        let tmp = PlaceRef::alloca(bx, arg.layout);
                        bx.store_fn_arg(arg, &mut llarg_idx, tmp);
                        LocalRef::Place(tmp)
                    } else {
                        let llarg = bx.get_param(llarg_idx);
                        llarg_idx += 1;
                        LocalRef::Place(PlaceRef::new_sized(llarg, arg.layout))
                    }
                }
                // Unsized indirect arguments
                PassMode::IndirectUnsized { attrs: _, meta_attrs: _ } => {
                    // As the storage for the indirect argument lives during
                    // the whole function call, we just copy the wide pointer.
                    let llarg = bx.get_param(llarg_idx);
                    llarg_idx += 1;
                    let llextra = bx.get_param(llarg_idx);
                    llarg_idx += 1;
                    let indirect_operand = OperandValue::Pair(llarg, llextra);

                    let tmp = PlaceRef::alloca_unsized_indirect(bx, arg.layout);
                    indirect_operand.store(bx, tmp);
                    LocalRef::UnsizedPlace(tmp)
                }
                _ => {
                    let tmp = PlaceRef::alloca(bx, arg.layout);
                    bx.store_fn_arg(arg, &mut llarg_idx, tmp);
                    LocalRef::Place(tmp)
                }
            }
        })
        .collect::<Vec<_>>();
    if bx.tcx().sess.opts.unstable_opts.codegen_emit_retag.is_some() {
        args = args
            .iter()
            .map(|arg| match arg {
                &LocalRef::Place(place_ref) => {
                    fx.codegen_retag_place(bx, place_ref, true);
                    LocalRef::Place(place_ref)
                }
                &LocalRef::UnsizedPlace(place_ref) => {
                    let operand = bx.load_operand(place_ref);
                    let retagged = fx.codegen_retag_operand(bx, operand, true);
                    assert!(matches!(retagged.val, OperandValue::Pair(_, _)));
                    retagged.val.store(bx, place_ref);
                    LocalRef::UnsizedPlace(place_ref)
                }
                &LocalRef::Operand(operand_ref) => {
                    let retagged = fx.codegen_retag_operand(bx, operand_ref, true);
                    LocalRef::Operand(retagged)
                }
                LocalRef::PendingOperand => LocalRef::PendingOperand,
            })
            .collect::<Vec<_>>();
        // If we branched during retagging, then we need to update the
        // start block to the new location.
        fx.cached_llbbs[mir::START_BLOCK] = CachedLlbb::Some(bx.llbb());
    }

    if fx.instance.def.requires_caller_location(bx.tcx()) {
        let mir_args = if let Some(num_untupled) = num_untupled {
            // Subtract off the tupled argument that gets 'expanded'
            args.len() - 1 + num_untupled
        } else {
            args.len()
        };
        assert_eq!(
            fx.fn_abi.args.len(),
            mir_args + 1,
            "#[track_caller] instance {:?} must have 1 more argument in their ABI than in their MIR",
            fx.instance
        );

        let arg = fx.fn_abi.args.last().unwrap();
        match arg.mode {
            PassMode::Direct(_) => (),
            _ => bug!("caller location must be PassMode::Direct, found {:?}", arg.mode),
        }

        fx.caller_location = Some(OperandRef {
            val: OperandValue::Immediate(bx.get_param(llarg_idx)),
            layout: arg.layout,
            move_annotation: None,
        });
    }

    args
}

fn find_cold_blocks<'tcx>(
    tcx: TyCtxt<'tcx>,
    mir: &mir::Body<'tcx>,
) -> DenseBitSet<mir::BasicBlock> {
    let local_decls = &mir.local_decls;

    let mut cold_blocks: DenseBitSet<mir::BasicBlock> =
        DenseBitSet::new_empty(mir.basic_blocks.len());

    // Traverse all basic blocks from end of the function to the start.
    for (bb, bb_data) in traversal::postorder(mir) {
        let terminator = bb_data.terminator();

        match terminator.kind {
            // If a BB ends with a call to a cold function, mark it as cold.
            mir::TerminatorKind::Call { ref func, .. }
            | mir::TerminatorKind::TailCall { ref func, .. }
                if let ty::FnDef(def_id, ..) = *func.ty(local_decls, tcx).kind()
                    && let attrs = tcx.codegen_fn_attrs(def_id)
                    && attrs.flags.contains(CodegenFnAttrFlags::COLD) =>
            {
                cold_blocks.insert(bb);
                continue;
            }

            // If a BB ends with an `unreachable`, also mark it as cold.
            mir::TerminatorKind::Unreachable => {
                cold_blocks.insert(bb);
                continue;
            }

            _ => {}
        }

        // If all successors of a BB are cold and there's at least one of them, mark this BB as cold
        let mut succ = terminator.successors();
        if let Some(first) = succ.next()
            && cold_blocks.contains(first)
            && succ.all(|s| cold_blocks.contains(s))
        {
            cold_blocks.insert(bb);
        }
    }

    cold_blocks
}