rustc_const_eval/interpret/

call.rs

//! Manages calling a concrete function (with known MIR body) with argument passing,
//! and returning the return value to the caller.

use std::assert_matches;
use std::borrow::Cow;

use either::{Left, Right};
use rustc_abi::{self as abi, ExternAbi, FieldIdx, Integer, VariantIdx};
use rustc_attr_ir::find_attr;
use rustc_hir::def_id::DefId;
use rustc_middle::mir;
use rustc_middle::ty::layout::{IntegerExt, TyAndLayout};
use rustc_middle::ty::{self, AdtDef, FieldDef, Instance, Ty, VariantDef};
use rustc_span::{bug, span_bug};
use rustc_target::callconv::{ArgAbi, FnAbi};
use tracing::field::Empty;
use tracing::{info, instrument, trace};

use super::{
    CtfeProvenance, EnteredTraceSpan, FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine,
    OpTy, PlaceTy, Projectable, Provenance, RetagMode, ReturnAction, ReturnContinuation, Scalar,
    interp_ok, throw_ub, throw_ub_format,
};
use crate::enter_trace_span;

/// An argument passed to a function.
#[derive(Clone, Debug)]
pub enum FnArg<'tcx, Prov: Provenance = CtfeProvenance> {
    /// Pass a copy of the given operand.
    Copy(OpTy<'tcx, Prov>),
    /// Allow for the argument to be passed in-place: destroy the value originally stored at that
    /// place and make the place inaccessible for the duration of the function call. This *must* be
    /// an in-memory place so that we can do the proper alias checks.
    InPlace(MPlaceTy<'tcx, Prov>),
}

impl<'tcx, Prov: Provenance> FnArg<'tcx, Prov> {
    pub fn layout(&self) -> &TyAndLayout<'tcx> {
        match self {
            FnArg::Copy(op) => &op.layout,
            FnArg::InPlace(mplace) => &mplace.layout,
        }
    }

    /// Make a copy of the given fn_arg. Any `InPlace` are degenerated to copies, no protection of the
    /// original memory occurs.
    pub fn copy_fn_arg(&self) -> OpTy<'tcx, Prov> {
        match self {
            FnArg::Copy(op) => op.clone(),
            FnArg::InPlace(mplace) => mplace.clone().into(),
        }
    }
}

impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
    /// Make a copy of the given fn_args. Any `InPlace` are degenerated to copies, no protection of the
    /// original memory occurs.
    pub fn copy_fn_args(args: &[FnArg<'tcx, M::Provenance>]) -> Vec<OpTy<'tcx, M::Provenance>> {
        args.iter().map(|fn_arg| fn_arg.copy_fn_arg()).collect()
    }

    /// Helper function for argument untupling.
    fn fn_arg_project_field(
        &self,
        arg: &FnArg<'tcx, M::Provenance>,
        field: FieldIdx,
    ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> {
        interp_ok(match arg {
            FnArg::Copy(op) => FnArg::Copy(self.project_field(op, field)?),
            FnArg::InPlace(mplace) => FnArg::InPlace(self.project_field(mplace, field)?),
        })
    }

    /// Returns whether the given type has trivial ABI.
    fn has_trivial_abi(&self, layout: TyAndLayout<'tcx>) -> InterpResult<'tcx, bool> {
        if !layout.is_1zst() {
            return interp_ok(false);
        }
        match *layout.ty.kind() {
            // Trivially trivial-ABI types (because Rust makes no promises about their ABI).
            ty::Tuple(..)
            | ty::Never
            | ty::FnDef(..)
            | ty::Closure(..)
            | ty::Coroutine(..)
            | ty::CoroutineClosure(..) => interp_ok(true),

            ty::Array(elem, _len) => {
                // 0-length arrays are in general *not* okay, but arrays of trivial-ABI types are.
                self.has_trivial_abi(self.layout_of(elem)?)
            }
            ty::Adt(adt_def, _args) => {
                if adt_def.repr().transparent() {
                    // All fields must have trivial ABI.
                    (0..layout.fields.count()).try_fold(true, |acc, idx| {
                        interp_ok(acc && self.has_trivial_abi(layout.field(self, idx))?)
                    })
                } else if adt_def.repr().c() {
                    interp_ok(false)
                } else {
                    // Can't be SIMD or Scalable (since this is a 1-ZST); only Rust is left.
                    assert!(adt_def.repr().rust());
                    interp_ok(true)
                }
            }
            // Types that are considered transparent in `unfold_transparent` should also act
            // like transparent types here.
            ty::Pat(base, _) => self.has_trivial_abi(self.layout_of(base)?),
            ty::UnsafeBinder(bound_ty) => {
                let ty = self.tcx.instantiate_bound_regions_with_erased(bound_ty.into());
                self.has_trivial_abi(self.layout_of(ty)?)
            }

            ty::Alias(..) => panic!("non-normalized type"),
            _ => interp_ok(false),
        }
    }

    /// Find the wrapped inner type of a transparent wrapper by going for the unique
    /// non-trivial-ABI field.
    ///
    /// We work with `TyAndLayout` here since that makes it much easier to iterate over all fields.
    fn unfold_transparent(
        &self,
        layout: TyAndLayout<'tcx>,
        may_unfold: impl Fn(AdtDef<'tcx>) -> bool,
    ) -> InterpResult<'tcx, TyAndLayout<'tcx>> {
        match *layout.ty.kind() {
            ty::Adt(adt_def, _) if adt_def.repr().transparent() && may_unfold(adt_def) => {
                assert_matches!(layout.variants, rustc_abi::Variants::Single { .. });
                // Look for non-trivial-ABI field(s).
                let mut found = None;
                for idx in 0..layout.fields.count() {
                    let field = layout.field(self, idx);
                    if self.has_trivial_abi(field)? {
                        continue;
                    }
                    // Found a non-trivial ABI field!
                    if found.is_some() {
                        // There is more than one such field.
                        // FIXME: we should just panic here. But currently such repr(transparent)
                        // types are still accepted. We just don't treat them as transparent.
                        return interp_ok(layout);
                    }
                    found = Some(field);
                }
                let Some(field) = found else {
                    // All fields have trivial ABI. That means this type is effectively `()`.
                    return interp_ok(self.layout_of(self.tcx.types.unit)?);
                };
                // Recurse.
                self.unfold_transparent(field, may_unfold)
            }
            ty::Pat(base, _) => self.unfold_transparent(self.layout_of(base)?, may_unfold),
            ty::UnsafeBinder(bound_ty) => {
                let ty = self.tcx.instantiate_bound_regions_with_erased(bound_ty.into());
                self.unfold_transparent(self.layout_of(ty)?, may_unfold)
            }
            // Not a transparent type, no further unfolding.
            _ => interp_ok(layout),
        }
    }

    /// Unwrap types that are guaranteed a null-pointer-optimization
    fn unfold_npo(&self, layout: TyAndLayout<'tcx>) -> InterpResult<'tcx, TyAndLayout<'tcx>> {
        // Check if this is an option-like type wrapping some type.
        let ty::Adt(def, args) = layout.ty.kind() else {
            // Not an ADT, so definitely no NPO.
            return interp_ok(layout);
        };
        if def.variants().len() != 2 {
            // Not a 2-variant enum, so no NPO.
            return interp_ok(layout);
        }
        assert!(def.is_enum());

        let all_fields_1zst = |variant: &VariantDef| -> InterpResult<'tcx, _> {
            for field in &variant.fields {
                let ty = field.ty(*self.tcx, *args).skip_norm_wip();
                let layout = self.layout_of(ty)?;
                if !layout.is_1zst() {
                    return interp_ok(false);
                }
            }
            interp_ok(true)
        };

        // If one variant consists entirely of 1-ZST, then the other variant
        // is the only "relevant" one for this check.
        let var0 = VariantIdx::from_u32(0);
        let var1 = VariantIdx::from_u32(1);
        let relevant_variant = if all_fields_1zst(def.variant(var0))? {
            def.variant(var1)
        } else if all_fields_1zst(def.variant(var1))? {
            def.variant(var0)
        } else {
            // No variant is all-1-ZST, so no NPO.
            return interp_ok(layout);
        };
        // The "relevant" variant must have exactly one field, and its type is the "inner" type.
        if relevant_variant.fields.len() != 1 {
            return interp_ok(layout);
        }
        let inner =
            relevant_variant.fields[FieldIdx::from_u32(0)].ty(*self.tcx, *args).skip_norm_wip();
        let inner = self.layout_of(inner)?;

        // Check if the inner type is one of the NPO-guaranteed ones.
        // For that we first unpeel transparent *structs* (but not unions).
        let is_npo =
            |def: AdtDef<'tcx>| find_attr!(self.tcx, def.did(), RustcNonnullOptimizationGuaranteed);
        let inner = self.unfold_transparent(inner, /* may_unfold */ |def| {
            // Stop at NPO types so that we don't miss that attribute in the check below!
            def.is_struct() && !is_npo(def)
        })?;
        interp_ok(match inner.ty.kind() {
            ty::Ref(..) | ty::FnPtr(..) => {
                // Option<&T> behaves like &T, and same for fn()
                inner
            }
            ty::Adt(def, _) if is_npo(*def) => {
                // Once we found a `nonnull_optimization_guaranteed` type, further strip off
                // newtype structs from it to find the underlying ABI type.
                self.unfold_transparent(inner, /* may_unfold */ |def| def.is_struct())?
            }
            _ => {
                // Everything else we do not unfold.
                layout
            }
        })
    }

    /// Determine whether the given types are identical types from the perspective of C.
    pub(super) fn identical_c_types(&self, caller_type: Ty<'tcx>, callee_type: Ty<'tcx>) -> bool {
        if caller_type == callee_type {
            return true;
        }

        // C considers two structs to be the same if they have the same name and the same
        // fields. We need a similar rules that that e.g. if caller and callee use the "same"
        // type from two different versions of the same crate, that call is accepted.
        let ty::Adt(caller_adt, caller_args) = caller_type.kind() else { return false };
        let ty::Adt(callee_adt, callee_args) = callee_type.kind() else { return false };

        // We want to reject compatibility between structs with and without `#[cfi_encoding]` attr,
        // see https://github.com/rust-lang/rust/pull/164739#discussion_r4360457828
        #[derive(PartialEq, Eq)]
        enum AbiNameClass {
            CfiEncoded,
            Normal,
        }

        let abi_name = |adt: AdtDef<'tcx>| {
            find_attr!(
                self.tcx,
                adt.did(),
                CfiEncoding { encoding } => (AbiNameClass::CfiEncoded, *encoding)
            )
            .unwrap_or_else(|| (AbiNameClass::Normal, self.tcx.item_name(adt.did())))
        };

        if !(
            // They must both be structs.
            (caller_adt.is_struct() && callee_adt.is_struct())
            // They must have equal `repr`, and it must be `repr(C)`.
            && (caller_adt.repr().c() && caller_adt.repr().equal_up_to_seed(&callee_adt.repr()))
            // They must have the same name.
            && abi_name(*caller_adt) == abi_name(*callee_adt)
        ) {
            return false;
        }

        // All fields must have the same names and types as well, where "same type" recursively
        // uses this check.
        let caller_fields = &caller_adt.non_enum_variant().fields;
        let callee_fields = &callee_adt.non_enum_variant().fields;
        caller_fields.len() == callee_fields.len()
            && caller_fields.iter().zip(callee_fields).all(|(caller_field, callee_field)| {
                if caller_field.name != callee_field.name {
                    return false; // Bail on different name.
                }
                // Ensure the normalized type is the same.
                let normalized_field_ty = |field: &FieldDef, args| {
                    self.tcx.normalize_erasing_regions(self.typing_env, field.ty(*self.tcx, args))
                };
                let caller_ty = normalized_field_ty(caller_field, *caller_args);
                let callee_ty = normalized_field_ty(callee_field, *callee_args);
                self.identical_c_types(caller_ty, callee_ty)
            })
    }

    /// Check if these two layouts look like they are fn-ABI-compatible.
    /// (We also compare the `PassMode`, so this doesn't have to check everything. But it turns out
    /// that only checking the `PassMode` is insufficient.)
    fn layout_compat(
        &self,
        caller: TyAndLayout<'tcx>,
        callee: TyAndLayout<'tcx>,
    ) -> InterpResult<'tcx, bool> {
        // Fast path: equal types are definitely compatible.
        if caller.ty == callee.ty {
            return interp_ok(true);
        }
        // Handle trivial-ABI types.
        if self.has_trivial_abi(caller)? && self.has_trivial_abi(callee)? {
            return interp_ok(true);
        }
        // Unfold newtypes and NPO optimizations.
        let unfold = |layout: TyAndLayout<'tcx>| {
            self.unfold_transparent(layout, /* may_unfold */ |_def| true)
                .and_then(|f| self.unfold_npo(f))
        };
        let caller = unfold(caller)?;
        let callee = unfold(callee)?;
        // Not-quite-so-fast path: if the types are c-equal now, they are compatible.
        // FIXME: This is *not* currently guaranteed by our ABI compatibility docs, but it is needed
        // for Miri itself when it checks whether shims were called with the right arguments.
        // We should eventually put this into the docs as well.
        if self.identical_c_types(caller.ty, callee.ty) {
            return interp_ok(true);
        }
        // Now see if these inner types are compatible.

        // Compatible pointer types. For thin pointers, we have to accept even non-`repr(transparent)`
        // things as compatible due to `DispatchFromDyn`. For instance, `Rc<i32>` and `*mut i32`
        // must be compatible. So we just accept everything with Pointer ABI as compatible,
        // even if this will accept some code that is not stably guaranteed to work.
        // This also handles function pointers.
        let thin_pointer = |layout: TyAndLayout<'tcx>| match layout.backend_repr {
            abi::BackendRepr::Scalar(s) => match s.primitive() {
                abi::Primitive::Pointer(addr_space) => Some(addr_space),
                _ => None,
            },
            _ => None,
        };
        if let (Some(caller), Some(callee)) = (thin_pointer(caller), thin_pointer(callee)) {
            return interp_ok(caller == callee);
        }
        // For wide pointers we have to get the pointee type.
        let pointee_ty = |ty: Ty<'tcx>| -> InterpResult<'tcx, Option<Ty<'tcx>>> {
            // We cannot use `builtin_deref` here since we need to reject `Box<T, MyAlloc>`.
            interp_ok(Some(match ty.kind() {
                ty::Ref(_, ty, _) => *ty,
                ty::RawPtr(ty, _) => *ty,
                // We only accept `Box` with the default allocator.
                _ if ty.is_box_global(*self.tcx) => ty.expect_boxed_ty(),
                _ => return interp_ok(None),
            }))
        };
        if let (Some(caller), Some(callee)) = (pointee_ty(caller.ty)?, pointee_ty(callee.ty)?) {
            // This is okay if they have the same metadata type.
            let meta_ty = |ty: Ty<'tcx>| {
                // Even if `ty` is normalized, the search for the unsized tail will project
                // to fields, which can yield non-normalized types. So we need to provide a
                // normalization function.
                let normalize = |ty| self.tcx.normalize_erasing_regions(self.typing_env, ty);
                ty.ptr_metadata_ty(*self.tcx, normalize)
            };
            return interp_ok(meta_ty(caller) == meta_ty(callee));
        }

        // Compatible integer types (in particular, usize vs ptr-sized-u32/u64).
        // `char` counts as `u32.`
        let int_ty = |ty: Ty<'tcx>| {
            Some(match ty.kind() {
                ty::Int(ity) => (Integer::from_int_ty(&self.tcx, *ity), /* signed */ true),
                ty::Uint(uty) => (Integer::from_uint_ty(&self.tcx, *uty), /* signed */ false),
                ty::Char => (Integer::I32, /* signed */ false),
                _ => return None,
            })
        };
        if let (Some(caller), Some(callee)) = (int_ty(caller.ty), int_ty(callee.ty)) {
            // This is okay if they are the same integer type.
            return interp_ok(caller == callee);
        }

        // The rest is incompatible.
        interp_ok(false)
    }

    /// Returns a `bool` saying whether the two arguments are ABI-compatible.
    pub fn check_argument_compat(
        &self,
        caller_abi: &ArgAbi<'tcx, Ty<'tcx>>,
        callee_abi: &ArgAbi<'tcx, Ty<'tcx>>,
    ) -> InterpResult<'tcx, bool> {
        // We do not want to accept things as ABI-compatible that just "happen to be" compatible on the current target,
        // so we implement a type-based check that reflects the guaranteed rules for ABI compatibility.
        if self.layout_compat(caller_abi.layout, callee_abi.layout)? {
            // Ensure that our checks imply actual ABI compatibility for this concrete call.
            // (This can fail e.g. if `#[rustc_nonnull_optimization_guaranteed]` is used incorrectly.)
            assert!(
                caller_abi.eq_abi(callee_abi),
                "these types were deemed ABI-compatible but do not actually have compatible `FnAbi`:\ncaller: {caller_abi:#?}\ncallee: {callee_abi:#?}"
            );
            interp_ok(true)
        } else {
            trace!(
                "check_argument_compat: incompatible ABIs:\ncaller: {:?}\ncallee: {:?}",
                caller_abi, callee_abi
            );
            interp_ok(false)
        }
    }

    /// Initialize a single callee argument, checking the types for compatibility.
    fn pass_argument<'x, 'y>(
        &mut self,
        caller_args: &mut impl Iterator<
            Item = (&'x FnArg<'tcx, M::Provenance>, &'y ArgAbi<'tcx, Ty<'tcx>>),
        >,
        callee_args_abis: &mut impl Iterator<Item = (usize, &'y ArgAbi<'tcx, Ty<'tcx>>)>,
        callee_arg: &mir::Place<'tcx>,
        callee_ty: Ty<'tcx>,
        already_live: bool,
    ) -> InterpResult<'tcx>
    where
        'tcx: 'x,
        'tcx: 'y,
    {
        // Get next callee arg.
        let (callee_arg_idx, callee_abi) = callee_args_abis.next().unwrap();
        assert_eq!(callee_ty, callee_abi.layout.ty);
        // Get next caller arg.
        let Some((caller_arg, caller_abi)) = caller_args.next() else {
            throw_ub_format!("calling a function with fewer arguments than it requires");
        };
        assert_eq!(caller_arg.layout().layout, caller_abi.layout.layout);
        // Sadly we cannot assert that `caller_arg.layout().ty` and `caller_abi.layout.ty` are
        // equal; in closures the types sometimes differ. We just hope that `caller_abi` is the
        // right type to print to the user.

        // Check compatibility
        if !self.check_argument_compat(caller_abi, callee_abi)? {
            throw_ub!(AbiMismatchArgument {
                arg_idx: callee_arg_idx,
                caller_ty: caller_abi.layout.ty,
                callee_ty: callee_abi.layout.ty
            });
        }
        // We work with a copy of the argument for now; if this is in-place argument passing, we
        // will later protect the source it comes from. This means the callee cannot observe if we
        // did in-place of by-copy argument passing, except for pointer equality tests.
        let caller_arg_copy = caller_arg.copy_fn_arg();
        if !already_live {
            let local = callee_arg.as_local().unwrap();
            let meta = caller_arg_copy.meta();
            // `check_argument_compat` ensures that if metadata is needed, both have the same type,
            // so we know they will use the metadata the same way.
            assert!(!meta.has_meta() || caller_arg_copy.layout.ty == callee_ty);

            self.storage_live_dyn(local, meta)?;
        }
        // Now we can finally actually evaluate the callee place.
        let callee_arg =
            self.eval_place(*callee_arg, /* skip_validity_for_simple_deref */ false)?;
        // We allow some transmutes here.
        // FIXME: Depending on the PassMode, this should reset some padding to uninitialized. (This
        // is true for all `copy_op`, but there are a lot of special cases for argument passing
        // specifically.)
        self.copy_op_allow_transmute(&caller_arg_copy, &callee_arg)?;
        // If this was an in-place pass, protect the place it comes from for the duration of the call.
        if let FnArg::InPlace(mplace) = caller_arg {
            M::protect_in_place_function_argument(self, mplace)?;
        }
        interp_ok(())
    }

    /// The main entry point for creating a new stack frame: performs ABI checks and initializes
    /// arguments.
    #[instrument(skip(self), level = "trace")]
    pub fn init_stack_frame(
        &mut self,
        instance: Instance<'tcx>,
        body: &'tcx mir::Body<'tcx>,
        caller_fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
        args: &[FnArg<'tcx, M::Provenance>],
        with_caller_location: bool,
        destination: &PlaceTy<'tcx, M::Provenance>,
        mut cont: ReturnContinuation,
    ) -> InterpResult<'tcx> {
        let _trace = enter_trace_span!(M, step::init_stack_frame, %instance, tracing_separate_thread = Empty);
        let def_id = instance.def_id();

        match (instance.def.requires_caller_location(*self.tcx), with_caller_location) {
            (false, true) => {
                throw_ub_format!(
                    "calling a non-`#[track_caller]` function from a `#[track_caller]` declaration"
                )
            }
            (true, false) => {
                throw_ub_format!(
                    "calling a `#[track_caller]` function from a non-`#[track_caller]` declaration"
                )
            }
            _ => {}
        }
        let caller_location = with_caller_location.then(|| self.caller_location());

        // The first order of business is to figure out the callee signature.
        // We leave the list of extra c-variadic args empty because the callee cannot know about
        // them, and we don't need them in the `FnAbi`. This also avoids ICEs if the callee is
        // not actually c-variadic.
        let extra_args = ty::List::empty();
        let callee_fn_abi = self.fn_abi_of_instance_no_deduced_attrs(instance, extra_args)?;

        if caller_fn_abi.conv != callee_fn_abi.conv {
            throw_ub_format!(
                "calling a function with calling convention \"{callee_conv}\" using calling convention \"{caller_conv}\"",
                callee_conv = callee_fn_abi.conv,
                caller_conv = caller_fn_abi.conv,
            )
        }

        if caller_fn_abi.c_variadic != callee_fn_abi.c_variadic {
            throw_ub!(CVariadicMismatch {
                caller_is_c_variadic: caller_fn_abi.c_variadic,
                callee_is_c_variadic: callee_fn_abi.c_variadic,
            });
        }
        if caller_fn_abi.c_variadic && caller_fn_abi.fixed_count != callee_fn_abi.fixed_count {
            throw_ub!(CVariadicFixedCountMismatch {
                caller: caller_fn_abi.fixed_count,
                callee: callee_fn_abi.fixed_count,
            });
        }

        // Check that all target features required by the callee (i.e., from
        // the attribute `#[target_feature(enable = ...)]`) are enabled at
        // compile time.
        M::check_fn_target_features(self, instance)?;

        // If the signature says this cannot unwind, reflect this in the unwind destination so that
        // we don't have to check this later. (`init_fn_call` already did this for the caller so
        // here we only have to check the callee.)
        if !callee_fn_abi.can_unwind {
            match &mut cont {
                ReturnContinuation::Stop { .. } => {}
                ReturnContinuation::Goto { unwind, .. } => {
                    *unwind = mir::UnwindAction::Unreachable;
                }
            }
        }

        // *Before* pushing the new frame, determine whether the return destination is in memory.
        // Need to use `place_to_op` to be *sure* we get the mplace if there is one.
        let destination_mplace = self.place_to_op(destination)?.as_mplace_or_imm().left();

        // Push the "raw" frame -- this leaves locals uninitialized.
        self.push_stack_frame_raw(instance, body, destination, cont)?;
        let preamble_span = self.frame().loc.unwrap_right(); // the span used for preamble errors

        trace!(
            "caller ABI: {:#?}, args: {:#?}",
            caller_fn_abi,
            args.iter()
                .map(|arg| (
                    arg.layout().ty,
                    match arg {
                        FnArg::Copy(op) => format!("copy({op:?})"),
                        FnArg::InPlace(mplace) => format!("in-place({mplace:?})"),
                    }
                ))
                .collect::<Vec<_>>()
        );
        trace!(
            "spread_arg: {:?}, locals: {:#?}",
            body.spread_arg,
            body.args_iter()
                .map(|local| (local, self.layout_of_local(self.frame(), local, None).unwrap().ty))
                .collect::<Vec<_>>()
        );

        // Initialize special caller_location "local", if this function has it.
        self.frame_mut().track_caller_arg = caller_location;

        // Determine whether there is a special VaList argument. This is always the
        // last argument, and since arguments start at index 1 that's `arg_count`.
        let va_list_arg = callee_fn_abi.c_variadic.then(|| mir::Local::from_usize(body.arg_count));
        // Determine whether this is a non-capturing closure. That's relevant as their first
        // argument can be skipped (and that's the only kind of argument skipping we allow).
        let is_non_capturing_closure =
            (matches!(instance.def, ty::InstanceKind::Shim(ty::ShimKind::ClosureOnce { .. }))
                || self.tcx.is_closure_like(def_id))
                && {
                    let arg = &callee_fn_abi.args[0];
                    matches!(arg.layout.ty.kind(), ty::Closure (_def, closure_args) if {
                        closure_args.as_closure().upvar_tys().is_empty()
                    })
                };

        // In principle, we have two iterators: Where the arguments come from, and where
        // they go to.

        // The "where they come from" part is easy, we expect the caller to do any special handling
        // that might be required here (e.g. for untupling).
        // If `with_caller_location` is set we pretend there is an extra argument (that
        // we will not pass normally; our `caller_location` intrinsic implementation handles
        // that separately).
        assert_eq!(
            args.len() + if with_caller_location { 1 } else { 0 },
            caller_fn_abi.args.len(),
            "mismatch between caller ABI and caller arguments",
        );
        let mut caller_args = args.iter().zip(caller_fn_abi.args.iter());

        // Now we have to spread them out across the callee's locals,
        // taking into account the `spread_arg`. If we could write
        // this is a single iterator (that handles `spread_arg`), then
        // `pass_argument` would be the loop body.
        let mut callee_args_abis = callee_fn_abi.args.iter().enumerate();
        // During argument passing, we want retagging with protectors.
        M::with_retag_mode(self, RetagMode::FnEntry, |ecx| {
            for local in body.args_iter() {
                // Update the span that we show in case of an error to point to this argument.
                ecx.frame_mut().loc = Right(body.local_decls[local].source_info.span);
                // Construct the destination place for this argument. At this point all
                // locals are still dead, so we cannot construct a `PlaceTy`.
                let dest = mir::Place::from(local);
                // `layout_of_local` does more than just the instantiation we need to get the
                // type, but the result gets cached so this avoids calling the instantiation
                // query *again* the next time this local is accessed.
                let ty = ecx.layout_of_local(ecx.frame(), local, None)?.ty;

                // Some arguments are special: the first (`self`) argument of a non-capturing
                // closure; the va_list argument; and the spread_arg.
                if is_non_capturing_closure && local == mir::Local::arg(0) {
                    assert!(va_list_arg.is_none());
                    assert!(Some(local) != body.spread_arg);
                    // This argument might be missing on the caller side. So just initialize it in
                    // the callee.
                    let (callee_arg_idx, callee_abi) = callee_args_abis.next().unwrap();
                    assert!(callee_abi.layout.is_1zst() && callee_abi.is_ignore());
                    ecx.storage_live(local)?;
                    // And skip it in the caller, if present. We can tell whether it is present by
                    // comparing the number of arguments on the caller and callee side.
                    if caller_fn_abi.args.len() == callee_fn_abi.args.len() {
                        let (_caller_arg, caller_abi) = caller_args.next().unwrap();
                        if !caller_abi.layout.is_1zst() {
                            // The caller gave us some other, non-ignorable argument.
                            throw_ub!(AbiMismatchArgument {
                                arg_idx: callee_arg_idx,
                                caller_ty: caller_abi.layout.ty,
                                callee_ty: callee_abi.layout.ty
                            });
                        }
                        assert!(caller_abi.is_ignore());
                    }
                } else if Some(local) == va_list_arg {
                    // This is the last callee-side argument of a variadic function.
                    // This argument is a VaList holding the remaining caller-side arguments.
                    ecx.storage_live(local)?;

                    let place =
                        ecx.eval_place(dest, /* skip_validity_for_simple_deref */ false)?;
                    let mplace = ecx.force_allocation(&place)?;

                    // There should be no more callee ABI arguments.
                    assert!(callee_args_abis.next().is_none());
                    // Consume the remaining arguments by putting them into the variable argument
                    // list. We disable retagging to avoid creating protected tags. Protection should
                    // only use callee-side information, and the varargs have no static callee-side type.
                    let varargs = M::with_retag_mode(ecx, RetagMode::None, |ecx| {
                        ecx.allocate_varargs(&mut caller_args)
                    })?;

                    // When the frame is dropped, these variable arguments are deallocated.
                    ecx.frame_mut().va_list = varargs.clone();
                    let key = ecx.va_list_ptr(varargs.into());

                    // Zero the VaList, so it is fully initialized.
                    ecx.write_bytes_ptr(
                        mplace.ptr(),
                        (0..mplace.layout.size.bytes()).map(|_| 0u8),
                    )?;

                    // Store the "key" pointer in the right field.
                    let key_mplace = ecx.va_list_key_field(&mplace)?;
                    ecx.write_pointer(key, &key_mplace)?;
                } else if Some(local) == body.spread_arg {
                    // Make the local live once, then fill in the value field by field.
                    ecx.storage_live(local)?;
                    // Must be a tuple
                    let ty::Tuple(fields) = ty.kind() else {
                        span_bug!(ecx.cur_span(), "non-tuple type for `spread_arg`: {ty}")
                    };
                    for (i, field_ty) in fields.iter().enumerate() {
                        let dest = dest.project_deeper(
                            &[mir::ProjectionElem::Field(FieldIdx::from_usize(i), field_ty)],
                            *ecx.tcx,
                        );
                        ecx.pass_argument(
                            &mut caller_args,
                            &mut callee_args_abis,
                            &dest,
                            field_ty,
                            /* already_live */ true,
                        )?;
                    }
                } else {
                    // Normal argument. Cannot mark it as live yet, it might be unsized!
                    ecx.pass_argument(
                        &mut caller_args,
                        &mut callee_args_abis,
                        &dest,
                        ty,
                        /* already_live */ false,
                    )?;
                }
            }
            interp_ok(())
        })?;

        // Don't forget to check the return type!
        self.frame_mut().loc = Right(body.local_decls[mir::RETURN_PLACE].source_info.span);
        if !self.check_argument_compat(&caller_fn_abi.ret, &callee_fn_abi.ret)? {
            throw_ub!(AbiMismatchReturn {
                caller_ty: caller_fn_abi.ret.layout.ty,
                callee_ty: callee_fn_abi.ret.layout.ty
            });
        }
        // Protect return place for in-place return value passing.
        // We only need to protect anything if this is actually an in-memory place.
        if let Some(mplace) = destination_mplace {
            M::protect_in_place_function_argument(self, &mplace)?;
        }

        // For the final checks, use same span as preamble since it is unclear what else to do.
        self.frame_mut().loc = Right(preamble_span);
        // If the callee needs a caller location, pretend we consume one more argument from the ABI.
        if instance.def.requires_caller_location(*self.tcx) {
            callee_args_abis.next().unwrap();
        }
        // Now we should have no more caller args or callee arg ABIs.
        assert!(
            callee_args_abis.next().is_none(),
            "mismatch between callee ABI and callee body arguments"
        );
        if caller_args.next().is_some() {
            throw_ub_format!("calling a function with more arguments than it expected");
        }

        // All function arguments have gotten initialized, call corresponding hook.
        M::after_stack_frame_init(self)?;

        // Done!
        self.push_stack_frame_done()
    }

    /// Initiate a call to this function -- pushing the stack frame and initializing the arguments.
    ///
    /// `caller_fn_abi` is used to determine if all the arguments are passed the proper way.
    /// However, we also need `caller_abi` to determine if we need to do untupling of arguments.
    ///
    /// `with_caller_location` indicates whether the caller passed a caller location. Miri
    /// implements caller locations without argument passing, but to match `FnAbi` we need to know
    /// when those arguments are present.
    pub(super) fn init_fn_call(
        &mut self,
        fn_val: FnVal<'tcx, M::ExtraFnVal>,
        (caller_abi, caller_fn_abi): (ExternAbi, Option<&FnAbi<'tcx, Ty<'tcx>>>),
        args: &[FnArg<'tcx, M::Provenance>],
        with_caller_location: bool,
        destination: &PlaceTy<'tcx, M::Provenance>,
        target: Option<mir::BasicBlock>,
        mut unwind: mir::UnwindAction,
    ) -> InterpResult<'tcx> {
        let _trace =
            enter_trace_span!(M, step::init_fn_call, tracing_separate_thread = Empty, ?fn_val)
                .or_if_tracing_disabled(|| trace!("init_fn_call: {:#?}", fn_val));

        // If the signature says this cannot unwind, reflect this in the unwind destination
        // so that we don't have to check this later.
        if caller_fn_abi.is_some_and(|abi| !abi.can_unwind) {
            unwind = mir::UnwindAction::Unreachable;
        }

        let instance = match fn_val {
            FnVal::Instance(instance) => instance,
            FnVal::Other(extra) => {
                let caller_fn_abi =
                    caller_fn_abi.expect("FnAbi should have been computed for this call");
                return M::call_extra_fn(
                    self,
                    extra,
                    caller_fn_abi,
                    args,
                    destination,
                    target,
                    unwind,
                );
            }
        };

        match instance.def {
            ty::InstanceKind::Intrinsic(def_id) => {
                assert!(self.tcx.intrinsic(def_id).is_some());
                assert!(!with_caller_location);
                // FIXME: Should `InPlace` arguments be reset to uninit?
                if let Some(fallback) = M::call_intrinsic(
                    self,
                    instance,
                    &Self::copy_fn_args(args),
                    destination,
                    target,
                    unwind,
                )? {
                    assert!(!self.tcx.intrinsic(fallback.def_id()).unwrap().must_be_overridden);
                    assert_matches!(fallback.def, ty::InstanceKind::Item(_));
                    // For `#[rustc_intrinsic] fn`, two `Instance`s are involved:
                    // the original `InstanceKind::Intrinsic` (no caller location arg), and
                    // the fallback `InstanceKind::Item` (has caller location arg if
                    // it's `#[track_caller]`).
                    // We replace the former with the latter.
                    return self.init_fn_call(
                        FnVal::Instance(fallback),
                        (caller_abi, caller_fn_abi),
                        args,
                        fallback.def.requires_caller_location(*self.tcx),
                        destination,
                        target,
                        unwind,
                    );
                } else {
                    interp_ok(())
                }
            }
            ty::InstanceKind::LlvmIntrinsic(_) => {
                // FIXME: Should `InPlace` arguments be reset to uninit?
                M::call_llvm_intrinsic(
                    self,
                    instance,
                    &Self::copy_fn_args(args),
                    destination,
                    target,
                )
            }
            ty::InstanceKind::Shim(ty::ShimKind::VTable(..))
            | ty::InstanceKind::Shim(ty::ShimKind::Reify(..))
            | ty::InstanceKind::Shim(ty::ShimKind::ClosureOnce { .. })
            | ty::InstanceKind::Shim(ty::ShimKind::ConstructCoroutineInClosure { .. })
            | ty::InstanceKind::Shim(ty::ShimKind::FnPtr(..))
            | ty::InstanceKind::Shim(ty::ShimKind::DropGlue(..))
            | ty::InstanceKind::Shim(ty::ShimKind::Clone(..))
            | ty::InstanceKind::Shim(ty::ShimKind::FnPtrAsPtr(..))
            | ty::InstanceKind::Shim(ty::ShimKind::FnPtrFromPtr(..))
            | ty::InstanceKind::Shim(ty::ShimKind::ThreadLocal(..))
            | ty::InstanceKind::Shim(ty::ShimKind::AsyncDropGlueCtor(..))
            | ty::InstanceKind::Shim(ty::ShimKind::AsyncDropGlue(..))
            | ty::InstanceKind::Shim(ty::ShimKind::FutureDropPoll(..))
            | ty::InstanceKind::Item(_) => {
                // We need MIR for this fn.
                // Note that this can be an intrinsic, if we are executing its fallback body.
                let caller_fn_abi =
                    caller_fn_abi.expect("FnAbi should have been computed for this call");
                let Some((body, instance)) = M::find_mir_or_eval_fn(
                    self,
                    instance,
                    caller_fn_abi,
                    args,
                    destination,
                    target,
                    unwind,
                )?
                else {
                    return interp_ok(());
                };

                // Special handling for the closure ABI: untuple the last argument.
                // FIXME(splat): un-tuple splatted arguments that were tupled in typecheck
                let args: Cow<'_, [FnArg<'tcx, M::Provenance>]> =
                    if caller_abi == ExternAbi::RustCall && !args.is_empty() {
                        // Untuple
                        let (untuple_arg, args) = args.split_last().unwrap();
                        let ty::Tuple(untuple_fields) = untuple_arg.layout().ty.kind() else {
                            span_bug!(self.cur_span(), "untuple argument must be a tuple")
                        };
                        trace!("init_fn_call: Will pass last argument by untupling");
                        Cow::from(
                            args.iter()
                                // The regular arguments.
                                .map(|a| interp_ok(a.clone()))
                                // The fields of the untupled argument.
                                .chain((0..untuple_fields.len()).map(|i| {
                                    self.fn_arg_project_field(untuple_arg, FieldIdx::from_usize(i))
                                }))
                                .collect::<InterpResult<'_, Vec<_>>>()?,
                        )
                    } else {
                        // Plain arg passing
                        Cow::from(args)
                    };

                self.init_stack_frame(
                    instance,
                    body,
                    caller_fn_abi,
                    &args,
                    with_caller_location,
                    destination,
                    ReturnContinuation::Goto { ret: target, unwind },
                )
            }
            // `InstanceKind::Virtual` does not have callable MIR. Calls to `Virtual` instances must be
            // codegen'd / interpreted as virtual calls through the vtable.
            ty::InstanceKind::Virtual(def_id, idx) => {
                let caller_fn_abi =
                    caller_fn_abi.expect("FnAbi should have been computed for this call");
                let mut args = args.to_vec();
                // We have to implement all "dyn-compatible receivers". So we have to go search for a
                // pointer or `dyn Trait` type, but it could be wrapped in newtypes. So recursively
                // unwrap those newtypes until we are there.
                // An `InPlace` does nothing here, we keep the original receiver intact. We can't
                // really pass the argument in-place anyway, and we are constructing a new
                // `Immediate` receiver.
                let mut receiver = args[0].copy_fn_arg();
                let receiver_place = loop {
                    match receiver.layout.ty.kind() {
                        ty::Ref(..) | ty::RawPtr(..) => {
                            // We do *not* use `deref_pointer` here: we don't want to conceptually
                            // create a place that must be dereferenceable, since the receiver might
                            // be a raw pointer and (for `*const dyn Trait`) we don't need to
                            // actually access memory to resolve this method.
                            // Also see <https://github.com/rust-lang/miri/issues/2786>.
                            let val = self.read_immediate(&receiver)?;
                            break self.imm_ptr_to_mplace(&val)?;
                        }
                        ty::Dynamic(..) => break receiver.assert_mem_place(), // no immediate unsized values
                        _ => {
                            // Not there yet, search for the only non-ZST field.
                            // (The rules for `DispatchFromDyn` ensure there's exactly one such field.)
                            let (idx, _) = receiver.layout.non_1zst_field(self).expect(
                                "not exactly one non-1-ZST field in a `DispatchFromDyn` type",
                            );
                            receiver = self.project_field(&receiver, idx)?;
                        }
                    }
                };

                // Obtain the underlying trait we are working on, and the adjusted receiver argument.
                // Doesn't have to be a `dyn Trait`, but the unsized tail must be `dyn Trait`.
                // (For that reason we also cannot use `unpack_dyn_trait`.)
                let receiver_tail =
                    self.tcx.struct_tail_for_codegen(receiver_place.layout.ty, self.typing_env);
                let ty::Dynamic(receiver_trait, _) = receiver_tail.kind() else {
                    span_bug!(self.cur_span(), "dynamic call on non-`dyn` type {}", receiver_tail)
                };
                assert!(receiver_place.layout.is_unsized());

                // Get the required information from the vtable.
                let vptr = receiver_place.meta().unwrap_meta().to_pointer(self);
                let dyn_ty = self.get_ptr_vtable_ty(vptr, Some(receiver_trait))?;
                let adjusted_recv = receiver_place.ptr();

                // Now determine the actual method to call. Usually we use the easy way of just
                // looking up the method at index `idx`.
                let vtable_entries = self.vtable_entries(receiver_trait.principal(), dyn_ty);
                let Some(ty::VtblEntry::Method(fn_inst)) = vtable_entries.get(idx).copied() else {
                    // FIXME(fee1-dead) these could be variants of the UB info enum instead of this
                    throw_ub_format!("`dyn` call trying to call something that is not a method");
                };
                trace!("Virtual call dispatches to {fn_inst:#?}");
                // We can also do the lookup based on `def_id` and `dyn_ty`, and check that that
                // produces the same result.
                self.assert_virtual_instance_matches_concrete(dyn_ty, def_id, instance, fn_inst);

                // Adjust receiver argument. Layout can be any (thin) ptr.
                let receiver_ty = Ty::new_mut_ptr(self.tcx.tcx, dyn_ty);
                args[0] = FnArg::Copy(
                    ImmTy::from_immediate(
                        Scalar::from_maybe_pointer(adjusted_recv, self).into(),
                        self.layout_of(receiver_ty)?,
                    )
                    .into(),
                );
                trace!("Patched receiver operand to {:#?}", args[0]);
                // Need to also adjust the type in the ABI. Strangely, the layout there is actually
                // already fine! Just the type is bogus. This is due to what `force_thin_self_ptr`
                // does in `fn_abi_new_uncached`; supposedly, codegen relies on having the bogus
                // type, so we just patch this up locally.
                let mut caller_fn_abi = caller_fn_abi.clone();
                caller_fn_abi.args[0].layout.ty = receiver_ty;

                // recurse with concrete function
                self.init_fn_call(
                    FnVal::Instance(fn_inst),
                    (caller_abi, Some(&caller_fn_abi)),
                    &args,
                    with_caller_location,
                    destination,
                    target,
                    unwind,
                )
            }
        }
    }

    fn assert_virtual_instance_matches_concrete(
        &self,
        dyn_ty: Ty<'tcx>,
        def_id: DefId,
        virtual_instance: ty::Instance<'tcx>,
        concrete_instance: ty::Instance<'tcx>,
    ) {
        let tcx = *self.tcx;

        let trait_def_id = tcx.parent(def_id);
        let virtual_trait_ref = ty::TraitRef::from_assoc(tcx, trait_def_id, virtual_instance.args);
        let existential_trait_ref = ty::ExistentialTraitRef::erase_self_ty(tcx, virtual_trait_ref);
        let concrete_trait_ref = existential_trait_ref.with_self_ty(tcx, dyn_ty);

        let concrete_method = {
            let _trace = enter_trace_span!(M, resolve::expect_resolve_for_vtable, ?def_id);
            Instance::expect_resolve_for_vtable(
                tcx,
                self.typing_env,
                def_id,
                virtual_instance.args.rebase_onto(tcx, trait_def_id, concrete_trait_ref.args),
                self.cur_span(),
            )
        };
        assert_eq!(concrete_instance, concrete_method);
    }

    /// Initiate a tail call to this function -- popping the current stack frame, pushing the new
    /// stack frame and initializing the arguments.
    pub(super) fn init_fn_tail_call(
        &mut self,
        fn_val: FnVal<'tcx, M::ExtraFnVal>,
        (caller_abi, caller_fn_abi): (ExternAbi, Option<&FnAbi<'tcx, Ty<'tcx>>>),
        args: &[FnArg<'tcx, M::Provenance>],
        with_caller_location: bool,
    ) -> InterpResult<'tcx> {
        trace!("init_fn_tail_call: {:#?}", fn_val);
        // This is the "canonical" implementation of tails calls,
        // a pop of the current stack frame, followed by a normal call
        // which pushes a new stack frame, with the return address from
        // the popped stack frame.
        //
        // Note that we cannot use `return_from_current_stack_frame`,
        // as that "executes" the goto to the return block, but we don't want to,
        // only the tail called function should return to the current return block.

        // The arguments need to all be copied since the current stack frame will be removed
        // before the callee even starts executing.
        // FIXME(explicit_tail_calls,#144855): does this match what codegen does?
        let args = args.iter().map(|fn_arg| FnArg::Copy(fn_arg.copy_fn_arg())).collect::<Vec<_>>();
        // Remove the frame from the stack.
        let frame = self.pop_stack_frame_raw()?;
        // Remember where this frame would have returned to.
        let ReturnContinuation::Goto { ret, unwind } = frame.return_cont() else {
            bug!("can't tailcall as root of the stack");
        };
        // There's no return value to deal with! Instead, we forward the old return place
        // to the new function.
        // FIXME(explicit_tail_calls):
        //   we should check if both caller&callee can/n't unwind,
        //   see <https://github.com/rust-lang/rust/pull/113128#issuecomment-1614979803>

        // Now push the new stack frame.
        self.init_fn_call(
            fn_val,
            (caller_abi, caller_fn_abi),
            &*args,
            with_caller_location,
            frame.return_place(),
            ret,
            unwind,
        )?;

        // Finally, clear the local variables. Has to be done after pushing to support
        // non-scalar arguments.
        // FIXME(explicit_tail_calls,#144855): revisit this once codegen supports indirect
        // arguments, to ensure the semantics are compatible.
        let return_action = self.cleanup_stack_frame(/* unwinding */ false, frame)?;
        assert_eq!(return_action, ReturnAction::Normal);

        interp_ok(())
    }

    pub(super) fn init_drop_in_place_call(
        &mut self,
        place: &PlaceTy<'tcx, M::Provenance>,
        instance: ty::Instance<'tcx>,
        target: mir::BasicBlock,
        unwind: mir::UnwindAction,
    ) -> InterpResult<'tcx> {
        trace!("init_drop_in_place_call: {:?},\n  instance={:?}", place, instance);
        // We take the address of the object. This may well be unaligned, which is fine
        // for us here. However, unaligned accesses will probably make the actual drop
        // implementation fail -- a problem shared by rustc.
        let place = self.force_allocation(place)?;

        // We behave a bit different from codegen here.
        // Codegen creates an `InstanceKind::Virtual` with index 0 (the slot of the drop method) and
        // then dispatches that to the normal call machinery. However, our call machinery currently
        // only supports calling `VtblEntry::Method`; it would choke on a `MetadataDropInPlace`. So
        // instead we do the virtual call stuff ourselves. It's easier here than in `eval_fn_call`
        // since we can just get a place of the underlying type and use `mplace_to_imm_ptr`.
        let place = match place.layout.ty.kind() {
            ty::Dynamic(data, _) => {
                // Dropping a trait object. Need to find actual drop fn.
                self.unpack_dyn_trait(&place, data)?
            }
            _ => {
                debug_assert_eq!(
                    instance,
                    ty::Instance::resolve_drop_glue(*self.tcx, place.layout.ty)
                );
                place
            }
        };

        let instance = {
            let _trace = enter_trace_span!(M, resolve::resolve_drop_glue, ty = ?place.layout.ty);
            ty::Instance::resolve_drop_glue(*self.tcx, place.layout.ty)
        };
        let fn_abi = self.fn_abi_of_instance_no_deduced_attrs(instance, ty::List::empty())?;

        let ref_ty = Ty::new_mut_ref(self.tcx.tcx, self.tcx.lifetimes.re_erased, place.layout.ty);
        let arg = self.mplace_to_imm_ptr(&place, Some(ref_ty))?;

        let ret = MPlaceTy::fake_alloc_zst(self.layout_of(self.tcx.types.unit)?);

        self.init_fn_call(
            FnVal::Instance(instance),
            (ExternAbi::Rust, Some(fn_abi)),
            &[FnArg::Copy(arg.into())],
            false,
            &ret.into(),
            Some(target),
            unwind,
        )
    }

    /// Pops the current frame from the stack, copies the return value to the caller, deallocates
    /// the memory for allocated locals, and jumps to an appropriate place.
    ///
    /// If `unwinding` is `false`, then we are performing a normal return
    /// from a function. In this case, we jump back into the frame of the caller,
    /// and continue execution as normal.
    ///
    /// If `unwinding` is `true`, then we are in the middle of a panic,
    /// and need to unwind this frame. In this case, we jump to the
    /// `cleanup` block for the function, which is responsible for running
    /// `Drop` impls for any locals that have been initialized at this point.
    /// The cleanup block ends with a special `Resume` terminator, which will
    /// cause us to continue unwinding.
    #[instrument(skip(self), level = "trace")]
    pub(super) fn return_from_current_stack_frame(
        &mut self,
        unwinding: bool,
    ) -> InterpResult<'tcx> {
        info!(
            "popping stack frame ({})",
            if unwinding { "during unwinding" } else { "returning from function" }
        );

        // Check `unwinding`.
        assert_eq!(
            unwinding,
            match self.frame().loc {
                Left(loc) => self.body().basic_blocks[loc.block].is_cleanup,
                Right(_) => true,
            }
        );
        if unwinding && self.frame_idx() == 0 {
            throw_ub_format!("unwinding past the topmost frame of the stack");
        }

        // Get out the return value. Must happen *before* the frame is popped as we have to get the
        // local's value out.
        let return_op =
            self.local_to_op(mir::RETURN_PLACE, None).expect("return place should always be live");
        // Remove the frame from the stack.
        let frame = self.pop_stack_frame_raw()?;
        // Copy the return value and remember the return continuation.
        if !unwinding {
            self.copy_op_allow_transmute(&return_op, frame.return_place())?;
            trace!("return value: {:?}", self.dump_place(frame.return_place()));
        }
        let return_cont = frame.return_cont();
        // Finish popping the stack frame.
        let return_action = self.cleanup_stack_frame(unwinding, frame)?;
        // Jump to the next block.
        match return_action {
            ReturnAction::Normal => {}
            ReturnAction::NoJump => {
                // The hook already did everything.
                return interp_ok(());
            }
            ReturnAction::NoCleanup => {
                // If we are not doing cleanup, also skip everything else.
                assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked");
                assert!(!unwinding, "tried to skip cleanup during unwinding");
                // Don't jump anywhere.
                return interp_ok(());
            }
        }

        // Normal return, figure out where to jump.
        if unwinding {
            // Follow the unwind edge.
            match return_cont {
                ReturnContinuation::Goto { unwind, .. } => {
                    // This must be the very last thing that happens, since it can in fact push a new stack frame.
                    self.unwind_to_block(unwind)
                }
                ReturnContinuation::Stop { .. } => {
                    panic!("encountered ReturnContinuation::Stop when unwinding!")
                }
            }
        } else {
            // Follow the normal return edge.
            match return_cont {
                ReturnContinuation::Goto { ret, .. } => self.return_to_block(ret),
                ReturnContinuation::Stop { .. } => {
                    assert!(
                        self.stack().is_empty(),
                        "only the bottommost frame can have ReturnContinuation::Stop"
                    );
                    interp_ok(())
                }
            }
        }
    }
}