rustc_mir_transform/

elaborate_drop.rs

use std::{fmt, iter, mem};

use itertools::Itertools;
use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx};
use rustc_data_structures::thin_vec::ThinVec;
use rustc_hir::attrs::lang_items::LangItem;
use rustc_hir::{CoroutineDesugaring, CoroutineKind};
use rustc_index::Idx;
use rustc_middle::mir::*;
use rustc_middle::ty::adjustment::PointerCoercion;
use rustc_middle::ty::util::{Discr, IntTypeExt};
use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt};
use rustc_middle::{bug, span_bug};
use rustc_mir_dataflow::DropFlagState;
use rustc_span::{DUMMY_SP, dummy_spanned};
use tracing::{debug, instrument};

use crate::coroutine::CTX_ARG;
use crate::patch::MirPatch;

/// Describes how/if a value should be dropped.
#[derive(Debug)]
pub(crate) enum DropStyle {
    /// The value is already dead at the drop location, no drop will be executed.
    Dead,

    /// The value is known to always be initialized at the drop location, drop will always be
    /// executed.
    Static,

    /// Whether the value needs to be dropped depends on its drop flag.
    Conditional,

    /// An "open" drop is one where only the fields of a value are dropped.
    ///
    /// For example, this happens when moving out of a struct field: The rest of the struct will be
    /// dropped in such an "open" drop. It is also used to generate drop glue for the individual
    /// components of a value, for example for dropping array elements.
    Open,
}

/// Which drop flags to affect/check with an operation.
#[derive(Debug)]
pub(crate) enum DropFlagMode {
    /// Only affect the top-level drop flag, not that of any contained fields.
    Shallow,
    /// Affect all nested drop flags in addition to the top-level one.
    Deep,
}

/// Describes if unwinding is necessary and where to unwind to if a panic occurs.
#[derive(Copy, Clone, Debug)]
pub(crate) enum Unwind {
    /// Unwind to this block.
    To(BasicBlock),
    /// Already in an unwind path, any panic will cause an abort.
    InCleanup,
}

impl Unwind {
    fn is_cleanup(self) -> bool {
        match self {
            Unwind::To(..) => false,
            Unwind::InCleanup => true,
        }
    }

    fn into_action(self) -> UnwindAction {
        match self {
            Unwind::To(bb) => UnwindAction::Cleanup(bb),
            Unwind::InCleanup => UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
        }
    }

    fn map<F>(self, f: F) -> Self
    where
        F: FnOnce(BasicBlock) -> BasicBlock,
    {
        match self {
            Unwind::To(bb) => Unwind::To(f(bb)),
            Unwind::InCleanup => Unwind::InCleanup,
        }
    }
}

pub(crate) trait DropElaborator<'a, 'tcx>: fmt::Debug {
    /// The type representing paths that can be moved out of.
    ///
    /// Users can move out of individual fields of a struct, such as `a.b.c`. This type is used to
    /// represent such move paths. Sometimes tracking individual move paths is not necessary, in
    /// which case this may be set to (for example) `()`.
    type Path: Copy + fmt::Debug;

    // Accessors

    fn patch_ref(&self) -> &MirPatch<'tcx>;
    fn patch(&mut self) -> &mut MirPatch<'tcx>;
    fn body(&self) -> &'a Body<'tcx>;
    fn tcx(&self) -> TyCtxt<'tcx>;
    fn typing_env(&self) -> ty::TypingEnv<'tcx>;
    fn allow_async_drops(&self) -> bool;

    // Drop logic

    /// Returns how `path` should be dropped, given `mode`.
    fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle;

    /// Returns the drop flag of `path` as a MIR `Operand` (or `None` if `path` has no drop flag).
    fn get_drop_flag(&mut self, path: Self::Path) -> Option<Operand<'tcx>>;

    /// Return the drop flag of `path`, if any.
    ///
    /// If `mode` is deep, drop flags of all child paths should be returned.
    fn drop_flags_for(&mut self, path: Self::Path, mode: DropFlagMode) -> Vec<Place<'tcx>>;

    // Subpaths

    /// Returns the subpath of a field of `path` (or `None` if there is no dedicated subpath).
    ///
    /// If this returns `None`, `field` will not get a dedicated drop flag.
    fn field_subpath(&self, path: Self::Path, field: FieldIdx) -> Option<Self::Path>;

    /// Returns the subpath of a dereference of `path` (or `None` if there is no dedicated subpath).
    ///
    /// If this returns `None`, `*path` will not get a dedicated drop flag.
    ///
    /// This is only relevant for `Box<T>`, where the contained `T` can be moved out of the box.
    fn deref_subpath(&self, path: Self::Path) -> Option<Self::Path>;

    /// Returns the subpath of downcasting `path` to one of its variants.
    ///
    /// If this returns `None`, the downcast of `path` will not get a dedicated drop flag.
    fn downcast_subpath(&self, path: Self::Path, variant: VariantIdx) -> Option<Self::Path>;

    /// Returns the subpath of indexing a fixed-size array `path`.
    ///
    /// If this returns `None`, elements of `path` will not get a dedicated drop flag.
    ///
    /// This is only relevant for array patterns, which can move out of individual array elements.
    fn array_subpath(&self, path: Self::Path, index: u64, size: u64) -> Option<Self::Path>;
}

#[derive(Debug)]
struct DropCtxt<'a, 'b, 'tcx, D>
where
    D: DropElaborator<'b, 'tcx>,
{
    elaborator: &'a mut D,

    source_info: SourceInfo,

    place: Place<'tcx>,
    path: D::Path,
    succ: BasicBlock,
    unwind: Unwind,
    dropline: Option<BasicBlock>,
}

/// "Elaborates" a drop of `place`/`path` and patches `bb`'s terminator to execute it.
///
/// The passed `elaborator` is used to determine what should happen at the drop terminator. It
/// decides whether the drop can be statically determined or whether it needs a dynamic drop flag,
/// and whether the drop is "open", i.e. should be expanded to drop all subfields of the dropped
/// value.
///
/// When this returns, the MIR patch in the `elaborator` contains the necessary changes.
pub(crate) fn elaborate_drop<'b, 'tcx, D>(
    elaborator: &mut D,
    source_info: SourceInfo,
    place: Place<'tcx>,
    path: D::Path,
    succ: BasicBlock,
    unwind: Unwind,
    bb: BasicBlock,
    dropline: Option<BasicBlock>,
) where
    D: DropElaborator<'b, 'tcx>,
    'tcx: 'b,
{
    DropCtxt { elaborator, source_info, place, path, succ, unwind, dropline }.elaborate_drop(bb)
}

impl<'a, 'b, 'tcx, D> DropCtxt<'a, 'b, 'tcx, D>
where
    D: DropElaborator<'b, 'tcx>,
    'tcx: 'b,
{
    #[instrument(level = "trace", skip(self), ret)]
    fn place_ty(&self, place: Place<'tcx>) -> Ty<'tcx> {
        if place.local < self.elaborator.body().local_decls.next_index() {
            place.ty(self.elaborator.body(), self.tcx()).ty
        } else {
            // We don't have a slice with all the locals, since some are in the patch.
            PlaceTy::from_ty(self.elaborator.patch_ref().local_ty(place.local))
                .multi_projection_ty(self.elaborator.tcx(), place.projection)
                .ty
        }
    }

    fn tcx(&self) -> TyCtxt<'tcx> {
        self.elaborator.tcx()
    }

    /// Async-drop `place: drop_ty`.
    ///
    /// Conceptually, we want to run `async_drop_in_place(&mut obj).await`.
    ///
    /// Await syntax does not exist in MIR, so we need to manually expand it into a poll-yield
    /// loop, essentially:
    /// ```mir
    ///   let fut = async_drop_in_place(&mut obj);
    ///   loop {
    ///     let pin_fut = Pin::new_unchecked(&mut fut);
    ///     match Future::poll(pin_fut, CTX_ARG) {
    ///       Poll::Ready => break,
    ///       Poll::Pending(..) => CTX_ARG = yield (),
    ///     }
    ///   }
    ///   // continue to `succ`
    /// ```
    ///
    /// We also need to ensure that async drop also happens on the coroutine drop path, ie. when
    /// `yield` branches along its `drop` target. This requires a second loop, this time jumping to
    /// `dropline`.
    ///
    /// Arguments:
    ///   `call_destructor_only`: call only `AsyncDrop::drop`, not full `async_drop_in_place` glue
    #[instrument(level = "debug", skip(self), ret)]
    fn build_async_drop(
        &mut self,
        place: Place<'tcx>,
        drop_ty: Ty<'tcx>,
        succ: BasicBlock,
        unwind: Unwind,
        dropline: Option<BasicBlock>,
        call_destructor_only: bool,
    ) -> BasicBlock {
        let tcx = self.tcx();
        let span = self.source_info.span;
        let obj_ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, drop_ty);

        let async_drop_fn_def_id = if call_destructor_only {
            // Resolving obj.<AsyncDrop::drop>()
            let async_drop_trait = tcx.require_lang_item(LangItem::AsyncDrop, span);
            tcx.associated_item_def_ids(async_drop_trait)[0]
        } else {
            // Resolving async_drop_in_place<T> function for drop_ty
            tcx.require_lang_item(LangItem::AsyncDropInPlace, span)
        };

        let fut_ty = tcx
            .instantiate_bound_regions_with_erased(
                tcx.fn_sig(async_drop_fn_def_id)
                    .instantiate(tcx, &[drop_ty.into()])
                    .skip_norm_wip(),
            )
            .output();
        let fut = self.new_temp(fut_ty);

        // Create an intermediate block that does StorageDead(fut) then jumps to succ.
        // This is necessary because we do not want to modify statements
        // in existing blocks, in case those are used somewhere else in MIR.
        let succ_with_dead = self.new_block_with_statements(
            unwind,
            vec![self.storage_dead(fut)],
            TerminatorKind::Goto { target: succ },
        );
        let dropline_with_dead = dropline.map(|target| {
            self.new_block_with_statements(
                unwind,
                vec![self.storage_dead(fut)],
                TerminatorKind::Goto { target },
            )
        });
        let unwind_with_dead = unwind.map(|target| {
            self.new_block_with_statements(
                Unwind::InCleanup,
                vec![self.storage_dead(fut)],
                TerminatorKind::Goto { target },
            )
        });

        // The yielded value depends on the kind of coroutine, to match what AST lowering does.
        let coroutine_kind = self.elaborator.body().coroutine_kind().unwrap();
        let yield_value = match coroutine_kind {
            // For async gen, we need `yield Poll<OptRet>::Pending`.
            CoroutineKind::Desugared(CoroutineDesugaring::AsyncGen, _) => {
                let full_yield_ty = self.elaborator.body().yield_ty().unwrap();
                let ty::Adt(_poll_adt, args) = *full_yield_ty.kind() else { bug!() };
                let ty::Adt(_option_adt, args) = *args.type_at(0).kind() else { bug!() };
                let yield_ty = args.type_at(0);
                Operand::unevaluated_constant(
                    tcx,
                    tcx.require_lang_item(LangItem::AsyncGenPending, span),
                    tcx.mk_args(&[yield_ty.into()]),
                    span,
                )
            }
            // For regular async fn, we need `yield ()`.
            CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => {
                Operand::zero_sized_constant(tcx.types.unit, span)
            }
            // `is_async_drop` should have checked that.
            _ => panic!("unexpected coroutine for async drop {coroutine_kind:?}"),
        };

        // The branching here is tricky and deserves some explanation.
        //
        // If we are in the drop code path, ie. we are currently dropping the coroutine.
        // The state machine follows the `drop` branch in the `yield` terminator.
        // To repeatedly poll the future, the `drop` branch must loop.
        // Meanwhile, the `resume` branch corresponds to anomalous execution,
        // trying to resume the coroutine while it is being dropped. So that branch panics
        // (`panic_bb`).
        let panic_bb = self.build_resumed_after_drop_abort_block(unwind_with_dead, coroutine_kind);
        let (drop_pin_bb, drop_resume_bb, drop_drop_bb) = self.build_pin_poll_yield_loop(
            CTX_ARG.into(),
            fut.into(),
            yield_value.clone(),
            // If `dropline_with_dead` is set, it points to the continuation of the drop execution.
            // Otherwise, we are already dropping the coroutine, and `succ_with_dead` does.
            dropline_with_dead.unwrap_or(succ_with_dead),
            unwind_with_dead,
        );
        self.elaborator
            .patch()
            .patch_terminator(drop_resume_bb, TerminatorKind::Goto { target: panic_bb });
        self.elaborator
            .patch()
            .patch_terminator(drop_drop_bb, TerminatorKind::Goto { target: drop_pin_bb });

        // If we are in the regular code path, `dropline_with_dead` is `Some`.
        //
        // In that case, the logic is reversed. Normal execution branches on `resume` from the
        // `yield` terminator. To repeatedly poll the future, that `resume` branch must loop.
        // When the future is dropped, the `yield` terminator branches to `drop`, which follows to
        // the previous loop `drop_pin_bb`.
        let succ_yield_loop = if dropline_with_dead.is_some() {
            let (pin_bb, resume_bb, drop_bb) = self.build_pin_poll_yield_loop(
                CTX_ARG.into(),
                fut.into(),
                yield_value,
                // `dropline_with_dead` is `Some`, so the previous loop point to it.
                succ_with_dead,
                unwind_with_dead,
            );
            self.elaborator
                .patch()
                .patch_terminator(resume_bb, TerminatorKind::Goto { target: pin_bb });
            self.elaborator
                .patch()
                .patch_terminator(drop_bb, TerminatorKind::Goto { target: drop_pin_bb });
            pin_bb
        } else {
            // We were already in the drop line, so return the loop we created for it.
            drop_pin_bb
        };

        // #2:call_drop_bb >>>
        //    call AsyncDrop::drop(pin_obj)
        // OR call async_drop_in_place(pin_obj.pointer)
        let pin_adt_def = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, span));
        let pin_obj_ty = Ty::new_adt(tcx, pin_adt_def, tcx.mk_args(&[obj_ref_ty.into()]));
        // Where we store the result of Pin<&drop_ty>::new_unchecked(&mut place).
        let pin_obj_local = self.new_temp(pin_obj_ty);
        let drop_arg = if call_destructor_only {
            // `AsyncDrop::drop` takes `self: Pin<&mut Self>`.
            Operand::Move(pin_obj_local.into())
        } else {
            // `async_drop_in_place` takes `obj: &mut T`.
            Operand::Copy(tcx.mk_place_field(pin_obj_local.into(), FieldIdx::ZERO, obj_ref_ty))
        };
        let call_drop_bb = self.new_block_with_statements(
            unwind_with_dead,
            vec![self.storage_live(fut)],
            TerminatorKind::Call {
                func: Operand::function_handle(tcx, async_drop_fn_def_id, &[drop_ty.into()], span),
                args: [dummy_spanned(drop_arg)].into(),
                destination: fut.into(),
                target: Some(succ_yield_loop),
                unwind: unwind_with_dead.into_action(),
                call_source: CallSource::Misc,
                fn_span: self.source_info.span,
            },
        );

        // #1:pin_obj_bb >>> call Pin<ObjTy>::new_unchecked(&mut obj)
        let obj_ref_place = Place::from(self.new_temp(obj_ref_ty));
        let pin_obj_new_unchecked_fn = tcx.require_lang_item(LangItem::PinNewUnchecked, span);
        let assign_obj_ref_place = self.assign(
            obj_ref_place,
            Rvalue::Ref(
                tcx.lifetimes.re_erased,
                BorrowKind::Mut { kind: MutBorrowKind::Default },
                place,
            ),
        );
        self.new_block_with_statements(
            unwind,
            vec![assign_obj_ref_place],
            TerminatorKind::Call {
                func: Operand::function_handle(
                    tcx,
                    pin_obj_new_unchecked_fn,
                    &[obj_ref_ty.into()],
                    span,
                ),
                args: [dummy_spanned(Operand::Move(obj_ref_place))].into(),
                destination: pin_obj_local.into(),
                target: Some(call_drop_bb),
                unwind: unwind.into_action(),
                call_source: CallSource::Misc,
                fn_span: span,
            },
        )
    }

    fn build_resumed_after_drop_abort_block(
        &mut self,
        unwind: Unwind,
        coroutine_kind: CoroutineKind,
    ) -> BasicBlock {
        let tcx = self.tcx();
        let panic_bb = self.new_block(unwind, TerminatorKind::Unreachable);
        let msg = AssertMessage::ResumedAfterDrop(coroutine_kind);
        let false_op = Operand::Constant(Box::new(ConstOperand {
            span: self.source_info.span,
            user_ty: None,
            const_: Const::from_bool(tcx, false),
        }));
        self.elaborator.patch().patch_terminator(
            panic_bb,
            TerminatorKind::Assert {
                cond: false_op,
                expected: true,
                msg: Box::new(msg),
                target: panic_bb,
                unwind: unwind.into_action(),
            },
        );
        panic_bb
    }

    /// Build a small MIR loop that pins and polls a future, yielding when
    /// the future returns `Poll::Pending` and continuing to `ready_target`
    /// when it returns `Poll::Ready`.
    ///
    /// Pseudo-code:
    /// ```mir
    /// pin_bb:
    ///   let pin_fut = Pin::new_unchecked(&mut fut_place);
    ///   match Future::poll(pin_fut, CTX_ARG) {
    ///     Poll::Ready => goto succ,
    ///     Poll::Pending(..) => CTX_ARG = yield () [resume: resume_bb, drop: drop_bb],
    ///   }
    /// ```
    ///
    ///  Returns: the tuple `(pin_bb, resume_bb, drop_bb)`.
    #[instrument(level = "trace", skip(self), ret)]
    fn build_pin_poll_yield_loop(
        &mut self,
        resume_place: Place<'tcx>,
        fut_place: Place<'tcx>,
        yield_value: Operand<'tcx>,
        succ: BasicBlock,
        unwind: Unwind,
    ) -> (BasicBlock, BasicBlock, BasicBlock) {
        let tcx = self.tcx();
        let source_info = self.source_info;

        let resume_arg_ty = resume_place.ty(self.elaborator.body(), tcx).ty;
        let context_ref_ty = Ty::new_task_context(tcx);

        let poll_adt_def = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, source_info.span));
        let poll_enum = Ty::new_adt(tcx, poll_adt_def, tcx.mk_args(&[tcx.types.unit.into()]));

        let fut_ty = self.elaborator.patch_ref().local_ty(fut_place.local);
        let fut_ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, fut_ty);

        let pin_adt_def = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, source_info.span));
        let fut_pin_ty = Ty::new_adt(tcx, pin_adt_def, tcx.mk_args(&[fut_ref_ty.into()]));

        // Coroutine `transform_async_context` assumes that the local `resume_arg` to a yield
        // is not used once, so create a special temp for it.
        let yield_resume_local = self.new_temp(resume_arg_ty);
        let resume_bb = self.new_block_with_statements(
            unwind,
            vec![
                self.assign(
                    resume_place,
                    Rvalue::Use(Operand::Move(yield_resume_local.into()), WithRetag::Yes),
                ),
                self.storage_dead(yield_resume_local),
            ],
            // This will be transformed by the caller.
            TerminatorKind::Unreachable,
        );
        let dropline_bb = self.new_block_with_statements(
            unwind,
            vec![
                self.assign(
                    resume_place,
                    Rvalue::Use(Operand::Move(yield_resume_local.into()), WithRetag::Yes),
                ),
                self.storage_dead(yield_resume_local),
            ],
            // This will be transformed by the caller.
            TerminatorKind::Unreachable,
        );
        let yield_bb = self.new_block_with_statements(
            unwind,
            vec![self.storage_live(yield_resume_local)],
            TerminatorKind::Yield {
                value: yield_value,
                resume: resume_bb,
                resume_arg: yield_resume_local.into(),
                drop: Some(dropline_bb),
            },
        );

        let poll_unit_local = self.new_temp(poll_enum);
        let switch_bb = {
            let poll_ready_variant =
                tcx.require_lang_item(LangItem::PollReady, self.source_info.span);
            let poll_ready_variant_idx = poll_adt_def.variant_index_with_id(poll_ready_variant);
            let poll_pending_variant =
                tcx.require_lang_item(LangItem::PollPending, self.source_info.span);
            let poll_pending_variant_idx = poll_adt_def.variant_index_with_id(poll_pending_variant);

            let Discr { val: poll_ready_discr, ty: poll_discr_ty } =
                poll_enum.discriminant_for_variant(tcx, poll_ready_variant_idx).unwrap();
            let Discr { val: poll_pending_discr, ty: _ } =
                poll_enum.discriminant_for_variant(tcx, poll_pending_variant_idx).unwrap();

            let poll_discr_local = self.new_temp(poll_discr_ty);
            let otherwise_bb = self.elaborator.patch().unreachable_no_cleanup_block();
            self.new_block_with_statements(
                unwind,
                vec![
                    self.assign(
                        poll_discr_local.into(),
                        Rvalue::Discriminant(poll_unit_local.into()),
                    ),
                ],
                TerminatorKind::SwitchInt {
                    discr: Operand::Move(poll_discr_local.into()),
                    targets: SwitchTargets::new(
                        [
                            // on `Ready`, exit the loop, jump to `succ`
                            (poll_ready_discr, succ),
                            // on `Pending`, yield and resume back into the loop
                            (poll_pending_discr, yield_bb),
                        ]
                        .into_iter(),
                        // otherwise: unreachable
                        otherwise_bb,
                    ),
                },
            )
        };

        let fut_pin_local = self.new_temp(fut_pin_ty);
        let context_ref_local = self.new_temp(context_ref_ty);

        let poll_fn = tcx.require_lang_item(LangItem::FuturePoll, source_info.span);
        let poll_bb = self.new_block_with_statements(
            unwind,
            Vec::new(),
            TerminatorKind::Call {
                func: Operand::function_handle(tcx, poll_fn, &[fut_ty.into()], source_info.span),
                args: [
                    dummy_spanned(Operand::Move(fut_pin_local.into())),
                    dummy_spanned(Operand::Move(context_ref_local.into())),
                ]
                .into(),
                destination: poll_unit_local.into(),
                target: Some(switch_bb),
                unwind: unwind.into_action(),
                call_source: CallSource::Misc,
                fn_span: source_info.span,
            },
        );

        let get_context_fn = tcx.require_lang_item(LangItem::GetContext, source_info.span);
        let get_context_bb = {
            // Coroutine `transform_async_context` assumes that the local argument to `GetContext`
            // is not used once, so create a special temp for it.
            let entry_resume_local = self.new_temp(resume_arg_ty);
            self.new_block_with_statements(
                unwind,
                vec![self.assign(
                    entry_resume_local.into(),
                    Rvalue::Use(Operand::Move(resume_place), WithRetag::Yes),
                )],
                TerminatorKind::Call {
                    func: Operand::function_handle(
                        tcx,
                        get_context_fn,
                        &[tcx.lifetimes.re_erased.into(), tcx.lifetimes.re_erased.into()],
                        source_info.span,
                    ),
                    args: [dummy_spanned(Operand::Move(entry_resume_local.into()))].into(),
                    destination: context_ref_local.into(),
                    target: Some(poll_bb),
                    unwind: unwind.into_action(),
                    call_source: CallSource::Misc,
                    fn_span: source_info.span,
                },
            )
        };

        let fut_ref_local = self.new_temp(fut_ref_ty);
        let fut_pin_new_unchecked_fn =
            tcx.require_lang_item(LangItem::PinNewUnchecked, source_info.span);
        let pin_bb = self.new_block_with_statements(
            unwind,
            vec![self.assign(
                fut_ref_local.into(),
                Rvalue::Ref(
                    tcx.lifetimes.re_erased,
                    BorrowKind::Mut { kind: MutBorrowKind::Default },
                    fut_place,
                ),
            )],
            TerminatorKind::Call {
                func: Operand::function_handle(
                    tcx,
                    fut_pin_new_unchecked_fn,
                    &[fut_ref_ty.into()],
                    source_info.span,
                ),
                args: [dummy_spanned(Operand::Move(fut_ref_local.into()))].into(),
                destination: fut_pin_local.into(),
                target: Some(get_context_bb),
                unwind: unwind.into_action(),
                call_source: CallSource::Misc,
                fn_span: source_info.span,
            },
        );

        (pin_bb, resume_bb, dropline_bb)
    }

    fn build_drop(&mut self, bb: BasicBlock) {
        let drop_ty = self.place_ty(self.place);
        if !self.elaborator.patch_ref().block(self.elaborator.body(), bb).is_cleanup
            && self.check_if_can_async_drop(drop_ty, false)
        {
            let async_drop_bb = self.build_async_drop(
                self.place,
                drop_ty,
                self.succ,
                self.unwind,
                self.dropline,
                false,
            );
            self.elaborator
                .patch()
                .patch_terminator(bb, TerminatorKind::Goto { target: async_drop_bb });
        } else {
            self.elaborator.patch().patch_terminator(
                bb,
                TerminatorKind::Drop {
                    place: self.place,
                    target: self.succ,
                    unwind: self.unwind.into_action(),
                    replace: false,
                    drop: None,
                },
            );
        }
    }

    /// Function to check if we can generate an async drop here
    fn check_if_can_async_drop(&mut self, drop_ty: Ty<'tcx>, call_destructor_only: bool) -> bool {
        if !self.elaborator.allow_async_drops()
            || !self
                .elaborator
                .body()
                .coroutine
                .as_ref()
                .is_some_and(|ck| ck.coroutine_kind.is_async_desugaring())
        {
            return false;
        }

        if drop_ty == self.place_ty(Local::arg(0).into()) {
            return false;
        }

        let is_async_drop_feature_enabled = if self.tcx().features().async_drop() {
            true
        } else {
            // Check if the type needing async drop comes from a dependency crate.
            if let ty::Adt(adt_def, _) = drop_ty.kind() {
                !adt_def.did().is_local() && adt_def.async_destructor(self.tcx()).is_some()
            } else {
                false
            }
        };

        // Short-circuit before calling needs_async_drop/is_async_drop, as those
        // require the `async_drop` lang item to exist (which may not be present
        // in minimal/custom core environments like cranelift's mini_core).
        if !is_async_drop_feature_enabled {
            return false;
        }

        let needs_async_drop = if call_destructor_only {
            drop_ty.is_async_drop(self.tcx(), self.elaborator.typing_env())
        } else {
            drop_ty.needs_async_drop(self.tcx(), self.elaborator.typing_env())
        };

        // Async drop in libstd/libcore would become insta-stable — catch that mistake.
        if needs_async_drop && self.tcx().features().staged_api() {
            span_bug!(
                self.source_info.span,
                "don't use async drop in libstd, it becomes insta-stable"
            );
        }

        needs_async_drop
    }

    /// This elaborates a single drop instruction, located at `bb`, and
    /// patches over it.
    ///
    /// The elaborated drop checks the drop flags to only drop what
    /// is initialized.
    ///
    /// In addition, the relevant drop flags also need to be cleared
    /// to avoid double-drops. However, in the middle of a complex
    /// drop, one must avoid clearing some of the flags before they
    /// are read, as that would cause a memory leak.
    ///
    /// In particular, when dropping an ADT, multiple fields may be
    /// joined together under the `rest` subpath. They are all controlled
    /// by the primary drop flag, but only the last rest-field dropped
    /// should clear it (and it must also not clear anything else).
    //
    // FIXME: I think we should just control the flags externally,
    // and then we do not need this machinery.
    #[instrument(level = "debug")]
    fn elaborate_drop(&mut self, bb: BasicBlock) {
        match self.elaborator.drop_style(self.path, DropFlagMode::Deep) {
            DropStyle::Dead => {
                self.elaborator
                    .patch()
                    .patch_terminator(bb, TerminatorKind::Goto { target: self.succ });
            }
            DropStyle::Static => {
                self.build_drop(bb);
            }
            DropStyle::Conditional => {
                let drop_bb = self.complete_drop(self.succ, self.unwind);
                self.elaborator
                    .patch()
                    .patch_terminator(bb, TerminatorKind::Goto { target: drop_bb });
            }
            DropStyle::Open => {
                let drop_bb = self.open_drop();
                self.elaborator
                    .patch()
                    .patch_terminator(bb, TerminatorKind::Goto { target: drop_bb });
            }
        }
    }

    /// Returns the place and move path for each field of `variant`,
    /// (the move path is `None` if the field is a rest field).
    fn move_paths_for_fields(
        &self,
        base_place: Place<'tcx>,
        variant_path: D::Path,
        variant: &'tcx ty::VariantDef,
        args: GenericArgsRef<'tcx>,
    ) -> Vec<(Place<'tcx>, Option<D::Path>)> {
        variant
            .fields
            .iter_enumerated()
            .map(|(field_idx, field)| {
                let subpath = self.elaborator.field_subpath(variant_path, field_idx);
                let tcx = self.tcx();

                match self.elaborator.typing_env().typing_mode().assert_not_erased() {
                    ty::TypingMode::PostAnalysis | ty::TypingMode::Codegen => {}
                    ty::TypingMode::Coherence
                    | ty::TypingMode::Reflection
                    | ty::TypingMode::Typeck { .. }
                    | ty::TypingMode::PostTypeckUntilBorrowck { .. }
                    | ty::TypingMode::PostBorrowck { .. } => {
                        bug!()
                    }
                }

                let field_ty = field.ty(tcx, args);
                // We silently leave an unnormalized type here to support polymorphic drop
                // elaboration for users of rustc internal APIs
                let field_ty = tcx
                    .try_normalize_erasing_regions(self.elaborator.typing_env(), field_ty)
                    .unwrap_or(field_ty.skip_norm_wip());

                (tcx.mk_place_field(base_place, field_idx, field_ty), subpath)
            })
            .filter(|path| self.should_retain_for_ladder(path))
            .collect()
    }

    #[instrument(level = "debug", skip(self), ret)]
    fn drop_subpath(
        &mut self,
        place: Place<'tcx>,
        path: Option<D::Path>,
        succ: BasicBlock,
        unwind: Unwind,
        dropline: Option<BasicBlock>,
    ) -> BasicBlock {
        if let Some(path) = path {
            DropCtxt {
                elaborator: self.elaborator,
                source_info: self.source_info,
                path,
                place,
                succ,
                unwind,
                dropline,
            }
            .elaborated_drop_block()
        } else {
            DropCtxt {
                elaborator: self.elaborator,
                source_info: self.source_info,
                place,
                succ,
                unwind,
                dropline,
                // Using `self.path` here to condition the drop on our own drop flag.
                path: self.path,
            }
            .complete_drop(succ, unwind)
        }
    }

    /// Creates one-half of the drop ladder for a list of fields, and return
    /// the list of steps in it in reverse order, with the first step
    /// dropping 0 fields and so on.
    ///
    /// `unwind_ladder` is such a list of steps in reverse order,
    /// which is called if the matching step of the drop glue panics.
    ///
    /// `dropline_ladder` is a similar list of steps in reverse order,
    /// which is called if the matching step of the drop glue will contain async drop
    /// (expanded later to Yield) and the containing coroutine will be dropped at this point.
    #[instrument(level = "debug", skip(self), ret)]
    fn drop_halfladder(
        &mut self,
        unwind_ladder: &[Unwind],
        dropline_ladder: &[Option<BasicBlock>],
        mut succ: BasicBlock,
        fields: &[(Place<'tcx>, Option<D::Path>)],
    ) -> Vec<BasicBlock> {
        iter::once(succ)
            .chain(itertools::izip!(fields.iter().rev(), unwind_ladder, dropline_ladder).map(
                |(&(place, path), &unwind_succ, &dropline_to)| {
                    succ = self.drop_subpath(place, path, succ, unwind_succ, dropline_to);
                    succ
                },
            ))
            .collect()
    }

    fn drop_ladder_bottom(&mut self) -> (BasicBlock, Unwind, Option<BasicBlock>) {
        // Clear the "master" drop flag at the end. This is needed
        // because the "master" drop protects the ADT's discriminant,
        // which is invalidated after the ADT is dropped.
        (
            self.drop_flag_reset_block(DropFlagMode::Shallow, self.succ, self.unwind),
            self.unwind,
            self.dropline,
        )
    }

    /// Whether this drop is useful. This is purely an optimization to avoid generating useless blocks.
    fn should_retain_for_ladder(&self, (place, subpath): &(Place<'tcx>, Option<D::Path>)) -> bool {
        if !self.place_ty(*place).needs_drop(self.tcx(), self.elaborator.typing_env()) {
            return false;
        }
        if let Some(subpath) = subpath
            && let DropStyle::Dead = self.elaborator.drop_style(*subpath, DropFlagMode::Deep)
        {
            return false;
        }
        true
    }

    /// Creates a full drop ladder, consisting of 2 connected half-drop-ladders
    ///
    /// For example, with 3 fields, the drop ladder is
    ///
    /// ```text
    /// .d0:
    ///     ELAB(drop location.0 [target=.d1, unwind=.c1])
    /// .d1:
    ///     ELAB(drop location.1 [target=.d2, unwind=.c2])
    /// .d2:
    ///     ELAB(drop location.2 [target=`self.succ`, unwind=`self.unwind`])
    /// .c1:
    ///     ELAB(drop location.1 [target=.c2])
    /// .c2:
    ///     ELAB(drop location.2 [target=`self.unwind`])
    /// ```
    ///
    /// For possible-async drops in coroutines we also need dropline ladder
    /// ```text
    /// .d0 (mainline):
    ///     ELAB(drop location.0 [target=.d1, unwind=.c1, drop=.e1])
    /// .d1 (mainline):
    ///     ELAB(drop location.1 [target=.d2, unwind=.c2, drop=.e2])
    /// .d2 (mainline):
    ///     ELAB(drop location.2 [target=`self.succ`, unwind=`self.unwind`, drop=`self.drop`])
    /// .c1 (unwind):
    ///     ELAB(drop location.1 [target=.c2])
    /// .c2 (unwind):
    ///     ELAB(drop location.2 [target=`self.unwind`])
    /// .e1 (dropline):
    ///     ELAB(drop location.1 [target=.e2, unwind=.c2])
    /// .e2 (dropline):
    ///     ELAB(drop location.2 [target=`self.drop`, unwind=`self.unwind`])
    /// ```
    ///
    /// NOTE: this does not clear the master drop flag, so you need
    /// to point succ/unwind on a `drop_ladder_bottom`.
    #[instrument(level = "debug", skip(self), ret)]
    fn drop_ladder(
        &mut self,
        mut fields: Vec<(Place<'tcx>, Option<D::Path>)>,
        succ: BasicBlock,
        unwind: Unwind,
        dropline: Option<BasicBlock>,
    ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
        assert!(
            if unwind.is_cleanup() { dropline.is_none() } else { true },
            "Dropline is set for cleanup drop ladder"
        );

        fields.retain(|path| self.should_retain_for_ladder(path));

        debug!("drop_ladder - fields needing drop: {:?}", fields);

        let dropline_ladder: Vec<Option<BasicBlock>> = vec![None; fields.len() + 1];
        let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1];
        let unwind_ladder: Vec<_> = if let Unwind::To(succ) = unwind {
            let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);
            halfladder.into_iter().map(Unwind::To).collect()
        } else {
            unwind_ladder
        };
        let dropline_ladder: Vec<_> = if let Some(succ) = dropline {
            let halfladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);
            halfladder.into_iter().map(Some).collect()
        } else {
            dropline_ladder
        };

        let normal_ladder = self.drop_halfladder(&unwind_ladder, &dropline_ladder, succ, &fields);

        (
            *normal_ladder.last().unwrap(),
            *unwind_ladder.last().unwrap(),
            *dropline_ladder.last().unwrap(),
        )
    }

    #[instrument(level = "debug", skip(self), ret)]
    fn open_drop_for_tuple(&mut self, tys: &[Ty<'tcx>]) -> BasicBlock {
        let fields = tys
            .iter()
            .enumerate()
            .map(|(i, &ty)| {
                (
                    self.tcx().mk_place_field(self.place, FieldIdx::new(i), ty),
                    self.elaborator.field_subpath(self.path, FieldIdx::new(i)),
                )
            })
            .collect();

        let (succ, unwind, dropline) = self.drop_ladder_bottom();
        self.drop_ladder(fields, succ, unwind, dropline).0
    }

    /// Drops the T contained in a `Box<T>` if it has not been moved out of
    #[instrument(level = "debug", ret)]
    fn open_drop_for_box_contents(
        &mut self,
        adt: ty::AdtDef<'tcx>,
        args: GenericArgsRef<'tcx>,
        succ: BasicBlock,
        unwind: Unwind,
        dropline: Option<BasicBlock>,
    ) -> BasicBlock {
        // drop glue is sent straight to codegen
        // box cannot be directly dereferenced
        let unique_ty =
            adt.non_enum_variant().fields[FieldIdx::ZERO].ty(self.tcx(), args).skip_norm_wip();
        let unique_variant = unique_ty.ty_adt_def().unwrap().non_enum_variant();
        let nonnull_ty = unique_variant.fields[FieldIdx::ZERO].ty(self.tcx(), args).skip_norm_wip();
        let ptr_ty = Ty::new_imm_ptr(self.tcx(), args[0].expect_ty());

        let unique_place = self.tcx().mk_place_field(self.place, FieldIdx::ZERO, unique_ty);
        let nonnull_place = self.tcx().mk_place_field(unique_place, FieldIdx::ZERO, nonnull_ty);

        let ptr_local = self.new_temp(ptr_ty);

        let interior = self.tcx().mk_place_deref(Place::from(ptr_local));
        let interior_path = self.elaborator.deref_subpath(self.path);

        let do_drop_bb = self.drop_subpath(interior, interior_path, succ, unwind, dropline);

        self.new_block_with_statements(
            unwind,
            vec![self.assign(
                Place::from(ptr_local),
                Rvalue::Cast(CastKind::Transmute, Operand::Copy(nonnull_place), ptr_ty),
            )],
            TerminatorKind::Goto { target: do_drop_bb },
        )
    }

    #[instrument(level = "debug", ret)]
    fn open_drop_for_adt(
        &mut self,
        adt: ty::AdtDef<'tcx>,
        args: GenericArgsRef<'tcx>,
    ) -> BasicBlock {
        if adt.variants().is_empty() {
            return self.new_block(self.unwind, TerminatorKind::Unreachable);
        }

        let skip_contents = adt.is_union() || adt.is_manually_drop();
        let (contents_succ, contents_unwind, contents_dropline) = if skip_contents {
            if adt.has_dtor(self.tcx()) && self.elaborator.get_drop_flag(self.path).is_some() {
                // the top-level drop flag is usually cleared by open_drop_for_adt_contents
                // types with destructors would still need an empty drop ladder to clear it

                // however, these types are only open dropped in `DropShimElaborator`
                // which does not have drop flags
                // a future box-like "DerefMove" trait would allow for this case to happen
                span_bug!(self.source_info.span, "open dropping partially moved union");
            }

            (self.succ, self.unwind, self.dropline)
        } else {
            self.open_drop_for_adt_contents(adt, args)
        };

        if adt.has_dtor(self.tcx()) {
            let destructor_block = if adt.is_box() {
                // we need to drop the inside of the box before running the destructor
                let succ = self.destructor_call_block_sync(contents_succ, contents_unwind);
                let unwind = contents_unwind
                    .map(|unwind| self.destructor_call_block_sync(unwind, Unwind::InCleanup));
                let dropline = contents_dropline
                    .map(|dropline| self.destructor_call_block_sync(dropline, contents_unwind));
                self.open_drop_for_box_contents(adt, args, succ, unwind, dropline)
            } else {
                self.destructor_call_block(contents_succ, contents_unwind, contents_dropline)
            };

            self.drop_flag_test_block(destructor_block, contents_succ, contents_unwind)
        } else {
            contents_succ
        }
    }

    fn open_drop_for_adt_contents(
        &mut self,
        adt: ty::AdtDef<'tcx>,
        args: GenericArgsRef<'tcx>,
    ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
        let (succ, unwind, dropline) = self.drop_ladder_bottom();
        if !adt.is_enum() {
            let fields =
                self.move_paths_for_fields(self.place, self.path, adt.variant(FIRST_VARIANT), args);
            self.drop_ladder(fields, succ, unwind, dropline)
        } else {
            self.open_drop_for_multivariant(adt, args, succ, unwind, dropline)
        }
    }

    fn open_drop_for_multivariant(
        &mut self,
        adt: ty::AdtDef<'tcx>,
        args: GenericArgsRef<'tcx>,
        succ: BasicBlock,
        unwind: Unwind,
        dropline: Option<BasicBlock>,
    ) -> (BasicBlock, Unwind, Option<BasicBlock>) {
        let mut values = Vec::with_capacity(adt.variants().len());
        let mut normal_blocks = Vec::with_capacity(adt.variants().len());
        let mut unwind_blocks =
            Vec::with_capacity(if unwind.is_cleanup() { 0 } else { adt.variants().len() });
        let mut dropline_blocks =
            Vec::with_capacity(if dropline.is_none() { 0 } else { adt.variants().len() });

        let mut have_otherwise_with_drop_glue = false;
        let mut have_otherwise = false;
        let tcx = self.tcx();

        for (variant_index, discr) in adt.discriminants(tcx) {
            let variant = &adt.variant(variant_index);
            let subpath = self.elaborator.downcast_subpath(self.path, variant_index);

            if let Some(variant_path) = subpath {
                let base_place = tcx.mk_place_elem(
                    self.place,
                    ProjectionElem::Downcast(Some(variant.name), variant_index),
                );
                let fields = self.move_paths_for_fields(base_place, variant_path, variant, args);
                values.push(discr.val);
                if let Unwind::To(unwind) = unwind {
                    // We can't use the half-ladder from the original
                    // drop ladder, because this breaks the
                    // "funclet can't have 2 successor funclets"
                    // requirement from MSVC:
                    //
                    //           switch       unwind-switch
                    //          /      \         /        \
                    //         v1.0    v2.0  v2.0-unwind  v1.0-unwind
                    //         |        |      /             |
                    //    v1.1-unwind  v2.1-unwind           |
                    //      ^                                |
                    //       \-------------------------------/
                    //
                    // Create a duplicate half-ladder to avoid that. We
                    // could technically only do this on MSVC, but I
                    // I want to minimize the divergence between MSVC
                    // and non-MSVC.

                    let unwind_ladder = vec![Unwind::InCleanup; fields.len() + 1];
                    let dropline_ladder: Vec<Option<BasicBlock>> = vec![None; fields.len() + 1];
                    let halfladder =
                        self.drop_halfladder(&unwind_ladder, &dropline_ladder, unwind, &fields);
                    unwind_blocks.push(halfladder.last().cloned().unwrap());
                }
                let (normal, _, drop_bb) = self.drop_ladder(fields, succ, unwind, dropline);
                normal_blocks.push(normal);
                if dropline.is_some() {
                    dropline_blocks.push(drop_bb.unwrap());
                }
            } else {
                have_otherwise = true;

                let typing_env = self.elaborator.typing_env();
                let have_field_with_drop_glue = variant
                    .fields
                    .iter()
                    .any(|field| field.ty(tcx, args).skip_norm_wip().needs_drop(tcx, typing_env));
                if have_field_with_drop_glue {
                    have_otherwise_with_drop_glue = true;
                }
            }
        }

        if !have_otherwise {
            values.pop();
        } else if !have_otherwise_with_drop_glue {
            normal_blocks.push(succ);
            if let Unwind::To(unwind) = unwind {
                unwind_blocks.push(unwind);
            }
            if let Some(dropline) = dropline {
                dropline_blocks.push(dropline);
            }
        } else {
            normal_blocks.push(self.drop_block(succ, unwind));
            if let Unwind::To(unwind) = unwind {
                unwind_blocks.push(self.drop_block(unwind, Unwind::InCleanup));
            }
            if let Some(dropline) = dropline {
                dropline_blocks.push(self.drop_block(dropline, unwind));
            }
        }

        (
            self.adt_switch_block(adt, normal_blocks, &values, succ, unwind),
            unwind.map(|unwind| {
                self.adt_switch_block(adt, unwind_blocks, &values, unwind, Unwind::InCleanup)
            }),
            dropline.map(|dropline| {
                self.adt_switch_block(adt, dropline_blocks, &values, dropline, unwind)
            }),
        )
    }

    fn adt_switch_block(
        &mut self,
        adt: ty::AdtDef<'tcx>,
        blocks: Vec<BasicBlock>,
        values: &[u128],
        succ: BasicBlock,
        unwind: Unwind,
    ) -> BasicBlock {
        let switch_block = blocks.iter().copied().all_equal_value().unwrap_or_else(|_| {
            // If there are multiple variants, then if something
            // is present within the enum the discriminant, tracked
            // by the rest path, must be initialized.
            //
            // Additionally, we do not want to switch on the
            // discriminant after it is free-ed, because that
            // way lies only trouble.
            let discr_ty = adt.repr().discr_type().to_ty(self.tcx());
            let discr = Place::from(self.new_temp(discr_ty));
            let discr_rv = Rvalue::Discriminant(self.place);
            self.new_block_with_statements(
                unwind,
                vec![self.assign(discr, discr_rv)],
                TerminatorKind::SwitchInt {
                    discr: Operand::Move(discr),
                    targets: SwitchTargets::new(
                        values.iter().copied().zip(blocks.iter().copied()),
                        *blocks.last().unwrap(),
                    ),
                },
            )
        });
        self.drop_flag_test_block(switch_block, succ, unwind)
    }

    #[instrument(level = "debug", skip(self), ret)]
    fn destructor_call_block_sync(&mut self, succ: BasicBlock, unwind: Unwind) -> BasicBlock {
        let tcx = self.tcx();
        let drop_trait = tcx.require_lang_item(LangItem::Drop, DUMMY_SP);
        let drop_fn = tcx.associated_item_def_ids(drop_trait)[0];
        let ty = self.place_ty(self.place);

        let ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, ty);
        let ref_place = self.new_temp(ref_ty);
        let unit_temp = Place::from(self.new_temp(tcx.types.unit));

        self.new_block_with_statements(
            unwind,
            vec![self.assign(
                Place::from(ref_place),
                Rvalue::Ref(
                    tcx.lifetimes.re_erased,
                    BorrowKind::Mut { kind: MutBorrowKind::Default },
                    self.place,
                ),
            )],
            TerminatorKind::Call {
                func: Operand::function_handle(tcx, drop_fn, &[ty.into()], self.source_info.span),
                args: [dummy_spanned(Operand::Move(Place::from(ref_place)))].into(),
                destination: unit_temp,
                target: Some(succ),
                unwind: unwind.into_action(),
                call_source: CallSource::Misc,
                fn_span: self.source_info.span,
            },
        )
    }

    #[instrument(level = "debug", skip(self), ret)]
    fn destructor_call_block(
        &mut self,
        succ: BasicBlock,
        unwind: Unwind,
        dropline: Option<BasicBlock>,
    ) -> BasicBlock {
        let ty = self.place_ty(self.place);
        if !unwind.is_cleanup() && self.check_if_can_async_drop(ty, true) {
            self.build_async_drop(self.place, ty, succ, unwind, dropline, true)
        } else {
            self.destructor_call_block_sync(succ, unwind)
        }
    }

    /// Create a loop that drops an array:
    ///
    /// ```text
    /// loop-block:
    ///    can_go = cur == len
    ///    if can_go then succ else drop-block
    /// drop-block:
    ///    ptr = &raw mut P[cur]
    ///    cur = cur + 1
    ///    drop(ptr)
    /// ```
    fn drop_loop(
        &mut self,
        succ: BasicBlock,
        cur: Local,
        len: Local,
        ety: Ty<'tcx>,
        unwind: Unwind,
        dropline: Option<BasicBlock>,
    ) -> BasicBlock {
        let copy = |place: Place<'tcx>| Operand::Copy(place);
        let move_ = |place: Place<'tcx>| Operand::Move(place);
        let tcx = self.tcx();

        let ptr_ty = Ty::new_mut_ptr(tcx, ety);
        let ptr = Place::from(self.new_temp(ptr_ty));
        let can_go = Place::from(self.new_temp(tcx.types.bool));
        let one = self.constant_usize(1);

        let drop_block = self.new_block_with_statements(
            unwind,
            vec![
                self.assign(
                    ptr,
                    Rvalue::RawPtr(RawPtrKind::Mut, tcx.mk_place_index(self.place, cur)),
                ),
                self.assign(
                    cur.into(),
                    Rvalue::BinaryOp(BinOp::Add, Box::new((move_(cur.into()), one))),
                ),
            ],
            // this gets overwritten by drop elaboration.
            TerminatorKind::Unreachable,
        );

        let loop_block = self.new_block_with_statements(
            unwind,
            vec![self.assign(
                can_go,
                Rvalue::BinaryOp(BinOp::Eq, Box::new((copy(Place::from(cur)), copy(len.into())))),
            )],
            TerminatorKind::if_(move_(can_go), succ, drop_block),
        );

        let place = tcx.mk_place_deref(ptr);
        if !unwind.is_cleanup() && self.check_if_can_async_drop(ety, false) {
            let async_drop_bb =
                self.build_async_drop(place, ety, loop_block, unwind, dropline, false);
            self.elaborator
                .patch()
                .patch_terminator(drop_block, TerminatorKind::Goto { target: async_drop_bb });
        } else {
            self.elaborator.patch().patch_terminator(
                drop_block,
                TerminatorKind::Drop {
                    place,
                    target: loop_block,
                    unwind: unwind.into_action(),
                    replace: false,
                    drop: None,
                },
            );
        }
        loop_block
    }

    #[instrument(level = "debug", skip(self), ret)]
    fn open_drop_for_array(
        &mut self,
        array_ty: Ty<'tcx>,
        ety: Ty<'tcx>,
        opt_size: Option<u64>,
    ) -> BasicBlock {
        let tcx = self.tcx();

        if let Some(size) = opt_size {
            enum ProjectionKind<Path> {
                Drop(std::ops::Range<u64>),
                Keep(u64, Path),
            }
            // Previously, we'd make a projection for every element in the array and create a drop
            // ladder if any `array_subpath` was `Some`, i.e. moving out with an array pattern.
            // This caused huge memory usage when generating the drops for large arrays, so we instead
            // record the *subslices* which are dropped and the *indexes* which are kept
            let mut drop_ranges = vec![];
            let mut dropping = true;
            let mut start = 0;
            for i in 0..size {
                let path = self.elaborator.array_subpath(self.path, i, size);
                if dropping && path.is_some() {
                    drop_ranges.push(ProjectionKind::Drop(start..i));
                    dropping = false;
                } else if !dropping && path.is_none() {
                    dropping = true;
                    start = i;
                }
                if let Some(path) = path {
                    drop_ranges.push(ProjectionKind::Keep(i, path));
                }
            }
            if !drop_ranges.is_empty() {
                if dropping {
                    drop_ranges.push(ProjectionKind::Drop(start..size));
                }
                let fields = drop_ranges
                    .iter()
                    .rev()
                    .map(|p| {
                        let (project, path) = match p {
                            ProjectionKind::Drop(r) => (
                                ProjectionElem::Subslice {
                                    from: r.start,
                                    to: r.end,
                                    from_end: false,
                                },
                                None,
                            ),
                            &ProjectionKind::Keep(offset, path) => (
                                ProjectionElem::ConstantIndex {
                                    offset,
                                    min_length: size,
                                    from_end: false,
                                },
                                Some(path),
                            ),
                        };
                        (tcx.mk_place_elem(self.place, project), path)
                    })
                    .collect::<Vec<_>>();
                let (succ, unwind, dropline) = self.drop_ladder_bottom();
                return self.drop_ladder(fields, succ, unwind, dropline).0;
            }
        }

        let array_ptr_ty = Ty::new_mut_ptr(tcx, array_ty);
        let array_ptr = self.new_temp(array_ptr_ty);

        let slice_ty = Ty::new_slice(tcx, ety);
        let slice_ptr_ty = Ty::new_mut_ptr(tcx, slice_ty);
        let slice_ptr = self.new_temp(slice_ptr_ty);

        let array_place = mem::replace(
            &mut self.place,
            Place::from(slice_ptr).project_deeper(&[PlaceElem::Deref], tcx),
        );
        let slice_block = self.drop_loop_trio_for_slice(ety);
        self.place = array_place;

        self.new_block_with_statements(
            self.unwind,
            vec![
                self.assign(Place::from(array_ptr), Rvalue::RawPtr(RawPtrKind::Mut, self.place)),
                self.assign(
                    Place::from(slice_ptr),
                    Rvalue::Cast(
                        CastKind::PointerCoercion(
                            PointerCoercion::Unsize,
                            CoercionSource::Implicit,
                        ),
                        Operand::Move(Place::from(array_ptr)),
                        slice_ptr_ty,
                    ),
                ),
            ],
            TerminatorKind::Goto { target: slice_block },
        )
    }

    /// Creates a trio of drop-loops of `place`, which drops its contents, even
    /// in the case of 1 panic or in the case of coroutine drop
    #[instrument(level = "debug", skip(self), ret)]
    fn drop_loop_trio_for_slice(&mut self, ety: Ty<'tcx>) -> BasicBlock {
        let tcx = self.tcx();
        let len = self.new_temp(tcx.types.usize);
        let cur = self.new_temp(tcx.types.usize);

        let unwind = self
            .unwind
            .map(|unwind| self.drop_loop(unwind, cur, len, ety, Unwind::InCleanup, None));

        let dropline =
            self.dropline.map(|dropline| self.drop_loop(dropline, cur, len, ety, unwind, None));

        let loop_block = self.drop_loop(self.succ, cur, len, ety, unwind, dropline);

        let [PlaceElem::Deref] = self.place.projection.as_slice() else {
            span_bug!(
                self.source_info.span,
                "Expected place for slice drop shim to be *_n, but it's {:?}",
                self.place,
            );
        };

        let zero = self.constant_usize(0);
        let drop_block = self.new_block_with_statements(
            unwind,
            vec![
                self.assign(
                    len.into(),
                    Rvalue::UnaryOp(
                        UnOp::PtrMetadata,
                        Operand::Copy(Place::from(self.place.local)),
                    ),
                ),
                self.assign(cur.into(), Rvalue::Use(zero, WithRetag::Yes)),
            ],
            TerminatorKind::Goto { target: loop_block },
        );

        // FIXME(#34708): handle partially-dropped array/slice elements.
        let reset_block = self.drop_flag_reset_block(DropFlagMode::Deep, drop_block, unwind);
        self.drop_flag_test_block(reset_block, self.succ, unwind)
    }

    /// The slow-path - create an "open", elaborated drop for a type
    /// which is moved-out-of only partially, and patch `bb` to a jump
    /// to it. This must not be called on ADTs with a destructor,
    /// as these can't be moved-out-of, except for `Box<T>`, which is
    /// special-cased.
    ///
    /// This creates a "drop ladder" that drops the needed fields of the
    /// ADT, both in the success case or if one of the destructors fail.
    fn open_drop(&mut self) -> BasicBlock {
        let ty = self.place_ty(self.place);
        match ty.kind() {
            ty::Closure(_, args) => self.open_drop_for_tuple(args.as_closure().upvar_tys()),
            ty::CoroutineClosure(_, args) => {
                self.open_drop_for_tuple(args.as_coroutine_closure().upvar_tys())
            }
            // Note that `elaborate_drops` only drops the upvars of a coroutine,
            // and this is ok because `open_drop` here can only be reached
            // within that own coroutine's resume function.
            // This should only happen for the self argument on the resume function.
            // It effectively only contains upvars until the coroutine transformation runs.
            // See librustc_body/transform/coroutine.rs for more details.
            ty::Coroutine(_, args) => self.open_drop_for_tuple(args.as_coroutine().upvar_tys()),
            ty::Tuple(fields) => self.open_drop_for_tuple(fields),
            ty::Adt(def, args) => self.open_drop_for_adt(*def, args),
            ty::Dynamic(..) => self.complete_drop(self.succ, self.unwind),
            ty::Array(ety, size) => {
                let size = size.try_to_target_usize(self.tcx());
                self.open_drop_for_array(ty, *ety, size)
            }
            ty::Slice(ety) => self.drop_loop_trio_for_slice(*ety),

            ty::UnsafeBinder(_) => {
                // Unsafe binders may elaborate drops if their inner type isn't copy.
                // This is enforced in typeck, so this should never happen.
                self.tcx().dcx().span_delayed_bug(
                    self.source_info.span,
                    "open drop for unsafe binder shouldn't be encountered",
                );
                self.new_block(self.unwind, TerminatorKind::Unreachable)
            }

            _ => span_bug!(self.source_info.span, "open drop from non-ADT `{:?}`", ty),
        }
    }

    #[instrument(level = "debug", skip(self), ret)]
    fn complete_drop(&mut self, succ: BasicBlock, unwind: Unwind) -> BasicBlock {
        let drop_block = self.drop_block(succ, unwind);
        self.drop_flag_test_block(drop_block, succ, unwind)
    }

    /// Creates a block that resets the drop flag. If `mode` is deep, all children drop flags will
    /// also be cleared.
    #[instrument(level = "debug", skip(self), ret)]
    fn drop_flag_reset_block(
        &mut self,
        mode: DropFlagMode,
        succ: BasicBlock,
        unwind: Unwind,
    ) -> BasicBlock {
        if unwind.is_cleanup() {
            // The drop flag isn't read again on the unwind path, so don't
            // bother setting it.
            return succ;
        }
        let flags = self.elaborator.drop_flags_for(self.path, mode);
        let statements: Vec<_> = flags
            .into_iter()
            .map(|flag| {
                self.assign(
                    flag,
                    Rvalue::Use(self.constant_bool(DropFlagState::Absent.value()), WithRetag::Yes),
                )
            })
            .collect();
        if statements.is_empty() {
            return succ;
        }
        self.new_block_with_statements(unwind, statements, TerminatorKind::Goto { target: succ })
    }

    #[instrument(level = "debug", skip(self), ret)]
    fn elaborated_drop_block(&mut self) -> BasicBlock {
        let blk = self.new_block(
            self.unwind,
            TerminatorKind::Drop {
                place: self.place,
                target: self.succ,
                unwind: self.unwind.into_action(),
                replace: false,
                drop: self.dropline,
            },
        );
        self.elaborate_drop(blk);
        blk
    }

    fn drop_block(&mut self, target: BasicBlock, unwind: Unwind) -> BasicBlock {
        let drop_ty = self.place_ty(self.place);
        if !unwind.is_cleanup() && self.check_if_can_async_drop(drop_ty, false) {
            self.build_async_drop(self.place, drop_ty, self.succ, unwind, self.dropline, false)
        } else {
            self.new_block(
                unwind,
                TerminatorKind::Drop {
                    place: self.place,
                    target,
                    unwind: unwind.into_action(),
                    replace: false,
                    drop: None,
                },
            )
        }
    }

    /// Returns the block to jump to in order to test the drop flag and execute the drop.
    ///
    /// Depending on the required `DropStyle`, this might be a generated block with an `if`
    /// terminator (for dynamic/open drops), or it might be `on_set` or `on_unset` itself, in case
    /// the drop can be statically determined.
    #[instrument(level = "debug", skip(self), ret)]
    fn drop_flag_test_block(
        &mut self,
        on_set: BasicBlock,
        on_unset: BasicBlock,
        unwind: Unwind,
    ) -> BasicBlock {
        let style = self.elaborator.drop_style(self.path, DropFlagMode::Shallow);
        match style {
            DropStyle::Dead => on_unset,
            DropStyle::Static => on_set,
            DropStyle::Conditional | DropStyle::Open => {
                let flag = self.elaborator.get_drop_flag(self.path).unwrap();
                let term = TerminatorKind::if_(flag, on_set, on_unset);
                self.new_block(unwind, term)
            }
        }
    }

    #[instrument(level = "trace", skip(self), ret)]
    fn new_block(&mut self, unwind: Unwind, k: TerminatorKind<'tcx>) -> BasicBlock {
        self.elaborator.patch().new_block(BasicBlockData::new(
            Some(Terminator { source_info: self.source_info, kind: k, attributes: ThinVec::new() }),
            unwind.is_cleanup(),
        ))
    }

    #[instrument(level = "trace", skip(self, statements), ret)]
    fn new_block_with_statements(
        &mut self,
        unwind: Unwind,
        statements: Vec<Statement<'tcx>>,
        k: TerminatorKind<'tcx>,
    ) -> BasicBlock {
        self.elaborator.patch().new_block(BasicBlockData::new_stmts(
            statements,
            Some(Terminator { source_info: self.source_info, kind: k, attributes: ThinVec::new() }),
            unwind.is_cleanup(),
        ))
    }

    fn new_temp(&mut self, ty: Ty<'tcx>) -> Local {
        self.elaborator.patch().new_temp(ty, self.source_info.span)
    }

    fn constant_usize(&self, val: u16) -> Operand<'tcx> {
        Operand::Constant(Box::new(ConstOperand {
            span: self.source_info.span,
            user_ty: None,
            const_: Const::from_usize(self.tcx(), val.into()),
        }))
    }

    fn constant_bool(&self, val: bool) -> Operand<'tcx> {
        Operand::Constant(Box::new(ConstOperand {
            span: self.source_info.span,
            user_ty: None,
            const_: Const::from_bool(self.tcx(), val),
        }))
    }

    fn assign(&self, lhs: Place<'tcx>, rhs: Rvalue<'tcx>) -> Statement<'tcx> {
        Statement::new(self.source_info, StatementKind::Assign(Box::new((lhs, rhs))))
    }

    fn storage_live(&self, local: Local) -> Statement<'tcx> {
        Statement::new(self.source_info, StatementKind::StorageLive(local))
    }

    fn storage_dead(&self, local: Local) -> Statement<'tcx> {
        Statement::new(self.source_info, StatementKind::StorageDead(local))
    }
}