rustc_mir_transform/
shim.rs

1use std::assert_matches::assert_matches;
2use std::{fmt, iter};
3
4use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx, VariantIdx};
5use rustc_hir as hir;
6use rustc_hir::def_id::DefId;
7use rustc_hir::lang_items::LangItem;
8use rustc_index::{Idx, IndexVec};
9use rustc_middle::mir::visit::{MutVisitor, PlaceContext};
10use rustc_middle::mir::*;
11use rustc_middle::query::Providers;
12use rustc_middle::ty::{
13    self, CoroutineArgs, CoroutineArgsExt, EarlyBinder, GenericArgs, Ty, TyCtxt,
14};
15use rustc_middle::{bug, span_bug};
16use rustc_span::source_map::{Spanned, dummy_spanned};
17use rustc_span::{DUMMY_SP, Span};
18use tracing::{debug, instrument};
19
20use crate::deref_separator::deref_finder;
21use crate::elaborate_drop::{DropElaborator, DropFlagMode, DropStyle, Unwind, elaborate_drop};
22use crate::patch::MirPatch;
23use crate::{
24    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, inline, instsimplify,
25    mentioned_items, pass_manager as pm, remove_noop_landing_pads, run_optimization_passes,
26    simplify,
27};
28
29mod async_destructor_ctor;
30
31pub(super) fn provide(providers: &mut Providers) {
32    providers.mir_shims = make_shim;
33}
34
35// Replace Pin<&mut ImplCoroutine> accesses (_1.0) into Pin<&mut ProxyCoroutine> accesses
36struct FixProxyFutureDropVisitor<'tcx> {
37    tcx: TyCtxt<'tcx>,
38    replace_to: Local,
39}
40
41impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> {
42    fn tcx(&self) -> TyCtxt<'tcx> {
43        self.tcx
44    }
45
46    fn visit_place(
47        &mut self,
48        place: &mut Place<'tcx>,
49        _context: PlaceContext,
50        _location: Location,
51    ) {
52        if place.local == Local::from_u32(1) {
53            if place.projection.len() == 1 {
54                assert!(matches!(
55                    place.projection.first(),
56                    Some(ProjectionElem::Field(FieldIdx::ZERO, _))
57                ));
58                *place = Place::from(self.replace_to);
59            } else if place.projection.len() == 2 {
60                assert!(matches!(place.projection[0], ProjectionElem::Field(FieldIdx::ZERO, _)));
61                assert!(matches!(place.projection[1], ProjectionElem::Deref));
62                *place =
63                    Place::from(self.replace_to).project_deeper(&[ProjectionElem::Deref], self.tcx);
64            }
65        }
66    }
67}
68
69fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> {
70    debug!("make_shim({:?})", instance);
71
72    let mut result = match instance {
73        ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance),
74        ty::InstanceKind::VTableShim(def_id) => {
75            let adjustment = Adjustment::Deref { source: DerefSource::MutPtr };
76            build_call_shim(tcx, instance, Some(adjustment), CallKind::Direct(def_id))
77        }
78        ty::InstanceKind::FnPtrShim(def_id, ty) => {
79            let trait_ = tcx.parent(def_id);
80            // Supports `Fn` or `async Fn` traits.
81            let adjustment = match tcx
82                .fn_trait_kind_from_def_id(trait_)
83                .or_else(|| tcx.async_fn_trait_kind_from_def_id(trait_))
84            {
85                Some(ty::ClosureKind::FnOnce) => Adjustment::Identity,
86                Some(ty::ClosureKind::Fn) => Adjustment::Deref { source: DerefSource::ImmRef },
87                Some(ty::ClosureKind::FnMut) => Adjustment::Deref { source: DerefSource::MutRef },
88                None => bug!("fn pointer {:?} is not an fn", ty),
89            };
90
91            build_call_shim(tcx, instance, Some(adjustment), CallKind::Indirect(ty))
92        }
93        // We are generating a call back to our def-id, which the
94        // codegen backend knows to turn to an actual call, be it
95        // a virtual call, or a direct call to a function for which
96        // indirect calls must be codegen'd differently than direct ones
97        // (such as `#[track_caller]`).
98        ty::InstanceKind::ReifyShim(def_id, _) => {
99            build_call_shim(tcx, instance, None, CallKind::Direct(def_id))
100        }
101        ty::InstanceKind::ClosureOnceShim { call_once: _, track_caller: _ } => {
102            let fn_mut = tcx.require_lang_item(LangItem::FnMut, DUMMY_SP);
103            let call_mut = tcx
104                .associated_items(fn_mut)
105                .in_definition_order()
106                .find(|it| it.is_fn())
107                .unwrap()
108                .def_id;
109
110            build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut))
111        }
112
113        ty::InstanceKind::ConstructCoroutineInClosureShim {
114            coroutine_closure_def_id,
115            receiver_by_ref,
116        } => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref),
117
118        ty::InstanceKind::DropGlue(def_id, ty) => {
119            // FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end
120            // of this function. Is this intentional?
121            if let Some(&ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
122                let coroutine_body = tcx.optimized_mir(coroutine_def_id);
123
124                let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind()
125                else {
126                    bug!()
127                };
128
129                // If this is a regular coroutine, grab its drop shim. If this is a coroutine
130                // that comes from a coroutine-closure, and the kind ty differs from the "maximum"
131                // kind that it supports, then grab the appropriate drop shim. This ensures that
132                // the future returned by `<[coroutine-closure] as AsyncFnOnce>::call_once` will
133                // drop the coroutine-closure's upvars.
134                let body = if id_args.as_coroutine().kind_ty() == args.as_coroutine().kind_ty() {
135                    coroutine_body.coroutine_drop().unwrap()
136                } else {
137                    assert_eq!(
138                        args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(),
139                        ty::ClosureKind::FnOnce
140                    );
141                    tcx.optimized_mir(tcx.coroutine_by_move_body_def_id(coroutine_def_id))
142                        .coroutine_drop()
143                        .unwrap()
144                };
145
146                let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args);
147                debug!("make_shim({:?}) = {:?}", instance, body);
148
149                pm::run_passes(
150                    tcx,
151                    &mut body,
152                    &[
153                        &mentioned_items::MentionedItems,
154                        &abort_unwinding_calls::AbortUnwindingCalls,
155                        &add_call_guards::CriticalCallEdges,
156                    ],
157                    Some(MirPhase::Runtime(RuntimePhase::Optimized)),
158                    pm::Optimizations::Allowed,
159                );
160
161                return body;
162            }
163
164            build_drop_shim(tcx, def_id, ty)
165        }
166        ty::InstanceKind::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance),
167        ty::InstanceKind::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty),
168        ty::InstanceKind::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty),
169        ty::InstanceKind::FutureDropPollShim(def_id, proxy_ty, impl_ty) => {
170            let mut body =
171                async_destructor_ctor::build_future_drop_poll_shim(tcx, def_id, proxy_ty, impl_ty);
172
173            pm::run_passes(
174                tcx,
175                &mut body,
176                &[
177                    &mentioned_items::MentionedItems,
178                    &abort_unwinding_calls::AbortUnwindingCalls,
179                    &add_call_guards::CriticalCallEdges,
180                ],
181                Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
182                pm::Optimizations::Allowed,
183            );
184            run_optimization_passes(tcx, &mut body);
185            debug!("make_shim({:?}) = {:?}", instance, body);
186            return body;
187        }
188        ty::InstanceKind::AsyncDropGlue(def_id, ty) => {
189            let mut body = async_destructor_ctor::build_async_drop_shim(tcx, def_id, ty);
190
191            // Main pass required here is StateTransform to convert sync drop ladder
192            // into coroutine.
193            // Others are minimal passes as for sync drop glue shim
194            pm::run_passes(
195                tcx,
196                &mut body,
197                &[
198                    &mentioned_items::MentionedItems,
199                    &abort_unwinding_calls::AbortUnwindingCalls,
200                    &add_call_guards::CriticalCallEdges,
201                    &simplify::SimplifyCfg::MakeShim,
202                    &crate::coroutine::StateTransform,
203                ],
204                Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
205                pm::Optimizations::Allowed,
206            );
207            run_optimization_passes(tcx, &mut body);
208            debug!("make_shim({:?}) = {:?}", instance, body);
209            return body;
210        }
211
212        ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty) => {
213            let body = async_destructor_ctor::build_async_destructor_ctor_shim(tcx, def_id, ty);
214            debug!("make_shim({:?}) = {:?}", instance, body);
215            return body;
216        }
217        ty::InstanceKind::Virtual(..) => {
218            bug!("InstanceKind::Virtual ({:?}) is for direct calls only", instance)
219        }
220        ty::InstanceKind::Intrinsic(_) => {
221            bug!("creating shims from intrinsics ({:?}) is unsupported", instance)
222        }
223    };
224    debug!("make_shim({:?}) = untransformed {:?}", instance, result);
225
226    deref_finder(tcx, &mut result, false);
227
228    // We don't validate MIR here because the shims may generate code that's
229    // only valid in a `PostAnalysis` param-env. However, since we do initial
230    // validation with the MirBuilt phase, which uses a user-facing param-env.
231    // This causes validation errors when TAITs are involved.
232    pm::run_passes_no_validate(
233        tcx,
234        &mut result,
235        &[
236            &mentioned_items::MentionedItems,
237            &add_moves_for_packed_drops::AddMovesForPackedDrops,
238            &remove_noop_landing_pads::RemoveNoopLandingPads,
239            &simplify::SimplifyCfg::MakeShim,
240            &instsimplify::InstSimplify::BeforeInline,
241            // Perform inlining of `#[rustc_force_inline]`-annotated callees.
242            &inline::ForceInline,
243            &abort_unwinding_calls::AbortUnwindingCalls,
244            &add_call_guards::CriticalCallEdges,
245        ],
246        Some(MirPhase::Runtime(RuntimePhase::Optimized)),
247    );
248
249    debug!("make_shim({:?}) = {:?}", instance, result);
250
251    result
252}
253
254#[derive(Copy, Clone, Debug, PartialEq)]
255enum DerefSource {
256    /// `fn shim(&self) { inner(*self )}`.
257    ImmRef,
258    /// `fn shim(&mut self) { inner(*self )}`.
259    MutRef,
260    /// `fn shim(*mut self) { inner(*self )}`.
261    MutPtr,
262}
263
264#[derive(Copy, Clone, Debug, PartialEq)]
265enum Adjustment {
266    /// Pass the receiver as-is.
267    Identity,
268
269    /// We get passed a reference or a raw pointer to `self` and call the target with `*self`.
270    ///
271    /// This either copies `self` (if `Self: Copy`, eg. for function items), or moves out of it
272    /// (for `VTableShim`, which effectively is passed `&own Self`).
273    Deref { source: DerefSource },
274
275    /// We get passed `self: Self` and call the target with `&mut self`.
276    ///
277    /// In this case we need to ensure that the `Self` is dropped after the call, as the callee
278    /// won't do it for us.
279    RefMut,
280}
281
282#[derive(Copy, Clone, Debug, PartialEq)]
283enum CallKind<'tcx> {
284    /// Call the `FnPtr` that was passed as the receiver.
285    Indirect(Ty<'tcx>),
286
287    /// Call a known `FnDef`.
288    Direct(DefId),
289}
290
291fn local_decls_for_sig<'tcx>(
292    sig: &ty::FnSig<'tcx>,
293    span: Span,
294) -> IndexVec<Local, LocalDecl<'tcx>> {
295    iter::once(LocalDecl::new(sig.output(), span))
296        .chain(sig.inputs().iter().map(|ity| LocalDecl::new(*ity, span).immutable()))
297        .collect()
298}
299
300fn dropee_emit_retag<'tcx>(
301    tcx: TyCtxt<'tcx>,
302    body: &mut Body<'tcx>,
303    mut dropee_ptr: Place<'tcx>,
304    span: Span,
305) -> Place<'tcx> {
306    if tcx.sess.opts.unstable_opts.mir_emit_retag {
307        let source_info = SourceInfo::outermost(span);
308        // We want to treat the function argument as if it was passed by `&mut`. As such, we
309        // generate
310        // ```
311        // temp = &mut *arg;
312        // Retag(temp, FnEntry)
313        // ```
314        // It's important that we do this first, before anything that depends on `dropee_ptr`
315        // has been put into the body.
316        let reborrow = Rvalue::Ref(
317            tcx.lifetimes.re_erased,
318            BorrowKind::Mut { kind: MutBorrowKind::Default },
319            tcx.mk_place_deref(dropee_ptr),
320        );
321        let ref_ty = reborrow.ty(body.local_decls(), tcx);
322        dropee_ptr = body.local_decls.push(LocalDecl::new(ref_ty, span)).into();
323        let new_statements = [
324            StatementKind::Assign(Box::new((dropee_ptr, reborrow))),
325            StatementKind::Retag(RetagKind::FnEntry, Box::new(dropee_ptr)),
326        ];
327        for s in new_statements {
328            body.basic_blocks_mut()[START_BLOCK].statements.push(Statement::new(source_info, s));
329        }
330    }
331    dropee_ptr
332}
333
334fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> {
335    debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty);
336
337    assert!(!matches!(ty, Some(ty) if ty.is_coroutine()));
338
339    let args = if let Some(ty) = ty {
340        tcx.mk_args(&[ty.into()])
341    } else {
342        GenericArgs::identity_for_item(tcx, def_id)
343    };
344    let sig = tcx.fn_sig(def_id).instantiate(tcx, args);
345    let sig = tcx.instantiate_bound_regions_with_erased(sig);
346    let span = tcx.def_span(def_id);
347
348    let source_info = SourceInfo::outermost(span);
349
350    let return_block = BasicBlock::new(1);
351    let mut blocks = IndexVec::with_capacity(2);
352    let block = |blocks: &mut IndexVec<_, _>, kind| {
353        blocks.push(BasicBlockData::new(Some(Terminator { source_info, kind }), false))
354    };
355    block(&mut blocks, TerminatorKind::Goto { target: return_block });
356    block(&mut blocks, TerminatorKind::Return);
357
358    let source = MirSource::from_instance(ty::InstanceKind::DropGlue(def_id, ty));
359    let mut body =
360        new_body(source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span);
361
362    // The first argument (index 0), but add 1 for the return value.
363    let dropee_ptr = Place::from(Local::new(1 + 0));
364    let dropee_ptr = dropee_emit_retag(tcx, &mut body, dropee_ptr, span);
365
366    if ty.is_some() {
367        let patch = {
368            let typing_env = ty::TypingEnv::post_analysis(tcx, def_id);
369            let mut elaborator = DropShimElaborator {
370                body: &body,
371                patch: MirPatch::new(&body),
372                tcx,
373                typing_env,
374                produce_async_drops: false,
375            };
376            let dropee = tcx.mk_place_deref(dropee_ptr);
377            let resume_block = elaborator.patch.resume_block();
378            elaborate_drop(
379                &mut elaborator,
380                source_info,
381                dropee,
382                (),
383                return_block,
384                Unwind::To(resume_block),
385                START_BLOCK,
386                None,
387            );
388            elaborator.patch
389        };
390        patch.apply(&mut body);
391    }
392
393    body
394}
395
396fn new_body<'tcx>(
397    source: MirSource<'tcx>,
398    basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
399    local_decls: IndexVec<Local, LocalDecl<'tcx>>,
400    arg_count: usize,
401    span: Span,
402) -> Body<'tcx> {
403    let mut body = Body::new(
404        source,
405        basic_blocks,
406        IndexVec::from_elem_n(
407            SourceScopeData {
408                span,
409                parent_scope: None,
410                inlined: None,
411                inlined_parent_scope: None,
412                local_data: ClearCrossCrate::Clear,
413            },
414            1,
415        ),
416        local_decls,
417        IndexVec::new(),
418        arg_count,
419        vec![],
420        span,
421        None,
422        // FIXME(compiler-errors): is this correct?
423        None,
424    );
425    // Shims do not directly mention any consts.
426    body.set_required_consts(Vec::new());
427    body
428}
429
430pub(super) struct DropShimElaborator<'a, 'tcx> {
431    pub body: &'a Body<'tcx>,
432    pub patch: MirPatch<'tcx>,
433    pub tcx: TyCtxt<'tcx>,
434    pub typing_env: ty::TypingEnv<'tcx>,
435    pub produce_async_drops: bool,
436}
437
438impl fmt::Debug for DropShimElaborator<'_, '_> {
439    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
440        f.debug_struct("DropShimElaborator").finish_non_exhaustive()
441    }
442}
443
444impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> {
445    type Path = ();
446
447    fn patch_ref(&self) -> &MirPatch<'tcx> {
448        &self.patch
449    }
450    fn patch(&mut self) -> &mut MirPatch<'tcx> {
451        &mut self.patch
452    }
453    fn body(&self) -> &'a Body<'tcx> {
454        self.body
455    }
456    fn tcx(&self) -> TyCtxt<'tcx> {
457        self.tcx
458    }
459    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
460        self.typing_env
461    }
462
463    fn terminator_loc(&self, bb: BasicBlock) -> Location {
464        self.patch.terminator_loc(self.body, bb)
465    }
466    fn allow_async_drops(&self) -> bool {
467        self.produce_async_drops
468    }
469
470    fn drop_style(&self, _path: Self::Path, mode: DropFlagMode) -> DropStyle {
471        match mode {
472            DropFlagMode::Shallow => {
473                // Drops for the contained fields are "shallow" and "static" - they will simply call
474                // the field's own drop glue.
475                DropStyle::Static
476            }
477            DropFlagMode::Deep => {
478                // The top-level drop is "deep" and "open" - it will be elaborated to a drop ladder
479                // dropping each field contained in the value.
480                DropStyle::Open
481            }
482        }
483    }
484
485    fn get_drop_flag(&mut self, _path: Self::Path) -> Option<Operand<'tcx>> {
486        None
487    }
488
489    fn clear_drop_flag(&mut self, _location: Location, _path: Self::Path, _mode: DropFlagMode) {}
490
491    fn field_subpath(&self, _path: Self::Path, _field: FieldIdx) -> Option<Self::Path> {
492        None
493    }
494    fn deref_subpath(&self, _path: Self::Path) -> Option<Self::Path> {
495        None
496    }
497    fn downcast_subpath(&self, _path: Self::Path, _variant: VariantIdx) -> Option<Self::Path> {
498        Some(())
499    }
500    fn array_subpath(&self, _path: Self::Path, _index: u64, _size: u64) -> Option<Self::Path> {
501        None
502    }
503}
504
505fn build_thread_local_shim<'tcx>(
506    tcx: TyCtxt<'tcx>,
507    instance: ty::InstanceKind<'tcx>,
508) -> Body<'tcx> {
509    let def_id = instance.def_id();
510
511    let span = tcx.def_span(def_id);
512    let source_info = SourceInfo::outermost(span);
513
514    let blocks = IndexVec::from_raw(vec![BasicBlockData::new_stmts(
515        vec![Statement::new(
516            source_info,
517            StatementKind::Assign(Box::new((
518                Place::return_place(),
519                Rvalue::ThreadLocalRef(def_id),
520            ))),
521        )],
522        Some(Terminator { source_info, kind: TerminatorKind::Return }),
523        false,
524    )]);
525
526    new_body(
527        MirSource::from_instance(instance),
528        blocks,
529        IndexVec::from_raw(vec![LocalDecl::new(tcx.thread_local_ptr_ty(def_id), span)]),
530        0,
531        span,
532    )
533}
534
535/// Builds a `Clone::clone` shim for `self_ty`. Here, `def_id` is `Clone::clone`.
536fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
537    debug!("build_clone_shim(def_id={:?})", def_id);
538
539    let mut builder = CloneShimBuilder::new(tcx, def_id, self_ty);
540
541    let dest = Place::return_place();
542    let src = tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
543
544    match self_ty.kind() {
545        ty::FnDef(..) | ty::FnPtr(..) => builder.copy_shim(),
546        ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()),
547        ty::CoroutineClosure(_, args) => {
548            builder.tuple_like_shim(dest, src, args.as_coroutine_closure().upvar_tys())
549        }
550        ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()),
551        ty::Coroutine(coroutine_def_id, args) => {
552            assert_eq!(tcx.coroutine_movability(*coroutine_def_id), hir::Movability::Movable);
553            builder.coroutine_shim(dest, src, *coroutine_def_id, args.as_coroutine())
554        }
555        _ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty),
556    };
557
558    builder.into_mir()
559}
560
561struct CloneShimBuilder<'tcx> {
562    tcx: TyCtxt<'tcx>,
563    def_id: DefId,
564    local_decls: IndexVec<Local, LocalDecl<'tcx>>,
565    blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
566    span: Span,
567    sig: ty::FnSig<'tcx>,
568}
569
570impl<'tcx> CloneShimBuilder<'tcx> {
571    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Self {
572        // we must instantiate the self_ty because it's
573        // otherwise going to be TySelf and we can't index
574        // or access fields of a Place of type TySelf.
575        let sig = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]);
576        let sig = tcx.instantiate_bound_regions_with_erased(sig);
577        let span = tcx.def_span(def_id);
578
579        CloneShimBuilder {
580            tcx,
581            def_id,
582            local_decls: local_decls_for_sig(&sig, span),
583            blocks: IndexVec::new(),
584            span,
585            sig,
586        }
587    }
588
589    fn into_mir(self) -> Body<'tcx> {
590        let source = MirSource::from_instance(ty::InstanceKind::CloneShim(
591            self.def_id,
592            self.sig.inputs_and_output[0],
593        ));
594        new_body(source, self.blocks, self.local_decls, self.sig.inputs().len(), self.span)
595    }
596
597    fn source_info(&self) -> SourceInfo {
598        SourceInfo::outermost(self.span)
599    }
600
601    fn block(
602        &mut self,
603        statements: Vec<Statement<'tcx>>,
604        kind: TerminatorKind<'tcx>,
605        is_cleanup: bool,
606    ) -> BasicBlock {
607        let source_info = self.source_info();
608        self.blocks.push(BasicBlockData::new_stmts(
609            statements,
610            Some(Terminator { source_info, kind }),
611            is_cleanup,
612        ))
613    }
614
615    /// Gives the index of an upcoming BasicBlock, with an offset.
616    /// offset=0 will give you the index of the next BasicBlock,
617    /// offset=1 will give the index of the next-to-next block,
618    /// offset=-1 will give you the index of the last-created block
619    fn block_index_offset(&self, offset: usize) -> BasicBlock {
620        BasicBlock::new(self.blocks.len() + offset)
621    }
622
623    fn make_statement(&self, kind: StatementKind<'tcx>) -> Statement<'tcx> {
624        Statement::new(self.source_info(), kind)
625    }
626
627    fn copy_shim(&mut self) {
628        let rcvr = self.tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
629        let ret_statement = self.make_statement(StatementKind::Assign(Box::new((
630            Place::return_place(),
631            Rvalue::Use(Operand::Copy(rcvr)),
632        ))));
633        self.block(vec![ret_statement], TerminatorKind::Return, false);
634    }
635
636    fn make_place(&mut self, mutability: Mutability, ty: Ty<'tcx>) -> Place<'tcx> {
637        let span = self.span;
638        let mut local = LocalDecl::new(ty, span);
639        if mutability.is_not() {
640            local = local.immutable();
641        }
642        Place::from(self.local_decls.push(local))
643    }
644
645    fn make_clone_call(
646        &mut self,
647        dest: Place<'tcx>,
648        src: Place<'tcx>,
649        ty: Ty<'tcx>,
650        next: BasicBlock,
651        cleanup: BasicBlock,
652    ) {
653        let tcx = self.tcx;
654
655        // `func == Clone::clone(&ty) -> ty`
656        let func_ty = Ty::new_fn_def(tcx, self.def_id, [ty]);
657        let func = Operand::Constant(Box::new(ConstOperand {
658            span: self.span,
659            user_ty: None,
660            const_: Const::zero_sized(func_ty),
661        }));
662
663        let ref_loc =
664            self.make_place(Mutability::Not, Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, ty));
665
666        // `let ref_loc: &ty = &src;`
667        let statement = self.make_statement(StatementKind::Assign(Box::new((
668            ref_loc,
669            Rvalue::Ref(tcx.lifetimes.re_erased, BorrowKind::Shared, src),
670        ))));
671
672        // `let loc = Clone::clone(ref_loc);`
673        self.block(
674            vec![statement],
675            TerminatorKind::Call {
676                func,
677                args: [Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }].into(),
678                destination: dest,
679                target: Some(next),
680                unwind: UnwindAction::Cleanup(cleanup),
681                call_source: CallSource::Normal,
682                fn_span: self.span,
683            },
684            false,
685        );
686    }
687
688    fn clone_fields<I>(
689        &mut self,
690        dest: Place<'tcx>,
691        src: Place<'tcx>,
692        target: BasicBlock,
693        mut unwind: BasicBlock,
694        tys: I,
695    ) -> BasicBlock
696    where
697        I: IntoIterator<Item = Ty<'tcx>>,
698    {
699        // For an iterator of length n, create 2*n + 1 blocks.
700        for (i, ity) in tys.into_iter().enumerate() {
701            // Each iteration creates two blocks, referred to here as block 2*i and block 2*i + 1.
702            //
703            // Block 2*i attempts to clone the field. If successful it branches to 2*i + 2 (the
704            // next clone block). If unsuccessful it branches to the previous unwind block, which
705            // is initially the `unwind` argument passed to this function.
706            //
707            // Block 2*i + 1 is the unwind block for this iteration. It drops the cloned value
708            // created by block 2*i. We store this block in `unwind` so that the next clone block
709            // will unwind to it if cloning fails.
710
711            let field = FieldIdx::new(i);
712            let src_field = self.tcx.mk_place_field(src, field, ity);
713
714            let dest_field = self.tcx.mk_place_field(dest, field, ity);
715
716            let next_unwind = self.block_index_offset(1);
717            let next_block = self.block_index_offset(2);
718            self.make_clone_call(dest_field, src_field, ity, next_block, unwind);
719            self.block(
720                vec![],
721                TerminatorKind::Drop {
722                    place: dest_field,
723                    target: unwind,
724                    unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
725                    replace: false,
726                    drop: None,
727                    async_fut: None,
728                },
729                /* is_cleanup */ true,
730            );
731            unwind = next_unwind;
732        }
733        // If all clones succeed then we end up here.
734        self.block(vec![], TerminatorKind::Goto { target }, false);
735        unwind
736    }
737
738    fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
739    where
740        I: IntoIterator<Item = Ty<'tcx>>,
741    {
742        self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
743        let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
744        let target = self.block(vec![], TerminatorKind::Return, false);
745
746        let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, tys);
747    }
748
749    fn coroutine_shim(
750        &mut self,
751        dest: Place<'tcx>,
752        src: Place<'tcx>,
753        coroutine_def_id: DefId,
754        args: CoroutineArgs<TyCtxt<'tcx>>,
755    ) {
756        self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
757        let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
758        // This will get overwritten with a switch once we know the target blocks
759        let switch = self.block(vec![], TerminatorKind::Unreachable, false);
760        let unwind = self.clone_fields(dest, src, switch, unwind, args.upvar_tys());
761        let target = self.block(vec![], TerminatorKind::Return, false);
762        let unreachable = self.block(vec![], TerminatorKind::Unreachable, false);
763        let mut cases = Vec::with_capacity(args.state_tys(coroutine_def_id, self.tcx).count());
764        for (index, state_tys) in args.state_tys(coroutine_def_id, self.tcx).enumerate() {
765            let variant_index = VariantIdx::new(index);
766            let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index);
767            let src = self.tcx.mk_place_downcast_unnamed(src, variant_index);
768            let clone_block = self.block_index_offset(1);
769            let start_block = self.block(
770                vec![self.make_statement(StatementKind::SetDiscriminant {
771                    place: Box::new(Place::return_place()),
772                    variant_index,
773                })],
774                TerminatorKind::Goto { target: clone_block },
775                false,
776            );
777            cases.push((index as u128, start_block));
778            let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, state_tys);
779        }
780        let discr_ty = args.discr_ty(self.tcx);
781        let temp = self.make_place(Mutability::Mut, discr_ty);
782        let rvalue = Rvalue::Discriminant(src);
783        let statement = self.make_statement(StatementKind::Assign(Box::new((temp, rvalue))));
784        match &mut self.blocks[switch] {
785            BasicBlockData { statements, terminator: Some(Terminator { kind, .. }), .. } => {
786                statements.push(statement);
787                *kind = TerminatorKind::SwitchInt {
788                    discr: Operand::Move(temp),
789                    targets: SwitchTargets::new(cases.into_iter(), unreachable),
790                };
791            }
792            BasicBlockData { terminator: None, .. } => unreachable!(),
793        }
794    }
795}
796
797/// Builds a "call" shim for `instance`. The shim calls the function specified by `call_kind`,
798/// first adjusting its first argument according to `rcvr_adjustment`.
799#[instrument(level = "debug", skip(tcx), ret)]
800fn build_call_shim<'tcx>(
801    tcx: TyCtxt<'tcx>,
802    instance: ty::InstanceKind<'tcx>,
803    rcvr_adjustment: Option<Adjustment>,
804    call_kind: CallKind<'tcx>,
805) -> Body<'tcx> {
806    // `FnPtrShim` contains the fn pointer type that a call shim is being built for - this is used
807    // to instantiate into the signature of the shim. It is not necessary for users of this
808    // MIR body to perform further instantiations (see `InstanceKind::has_polymorphic_mir_body`).
809    let (sig_args, untuple_args) = if let ty::InstanceKind::FnPtrShim(_, ty) = instance {
810        let sig = tcx.instantiate_bound_regions_with_erased(ty.fn_sig(tcx));
811
812        let untuple_args = sig.inputs();
813
814        // Create substitutions for the `Self` and `Args` generic parameters of the shim body.
815        let arg_tup = Ty::new_tup(tcx, untuple_args);
816
817        (Some([ty.into(), arg_tup.into()]), Some(untuple_args))
818    } else {
819        (None, None)
820    };
821
822    let def_id = instance.def_id();
823
824    let sig = tcx.fn_sig(def_id);
825    let sig = sig.map_bound(|sig| tcx.instantiate_bound_regions_with_erased(sig));
826
827    assert_eq!(sig_args.is_some(), !instance.has_polymorphic_mir_body());
828    let mut sig = if let Some(sig_args) = sig_args {
829        sig.instantiate(tcx, &sig_args)
830    } else {
831        sig.instantiate_identity()
832    };
833
834    if let CallKind::Indirect(fnty) = call_kind {
835        // `sig` determines our local decls, and thus the callee type in the `Call` terminator. This
836        // can only be an `FnDef` or `FnPtr`, but currently will be `Self` since the types come from
837        // the implemented `FnX` trait.
838
839        // Apply the opposite adjustment to the MIR input.
840        let mut inputs_and_output = sig.inputs_and_output.to_vec();
841
842        // Initial signature is `fn(&? Self, Args) -> Self::Output` where `Args` is a tuple of the
843        // fn arguments. `Self` may be passed via (im)mutable reference or by-value.
844        assert_eq!(inputs_and_output.len(), 3);
845
846        // `Self` is always the original fn type `ty`. The MIR call terminator is only defined for
847        // `FnDef` and `FnPtr` callees, not the `Self` type param.
848        let self_arg = &mut inputs_and_output[0];
849        *self_arg = match rcvr_adjustment.unwrap() {
850            Adjustment::Identity => fnty,
851            Adjustment::Deref { source } => match source {
852                DerefSource::ImmRef => Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fnty),
853                DerefSource::MutRef => Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, fnty),
854                DerefSource::MutPtr => Ty::new_mut_ptr(tcx, fnty),
855            },
856            Adjustment::RefMut => bug!("`RefMut` is never used with indirect calls: {instance:?}"),
857        };
858        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
859    }
860
861    // FIXME: Avoid having to adjust the signature both here and in
862    // `fn_sig_for_fn_abi`.
863    if let ty::InstanceKind::VTableShim(..) = instance {
864        // Modify fn(self, ...) to fn(self: *mut Self, ...)
865        let mut inputs_and_output = sig.inputs_and_output.to_vec();
866        let self_arg = &mut inputs_and_output[0];
867        debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param);
868        *self_arg = Ty::new_mut_ptr(tcx, *self_arg);
869        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
870    }
871
872    let span = tcx.def_span(def_id);
873
874    debug!(?sig);
875
876    let mut local_decls = local_decls_for_sig(&sig, span);
877    let source_info = SourceInfo::outermost(span);
878
879    let destination = Place::return_place();
880
881    let rcvr_place = || {
882        assert!(rcvr_adjustment.is_some());
883        Place::from(Local::new(1))
884    };
885    let mut statements = vec![];
886
887    let rcvr = rcvr_adjustment.map(|rcvr_adjustment| match rcvr_adjustment {
888        Adjustment::Identity => Operand::Move(rcvr_place()),
889        Adjustment::Deref { source: _ } => Operand::Move(tcx.mk_place_deref(rcvr_place())),
890        Adjustment::RefMut => {
891            // let rcvr = &mut rcvr;
892            let ref_rcvr = local_decls.push(
893                LocalDecl::new(
894                    Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, sig.inputs()[0]),
895                    span,
896                )
897                .immutable(),
898            );
899            let borrow_kind = BorrowKind::Mut { kind: MutBorrowKind::Default };
900            statements.push(Statement::new(
901                source_info,
902                StatementKind::Assign(Box::new((
903                    Place::from(ref_rcvr),
904                    Rvalue::Ref(tcx.lifetimes.re_erased, borrow_kind, rcvr_place()),
905                ))),
906            ));
907            Operand::Move(Place::from(ref_rcvr))
908        }
909    });
910
911    let (callee, mut args) = match call_kind {
912        // `FnPtr` call has no receiver. Args are untupled below.
913        CallKind::Indirect(_) => (rcvr.unwrap(), vec![]),
914
915        // `FnDef` call with optional receiver.
916        CallKind::Direct(def_id) => {
917            let ty = tcx.type_of(def_id).instantiate_identity();
918            (
919                Operand::Constant(Box::new(ConstOperand {
920                    span,
921                    user_ty: None,
922                    const_: Const::zero_sized(ty),
923                })),
924                rcvr.into_iter().collect::<Vec<_>>(),
925            )
926        }
927    };
928
929    let mut arg_range = 0..sig.inputs().len();
930
931    // Take the `self` ("receiver") argument out of the range (it's adjusted above).
932    if rcvr_adjustment.is_some() {
933        arg_range.start += 1;
934    }
935
936    // Take the last argument, if we need to untuple it (handled below).
937    if untuple_args.is_some() {
938        arg_range.end -= 1;
939    }
940
941    // Pass all of the non-special arguments directly.
942    args.extend(arg_range.map(|i| Operand::Move(Place::from(Local::new(1 + i)))));
943
944    // Untuple the last argument, if we have to.
945    if let Some(untuple_args) = untuple_args {
946        let tuple_arg = Local::new(1 + (sig.inputs().len() - 1));
947        args.extend(untuple_args.iter().enumerate().map(|(i, ity)| {
948            Operand::Move(tcx.mk_place_field(Place::from(tuple_arg), FieldIdx::new(i), *ity))
949        }));
950    }
951
952    let n_blocks = if let Some(Adjustment::RefMut) = rcvr_adjustment { 5 } else { 2 };
953    let mut blocks = IndexVec::with_capacity(n_blocks);
954    let block = |blocks: &mut IndexVec<_, _>, statements, kind, is_cleanup| {
955        blocks.push(BasicBlockData::new_stmts(
956            statements,
957            Some(Terminator { source_info, kind }),
958            is_cleanup,
959        ))
960    };
961
962    // BB #0
963    let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect();
964    block(
965        &mut blocks,
966        statements,
967        TerminatorKind::Call {
968            func: callee,
969            args,
970            destination,
971            target: Some(BasicBlock::new(1)),
972            unwind: if let Some(Adjustment::RefMut) = rcvr_adjustment {
973                UnwindAction::Cleanup(BasicBlock::new(3))
974            } else {
975                UnwindAction::Continue
976            },
977            call_source: CallSource::Misc,
978            fn_span: span,
979        },
980        false,
981    );
982
983    if let Some(Adjustment::RefMut) = rcvr_adjustment {
984        // BB #1 - drop for Self
985        block(
986            &mut blocks,
987            vec![],
988            TerminatorKind::Drop {
989                place: rcvr_place(),
990                target: BasicBlock::new(2),
991                unwind: UnwindAction::Continue,
992                replace: false,
993                drop: None,
994                async_fut: None,
995            },
996            false,
997        );
998    }
999    // BB #1/#2 - return
1000    let stmts = vec![];
1001    block(&mut blocks, stmts, TerminatorKind::Return, false);
1002    if let Some(Adjustment::RefMut) = rcvr_adjustment {
1003        // BB #3 - drop if closure panics
1004        block(
1005            &mut blocks,
1006            vec![],
1007            TerminatorKind::Drop {
1008                place: rcvr_place(),
1009                target: BasicBlock::new(4),
1010                unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
1011                replace: false,
1012                drop: None,
1013                async_fut: None,
1014            },
1015            /* is_cleanup */ true,
1016        );
1017
1018        // BB #4 - resume
1019        block(&mut blocks, vec![], TerminatorKind::UnwindResume, true);
1020    }
1021
1022    let mut body =
1023        new_body(MirSource::from_instance(instance), blocks, local_decls, sig.inputs().len(), span);
1024
1025    if let ExternAbi::RustCall = sig.abi {
1026        body.spread_arg = Some(Local::new(sig.inputs().len()));
1027    }
1028
1029    body
1030}
1031
1032pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
1033    debug_assert!(tcx.is_constructor(ctor_id));
1034
1035    let typing_env = ty::TypingEnv::post_analysis(tcx, ctor_id);
1036
1037    // Normalize the sig.
1038    let sig = tcx
1039        .fn_sig(ctor_id)
1040        .instantiate_identity()
1041        .no_bound_vars()
1042        .expect("LBR in ADT constructor signature");
1043    let sig = tcx.normalize_erasing_regions(typing_env, sig);
1044
1045    let ty::Adt(adt_def, args) = sig.output().kind() else {
1046        bug!("unexpected type for ADT ctor {:?}", sig.output());
1047    };
1048
1049    debug!("build_ctor: ctor_id={:?} sig={:?}", ctor_id, sig);
1050
1051    let span = tcx.def_span(ctor_id);
1052
1053    let local_decls = local_decls_for_sig(&sig, span);
1054
1055    let source_info = SourceInfo::outermost(span);
1056
1057    let variant_index =
1058        if adt_def.is_enum() { adt_def.variant_index_with_ctor_id(ctor_id) } else { FIRST_VARIANT };
1059
1060    // Generate the following MIR:
1061    //
1062    // (return as Variant).field0 = arg0;
1063    // (return as Variant).field1 = arg1;
1064    //
1065    // return;
1066    debug!("build_ctor: variant_index={:?}", variant_index);
1067
1068    let kind = AggregateKind::Adt(adt_def.did(), variant_index, args, None, None);
1069    let variant = adt_def.variant(variant_index);
1070    let statement = Statement::new(
1071        source_info,
1072        StatementKind::Assign(Box::new((
1073            Place::return_place(),
1074            Rvalue::Aggregate(
1075                Box::new(kind),
1076                (0..variant.fields.len())
1077                    .map(|idx| Operand::Move(Place::from(Local::new(idx + 1))))
1078                    .collect(),
1079            ),
1080        ))),
1081    );
1082
1083    let start_block = BasicBlockData::new_stmts(
1084        vec![statement],
1085        Some(Terminator { source_info, kind: TerminatorKind::Return }),
1086        false,
1087    );
1088
1089    let source = MirSource::item(ctor_id);
1090    let mut body = new_body(
1091        source,
1092        IndexVec::from_elem_n(start_block, 1),
1093        local_decls,
1094        sig.inputs().len(),
1095        span,
1096    );
1097    // A constructor doesn't mention any other items (and we don't run the usual optimization passes
1098    // so this would otherwise not get filled).
1099    body.set_mentioned_items(Vec::new());
1100
1101    crate::pass_manager::dump_mir_for_phase_change(tcx, &body);
1102
1103    body
1104}
1105
1106/// ```ignore (pseudo-impl)
1107/// impl FnPtr for fn(u32) {
1108///     fn addr(self) -> usize {
1109///         self as usize
1110///     }
1111/// }
1112/// ```
1113fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
1114    assert_matches!(self_ty.kind(), ty::FnPtr(..), "expected fn ptr, found {self_ty}");
1115    let span = tcx.def_span(def_id);
1116    let Some(sig) = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]).no_bound_vars() else {
1117        span_bug!(span, "FnPtr::addr with bound vars for `{self_ty}`");
1118    };
1119    let locals = local_decls_for_sig(&sig, span);
1120
1121    let source_info = SourceInfo::outermost(span);
1122    // FIXME: use `expose_provenance` once we figure out whether function pointers have meaningful
1123    // provenance.
1124    let rvalue = Rvalue::Cast(
1125        CastKind::FnPtrToPtr,
1126        Operand::Move(Place::from(Local::new(1))),
1127        Ty::new_imm_ptr(tcx, tcx.types.unit),
1128    );
1129    let stmt = Statement::new(
1130        source_info,
1131        StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1132    );
1133    let statements = vec![stmt];
1134    let start_block = BasicBlockData::new_stmts(
1135        statements,
1136        Some(Terminator { source_info, kind: TerminatorKind::Return }),
1137        false,
1138    );
1139    let source = MirSource::from_instance(ty::InstanceKind::FnPtrAddrShim(def_id, self_ty));
1140    new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span)
1141}
1142
1143fn build_construct_coroutine_by_move_shim<'tcx>(
1144    tcx: TyCtxt<'tcx>,
1145    coroutine_closure_def_id: DefId,
1146    receiver_by_ref: bool,
1147) -> Body<'tcx> {
1148    let mut self_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity();
1149    let mut self_local: Place<'tcx> = Local::from_usize(1).into();
1150    let ty::CoroutineClosure(_, args) = *self_ty.kind() else {
1151        bug!();
1152    };
1153
1154    // We use `&Self` here because we only need to emit an ABI-compatible shim body,
1155    // rather than match the signature exactly (which might take `&mut self` instead).
1156    //
1157    // We adjust the `self_local` to be a deref since we want to copy fields out of
1158    // a reference to the closure.
1159    if receiver_by_ref {
1160        self_local = tcx.mk_place_deref(self_local);
1161        self_ty = Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, self_ty);
1162    }
1163
1164    let poly_sig = args.as_coroutine_closure().coroutine_closure_sig().map_bound(|sig| {
1165        tcx.mk_fn_sig(
1166            [self_ty].into_iter().chain(sig.tupled_inputs_ty.tuple_fields()),
1167            sig.to_coroutine_given_kind_and_upvars(
1168                tcx,
1169                args.as_coroutine_closure().parent_args(),
1170                tcx.coroutine_for_closure(coroutine_closure_def_id),
1171                ty::ClosureKind::FnOnce,
1172                tcx.lifetimes.re_erased,
1173                args.as_coroutine_closure().tupled_upvars_ty(),
1174                args.as_coroutine_closure().coroutine_captures_by_ref_ty(),
1175            ),
1176            sig.c_variadic,
1177            sig.safety,
1178            sig.abi,
1179        )
1180    });
1181    let sig = tcx.liberate_late_bound_regions(coroutine_closure_def_id, poly_sig);
1182    let ty::Coroutine(coroutine_def_id, coroutine_args) = *sig.output().kind() else {
1183        bug!();
1184    };
1185
1186    let span = tcx.def_span(coroutine_closure_def_id);
1187    let locals = local_decls_for_sig(&sig, span);
1188
1189    let mut fields = vec![];
1190
1191    // Move all of the closure args.
1192    for idx in 1..sig.inputs().len() {
1193        fields.push(Operand::Move(Local::from_usize(idx + 1).into()));
1194    }
1195
1196    for (idx, ty) in args.as_coroutine_closure().upvar_tys().iter().enumerate() {
1197        if receiver_by_ref {
1198            // The only situation where it's possible is when we capture immuatable references,
1199            // since those don't need to be reborrowed with the closure's env lifetime. Since
1200            // references are always `Copy`, just emit a copy.
1201            if !matches!(ty.kind(), ty::Ref(_, _, hir::Mutability::Not)) {
1202                // This copy is only sound if it's a `&T`. This may be
1203                // reachable e.g. when eagerly computing the `Fn` instance
1204                // of an async closure that doesn't borrowck.
1205                tcx.dcx().delayed_bug(format!(
1206                    "field should be captured by immutable ref if we have \
1207                    an `Fn` instance, but it was: {ty}"
1208                ));
1209            }
1210            fields.push(Operand::Copy(tcx.mk_place_field(
1211                self_local,
1212                FieldIdx::from_usize(idx),
1213                ty,
1214            )));
1215        } else {
1216            fields.push(Operand::Move(tcx.mk_place_field(
1217                self_local,
1218                FieldIdx::from_usize(idx),
1219                ty,
1220            )));
1221        }
1222    }
1223
1224    let source_info = SourceInfo::outermost(span);
1225    let rvalue = Rvalue::Aggregate(
1226        Box::new(AggregateKind::Coroutine(coroutine_def_id, coroutine_args)),
1227        IndexVec::from_raw(fields),
1228    );
1229    let stmt = Statement::new(
1230        source_info,
1231        StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1232    );
1233    let statements = vec![stmt];
1234    let start_block = BasicBlockData::new_stmts(
1235        statements,
1236        Some(Terminator { source_info, kind: TerminatorKind::Return }),
1237        false,
1238    );
1239
1240    let source = MirSource::from_instance(ty::InstanceKind::ConstructCoroutineInClosureShim {
1241        coroutine_closure_def_id,
1242        receiver_by_ref,
1243    });
1244
1245    let body =
1246        new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span);
1247
1248    let pass_name =
1249        if receiver_by_ref { "coroutine_closure_by_ref" } else { "coroutine_closure_by_move" };
1250    if let Some(dumper) = MirDumper::new(tcx, pass_name, &body) {
1251        dumper.dump_mir(&body);
1252    }
1253
1254    body
1255}