rustc_mir_transform/
shim.rs

Help
1use std::assert_matches::assert_matches;
2use std::{fmt, iter};
3
4use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx, VariantIdx};
5use rustc_hir as hir;
6use rustc_hir::def_id::DefId;
7use rustc_hir::lang_items::LangItem;
8use rustc_index::{Idx, IndexVec};
9use rustc_middle::mir::*;
10use rustc_middle::query::Providers;
11use rustc_middle::ty::adjustment::PointerCoercion;
12use rustc_middle::ty::{
13    self, CoroutineArgs, CoroutineArgsExt, EarlyBinder, GenericArgs, Ty, TyCtxt,
14};
15use rustc_middle::{bug, span_bug};
16use rustc_span::source_map::Spanned;
17use rustc_span::{DUMMY_SP, Span};
18use tracing::{debug, instrument};
19
20use crate::elaborate_drop::{DropElaborator, DropFlagMode, DropStyle, Unwind, elaborate_drop};
21use crate::patch::MirPatch;
22use crate::{
23    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, inline,
24    instsimplify, mentioned_items, pass_manager as pm, remove_noop_landing_pads, simplify,
25};
26
27mod async_destructor_ctor;
28
29pub(super) fn provide(providers: &mut Providers) {
30    providers.mir_shims = make_shim;
31}
32
33fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> {
34    debug!("make_shim({:?})", instance);
35
36    let mut result = match instance {
37        ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance),
38        ty::InstanceKind::VTableShim(def_id) => {
39            let adjustment = Adjustment::Deref { source: DerefSource::MutPtr };
40            build_call_shim(tcx, instance, Some(adjustment), CallKind::Direct(def_id))
41        }
42        ty::InstanceKind::FnPtrShim(def_id, ty) => {
43            let trait_ = tcx.trait_of_item(def_id).unwrap();
44            // Supports `Fn` or `async Fn` traits.
45            let adjustment = match tcx
46                .fn_trait_kind_from_def_id(trait_)
47                .or_else(|| tcx.async_fn_trait_kind_from_def_id(trait_))
48            {
49                Some(ty::ClosureKind::FnOnce) => Adjustment::Identity,
50                Some(ty::ClosureKind::Fn) => Adjustment::Deref { source: DerefSource::ImmRef },
51                Some(ty::ClosureKind::FnMut) => Adjustment::Deref { source: DerefSource::MutRef },
52                None => bug!("fn pointer {:?} is not an fn", ty),
53            };
54
55            build_call_shim(tcx, instance, Some(adjustment), CallKind::Indirect(ty))
56        }
57        // We are generating a call back to our def-id, which the
58        // codegen backend knows to turn to an actual call, be it
59        // a virtual call, or a direct call to a function for which
60        // indirect calls must be codegen'd differently than direct ones
61        // (such as `#[track_caller]`).
62        ty::InstanceKind::ReifyShim(def_id, _) => {
63            build_call_shim(tcx, instance, None, CallKind::Direct(def_id))
64        }
65        ty::InstanceKind::ClosureOnceShim { call_once: _, track_caller: _ } => {
66            let fn_mut = tcx.require_lang_item(LangItem::FnMut, None);
67            let call_mut = tcx
68                .associated_items(fn_mut)
69                .in_definition_order()
70                .find(|it| it.kind == ty::AssocKind::Fn)
71                .unwrap()
72                .def_id;
73
74            build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut))
75        }
76
77        ty::InstanceKind::ConstructCoroutineInClosureShim {
78            coroutine_closure_def_id,
79            receiver_by_ref,
80        } => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref),
81
82        ty::InstanceKind::DropGlue(def_id, ty) => {
83            // FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end
84            // of this function. Is this intentional?
85            if let Some(&ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
86                let coroutine_body = tcx.optimized_mir(coroutine_def_id);
87
88                let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind()
89                else {
90                    bug!()
91                };
92
93                // If this is a regular coroutine, grab its drop shim. If this is a coroutine
94                // that comes from a coroutine-closure, and the kind ty differs from the "maximum"
95                // kind that it supports, then grab the appropriate drop shim. This ensures that
96                // the future returned by `<[coroutine-closure] as AsyncFnOnce>::call_once` will
97                // drop the coroutine-closure's upvars.
98                let body = if id_args.as_coroutine().kind_ty() == args.as_coroutine().kind_ty() {
99                    coroutine_body.coroutine_drop().unwrap()
100                } else {
101                    assert_eq!(
102                        args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(),
103                        ty::ClosureKind::FnOnce
104                    );
105                    tcx.optimized_mir(tcx.coroutine_by_move_body_def_id(coroutine_def_id))
106                        .coroutine_drop()
107                        .unwrap()
108                };
109
110                let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args);
111                debug!("make_shim({:?}) = {:?}", instance, body);
112
113                pm::run_passes(
114                    tcx,
115                    &mut body,
116                    &[
117                        &mentioned_items::MentionedItems,
118                        &abort_unwinding_calls::AbortUnwindingCalls,
119                        &add_call_guards::CriticalCallEdges,
120                    ],
121                    Some(MirPhase::Runtime(RuntimePhase::Optimized)),
122                    pm::Optimizations::Allowed,
123                );
124
125                return body;
126            }
127
128            build_drop_shim(tcx, def_id, ty)
129        }
130        ty::InstanceKind::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance),
131        ty::InstanceKind::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty),
132        ty::InstanceKind::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty),
133        ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty) => {
134            async_destructor_ctor::build_async_destructor_ctor_shim(tcx, def_id, ty)
135        }
136        ty::InstanceKind::Virtual(..) => {
137            bug!("InstanceKind::Virtual ({:?}) is for direct calls only", instance)
138        }
139        ty::InstanceKind::Intrinsic(_) => {
140            bug!("creating shims from intrinsics ({:?}) is unsupported", instance)
141        }
142    };
143    debug!("make_shim({:?}) = untransformed {:?}", instance, result);
144
145    // We don't validate MIR here because the shims may generate code that's
146    // only valid in a `PostAnalysis` param-env. However, since we do initial
147    // validation with the MirBuilt phase, which uses a user-facing param-env.
148    // This causes validation errors when TAITs are involved.
149    pm::run_passes_no_validate(
150        tcx,
151        &mut result,
152        &[
153            &mentioned_items::MentionedItems,
154            &add_moves_for_packed_drops::AddMovesForPackedDrops,
155            &deref_separator::Derefer,
156            &remove_noop_landing_pads::RemoveNoopLandingPads,
157            &simplify::SimplifyCfg::MakeShim,
158            &instsimplify::InstSimplify::BeforeInline,
159            // Perform inlining of `#[rustc_force_inline]`-annotated callees.
160            &inline::ForceInline,
161            &abort_unwinding_calls::AbortUnwindingCalls,
162            &add_call_guards::CriticalCallEdges,
163        ],
164        Some(MirPhase::Runtime(RuntimePhase::Optimized)),
165    );
166
167    debug!("make_shim({:?}) = {:?}", instance, result);
168
169    result
170}
171
172#[derive(Copy, Clone, Debug, PartialEq)]
173enum DerefSource {
174    /// `fn shim(&self) { inner(*self )}`.
175    ImmRef,
176    /// `fn shim(&mut self) { inner(*self )}`.
177    MutRef,
178    /// `fn shim(*mut self) { inner(*self )}`.
179    MutPtr,
180}
181
182#[derive(Copy, Clone, Debug, PartialEq)]
183enum Adjustment {
184    /// Pass the receiver as-is.
185    Identity,
186
187    /// We get passed a reference or a raw pointer to `self` and call the target with `*self`.
188    ///
189    /// This either copies `self` (if `Self: Copy`, eg. for function items), or moves out of it
190    /// (for `VTableShim`, which effectively is passed `&own Self`).
191    Deref { source: DerefSource },
192
193    /// We get passed `self: Self` and call the target with `&mut self`.
194    ///
195    /// In this case we need to ensure that the `Self` is dropped after the call, as the callee
196    /// won't do it for us.
197    RefMut,
198}
199
200#[derive(Copy, Clone, Debug, PartialEq)]
201enum CallKind<'tcx> {
202    /// Call the `FnPtr` that was passed as the receiver.
203    Indirect(Ty<'tcx>),
204
205    /// Call a known `FnDef`.
206    Direct(DefId),
207}
208
209fn local_decls_for_sig<'tcx>(
210    sig: &ty::FnSig<'tcx>,
211    span: Span,
212) -> IndexVec<Local, LocalDecl<'tcx>> {
213    iter::once(LocalDecl::new(sig.output(), span))
214        .chain(sig.inputs().iter().map(|ity| LocalDecl::new(*ity, span).immutable()))
215        .collect()
216}
217
218fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> {
219    debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty);
220
221    assert!(!matches!(ty, Some(ty) if ty.is_coroutine()));
222
223    let args = if let Some(ty) = ty {
224        tcx.mk_args(&[ty.into()])
225    } else {
226        GenericArgs::identity_for_item(tcx, def_id)
227    };
228    let sig = tcx.fn_sig(def_id).instantiate(tcx, args);
229    let sig = tcx.instantiate_bound_regions_with_erased(sig);
230    let span = tcx.def_span(def_id);
231
232    let source_info = SourceInfo::outermost(span);
233
234    let return_block = BasicBlock::new(1);
235    let mut blocks = IndexVec::with_capacity(2);
236    let block = |blocks: &mut IndexVec<_, _>, kind| {
237        blocks.push(BasicBlockData {
238            statements: vec![],
239            terminator: Some(Terminator { source_info, kind }),
240            is_cleanup: false,
241        })
242    };
243    block(&mut blocks, TerminatorKind::Goto { target: return_block });
244    block(&mut blocks, TerminatorKind::Return);
245
246    let source = MirSource::from_instance(ty::InstanceKind::DropGlue(def_id, ty));
247    let mut body =
248        new_body(source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span);
249
250    // The first argument (index 0), but add 1 for the return value.
251    let mut dropee_ptr = Place::from(Local::new(1 + 0));
252    if tcx.sess.opts.unstable_opts.mir_emit_retag {
253        // We want to treat the function argument as if it was passed by `&mut`. As such, we
254        // generate
255        // ```
256        // temp = &mut *arg;
257        // Retag(temp, FnEntry)
258        // ```
259        // It's important that we do this first, before anything that depends on `dropee_ptr`
260        // has been put into the body.
261        let reborrow = Rvalue::Ref(
262            tcx.lifetimes.re_erased,
263            BorrowKind::Mut { kind: MutBorrowKind::Default },
264            tcx.mk_place_deref(dropee_ptr),
265        );
266        let ref_ty = reborrow.ty(body.local_decls(), tcx);
267        dropee_ptr = body.local_decls.push(LocalDecl::new(ref_ty, span)).into();
268        let new_statements = [
269            StatementKind::Assign(Box::new((dropee_ptr, reborrow))),
270            StatementKind::Retag(RetagKind::FnEntry, Box::new(dropee_ptr)),
271        ];
272        for s in new_statements {
273            body.basic_blocks_mut()[START_BLOCK]
274                .statements
275                .push(Statement { source_info, kind: s });
276        }
277    }
278
279    if ty.is_some() {
280        let patch = {
281            let typing_env = ty::TypingEnv::post_analysis(tcx, def_id);
282            let mut elaborator =
283                DropShimElaborator { body: &body, patch: MirPatch::new(&body), tcx, typing_env };
284            let dropee = tcx.mk_place_deref(dropee_ptr);
285            let resume_block = elaborator.patch.resume_block();
286            elaborate_drop(
287                &mut elaborator,
288                source_info,
289                dropee,
290                (),
291                return_block,
292                Unwind::To(resume_block),
293                START_BLOCK,
294            );
295            elaborator.patch
296        };
297        patch.apply(&mut body);
298    }
299
300    body
301}
302
303fn new_body<'tcx>(
304    source: MirSource<'tcx>,
305    basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
306    local_decls: IndexVec<Local, LocalDecl<'tcx>>,
307    arg_count: usize,
308    span: Span,
309) -> Body<'tcx> {
310    let mut body = Body::new(
311        source,
312        basic_blocks,
313        IndexVec::from_elem_n(
314            SourceScopeData {
315                span,
316                parent_scope: None,
317                inlined: None,
318                inlined_parent_scope: None,
319                local_data: ClearCrossCrate::Clear,
320            },
321            1,
322        ),
323        local_decls,
324        IndexVec::new(),
325        arg_count,
326        vec![],
327        span,
328        None,
329        // FIXME(compiler-errors): is this correct?
330        None,
331    );
332    // Shims do not directly mention any consts.
333    body.set_required_consts(Vec::new());
334    body
335}
336
337pub(super) struct DropShimElaborator<'a, 'tcx> {
338    pub body: &'a Body<'tcx>,
339    pub patch: MirPatch<'tcx>,
340    pub tcx: TyCtxt<'tcx>,
341    pub typing_env: ty::TypingEnv<'tcx>,
342}
343
344impl fmt::Debug for DropShimElaborator<'_, '_> {
345    fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
346        Ok(())
347    }
348}
349
350impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> {
351    type Path = ();
352
353    fn patch(&mut self) -> &mut MirPatch<'tcx> {
354        &mut self.patch
355    }
356    fn body(&self) -> &'a Body<'tcx> {
357        self.body
358    }
359    fn tcx(&self) -> TyCtxt<'tcx> {
360        self.tcx
361    }
362    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
363        self.typing_env
364    }
365
366    fn drop_style(&self, _path: Self::Path, mode: DropFlagMode) -> DropStyle {
367        match mode {
368            DropFlagMode::Shallow => {
369                // Drops for the contained fields are "shallow" and "static" - they will simply call
370                // the field's own drop glue.
371                DropStyle::Static
372            }
373            DropFlagMode::Deep => {
374                // The top-level drop is "deep" and "open" - it will be elaborated to a drop ladder
375                // dropping each field contained in the value.
376                DropStyle::Open
377            }
378        }
379    }
380
381    fn get_drop_flag(&mut self, _path: Self::Path) -> Option<Operand<'tcx>> {
382        None
383    }
384
385    fn clear_drop_flag(&mut self, _location: Location, _path: Self::Path, _mode: DropFlagMode) {}
386
387    fn field_subpath(&self, _path: Self::Path, _field: FieldIdx) -> Option<Self::Path> {
388        None
389    }
390    fn deref_subpath(&self, _path: Self::Path) -> Option<Self::Path> {
391        None
392    }
393    fn downcast_subpath(&self, _path: Self::Path, _variant: VariantIdx) -> Option<Self::Path> {
394        Some(())
395    }
396    fn array_subpath(&self, _path: Self::Path, _index: u64, _size: u64) -> Option<Self::Path> {
397        None
398    }
399}
400
401fn build_thread_local_shim<'tcx>(
402    tcx: TyCtxt<'tcx>,
403    instance: ty::InstanceKind<'tcx>,
404) -> Body<'tcx> {
405    let def_id = instance.def_id();
406
407    let span = tcx.def_span(def_id);
408    let source_info = SourceInfo::outermost(span);
409
410    let blocks = IndexVec::from_raw(vec![BasicBlockData {
411        statements: vec![Statement {
412            source_info,
413            kind: StatementKind::Assign(Box::new((
414                Place::return_place(),
415                Rvalue::ThreadLocalRef(def_id),
416            ))),
417        }],
418        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
419        is_cleanup: false,
420    }]);
421
422    new_body(
423        MirSource::from_instance(instance),
424        blocks,
425        IndexVec::from_raw(vec![LocalDecl::new(tcx.thread_local_ptr_ty(def_id), span)]),
426        0,
427        span,
428    )
429}
430
431/// Builds a `Clone::clone` shim for `self_ty`. Here, `def_id` is `Clone::clone`.
432fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
433    debug!("build_clone_shim(def_id={:?})", def_id);
434
435    let mut builder = CloneShimBuilder::new(tcx, def_id, self_ty);
436
437    let dest = Place::return_place();
438    let src = tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
439
440    match self_ty.kind() {
441        ty::FnDef(..) | ty::FnPtr(..) => builder.copy_shim(),
442        ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()),
443        ty::CoroutineClosure(_, args) => {
444            builder.tuple_like_shim(dest, src, args.as_coroutine_closure().upvar_tys())
445        }
446        ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()),
447        ty::Coroutine(coroutine_def_id, args) => {
448            assert_eq!(tcx.coroutine_movability(*coroutine_def_id), hir::Movability::Movable);
449            builder.coroutine_shim(dest, src, *coroutine_def_id, args.as_coroutine())
450        }
451        _ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty),
452    };
453
454    builder.into_mir()
455}
456
457struct CloneShimBuilder<'tcx> {
458    tcx: TyCtxt<'tcx>,
459    def_id: DefId,
460    local_decls: IndexVec<Local, LocalDecl<'tcx>>,
461    blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
462    span: Span,
463    sig: ty::FnSig<'tcx>,
464}
465
466impl<'tcx> CloneShimBuilder<'tcx> {
467    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Self {
468        // we must instantiate the self_ty because it's
469        // otherwise going to be TySelf and we can't index
470        // or access fields of a Place of type TySelf.
471        let sig = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]);
472        let sig = tcx.instantiate_bound_regions_with_erased(sig);
473        let span = tcx.def_span(def_id);
474
475        CloneShimBuilder {
476            tcx,
477            def_id,
478            local_decls: local_decls_for_sig(&sig, span),
479            blocks: IndexVec::new(),
480            span,
481            sig,
482        }
483    }
484
485    fn into_mir(self) -> Body<'tcx> {
486        let source = MirSource::from_instance(ty::InstanceKind::CloneShim(
487            self.def_id,
488            self.sig.inputs_and_output[0],
489        ));
490        new_body(source, self.blocks, self.local_decls, self.sig.inputs().len(), self.span)
491    }
492
493    fn source_info(&self) -> SourceInfo {
494        SourceInfo::outermost(self.span)
495    }
496
497    fn block(
498        &mut self,
499        statements: Vec<Statement<'tcx>>,
500        kind: TerminatorKind<'tcx>,
501        is_cleanup: bool,
502    ) -> BasicBlock {
503        let source_info = self.source_info();
504        self.blocks.push(BasicBlockData {
505            statements,
506            terminator: Some(Terminator { source_info, kind }),
507            is_cleanup,
508        })
509    }
510
511    /// Gives the index of an upcoming BasicBlock, with an offset.
512    /// offset=0 will give you the index of the next BasicBlock,
513    /// offset=1 will give the index of the next-to-next block,
514    /// offset=-1 will give you the index of the last-created block
515    fn block_index_offset(&self, offset: usize) -> BasicBlock {
516        BasicBlock::new(self.blocks.len() + offset)
517    }
518
519    fn make_statement(&self, kind: StatementKind<'tcx>) -> Statement<'tcx> {
520        Statement { source_info: self.source_info(), kind }
521    }
522
523    fn copy_shim(&mut self) {
524        let rcvr = self.tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
525        let ret_statement = self.make_statement(StatementKind::Assign(Box::new((
526            Place::return_place(),
527            Rvalue::Use(Operand::Copy(rcvr)),
528        ))));
529        self.block(vec![ret_statement], TerminatorKind::Return, false);
530    }
531
532    fn make_place(&mut self, mutability: Mutability, ty: Ty<'tcx>) -> Place<'tcx> {
533        let span = self.span;
534        let mut local = LocalDecl::new(ty, span);
535        if mutability.is_not() {
536            local = local.immutable();
537        }
538        Place::from(self.local_decls.push(local))
539    }
540
541    fn make_clone_call(
542        &mut self,
543        dest: Place<'tcx>,
544        src: Place<'tcx>,
545        ty: Ty<'tcx>,
546        next: BasicBlock,
547        cleanup: BasicBlock,
548    ) {
549        let tcx = self.tcx;
550
551        // `func == Clone::clone(&ty) -> ty`
552        let func_ty = Ty::new_fn_def(tcx, self.def_id, [ty]);
553        let func = Operand::Constant(Box::new(ConstOperand {
554            span: self.span,
555            user_ty: None,
556            const_: Const::zero_sized(func_ty),
557        }));
558
559        let ref_loc =
560            self.make_place(Mutability::Not, Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, ty));
561
562        // `let ref_loc: &ty = &src;`
563        let statement = self.make_statement(StatementKind::Assign(Box::new((
564            ref_loc,
565            Rvalue::Ref(tcx.lifetimes.re_erased, BorrowKind::Shared, src),
566        ))));
567
568        // `let loc = Clone::clone(ref_loc);`
569        self.block(
570            vec![statement],
571            TerminatorKind::Call {
572                func,
573                args: [Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }].into(),
574                destination: dest,
575                target: Some(next),
576                unwind: UnwindAction::Cleanup(cleanup),
577                call_source: CallSource::Normal,
578                fn_span: self.span,
579            },
580            false,
581        );
582    }
583
584    fn clone_fields<I>(
585        &mut self,
586        dest: Place<'tcx>,
587        src: Place<'tcx>,
588        target: BasicBlock,
589        mut unwind: BasicBlock,
590        tys: I,
591    ) -> BasicBlock
592    where
593        I: IntoIterator<Item = Ty<'tcx>>,
594    {
595        // For an iterator of length n, create 2*n + 1 blocks.
596        for (i, ity) in tys.into_iter().enumerate() {
597            // Each iteration creates two blocks, referred to here as block 2*i and block 2*i + 1.
598            //
599            // Block 2*i attempts to clone the field. If successful it branches to 2*i + 2 (the
600            // next clone block). If unsuccessful it branches to the previous unwind block, which
601            // is initially the `unwind` argument passed to this function.
602            //
603            // Block 2*i + 1 is the unwind block for this iteration. It drops the cloned value
604            // created by block 2*i. We store this block in `unwind` so that the next clone block
605            // will unwind to it if cloning fails.
606
607            let field = FieldIdx::new(i);
608            let src_field = self.tcx.mk_place_field(src, field, ity);
609
610            let dest_field = self.tcx.mk_place_field(dest, field, ity);
611
612            let next_unwind = self.block_index_offset(1);
613            let next_block = self.block_index_offset(2);
614            self.make_clone_call(dest_field, src_field, ity, next_block, unwind);
615            self.block(
616                vec![],
617                TerminatorKind::Drop {
618                    place: dest_field,
619                    target: unwind,
620                    unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
621                    replace: false,
622                },
623                /* is_cleanup */ true,
624            );
625            unwind = next_unwind;
626        }
627        // If all clones succeed then we end up here.
628        self.block(vec![], TerminatorKind::Goto { target }, false);
629        unwind
630    }
631
632    fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
633    where
634        I: IntoIterator<Item = Ty<'tcx>>,
635    {
636        self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
637        let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
638        let target = self.block(vec![], TerminatorKind::Return, false);
639
640        let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, tys);
641    }
642
643    fn coroutine_shim(
644        &mut self,
645        dest: Place<'tcx>,
646        src: Place<'tcx>,
647        coroutine_def_id: DefId,
648        args: CoroutineArgs<TyCtxt<'tcx>>,
649    ) {
650        self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
651        let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
652        // This will get overwritten with a switch once we know the target blocks
653        let switch = self.block(vec![], TerminatorKind::Unreachable, false);
654        let unwind = self.clone_fields(dest, src, switch, unwind, args.upvar_tys());
655        let target = self.block(vec![], TerminatorKind::Return, false);
656        let unreachable = self.block(vec![], TerminatorKind::Unreachable, false);
657        let mut cases = Vec::with_capacity(args.state_tys(coroutine_def_id, self.tcx).count());
658        for (index, state_tys) in args.state_tys(coroutine_def_id, self.tcx).enumerate() {
659            let variant_index = VariantIdx::new(index);
660            let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index);
661            let src = self.tcx.mk_place_downcast_unnamed(src, variant_index);
662            let clone_block = self.block_index_offset(1);
663            let start_block = self.block(
664                vec![self.make_statement(StatementKind::SetDiscriminant {
665                    place: Box::new(Place::return_place()),
666                    variant_index,
667                })],
668                TerminatorKind::Goto { target: clone_block },
669                false,
670            );
671            cases.push((index as u128, start_block));
672            let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, state_tys);
673        }
674        let discr_ty = args.discr_ty(self.tcx);
675        let temp = self.make_place(Mutability::Mut, discr_ty);
676        let rvalue = Rvalue::Discriminant(src);
677        let statement = self.make_statement(StatementKind::Assign(Box::new((temp, rvalue))));
678        match &mut self.blocks[switch] {
679            BasicBlockData { statements, terminator: Some(Terminator { kind, .. }), .. } => {
680                statements.push(statement);
681                *kind = TerminatorKind::SwitchInt {
682                    discr: Operand::Move(temp),
683                    targets: SwitchTargets::new(cases.into_iter(), unreachable),
684                };
685            }
686            BasicBlockData { terminator: None, .. } => unreachable!(),
687        }
688    }
689}
690
691/// Builds a "call" shim for `instance`. The shim calls the function specified by `call_kind`,
692/// first adjusting its first argument according to `rcvr_adjustment`.
693#[instrument(level = "debug", skip(tcx), ret)]
694fn build_call_shim<'tcx>(
695    tcx: TyCtxt<'tcx>,
696    instance: ty::InstanceKind<'tcx>,
697    rcvr_adjustment: Option<Adjustment>,
698    call_kind: CallKind<'tcx>,
699) -> Body<'tcx> {
700    // `FnPtrShim` contains the fn pointer type that a call shim is being built for - this is used
701    // to instantiate into the signature of the shim. It is not necessary for users of this
702    // MIR body to perform further instantiations (see `InstanceKind::has_polymorphic_mir_body`).
703    let (sig_args, untuple_args) = if let ty::InstanceKind::FnPtrShim(_, ty) = instance {
704        let sig = tcx.instantiate_bound_regions_with_erased(ty.fn_sig(tcx));
705
706        let untuple_args = sig.inputs();
707
708        // Create substitutions for the `Self` and `Args` generic parameters of the shim body.
709        let arg_tup = Ty::new_tup(tcx, untuple_args);
710
711        (Some([ty.into(), arg_tup.into()]), Some(untuple_args))
712    } else {
713        (None, None)
714    };
715
716    let def_id = instance.def_id();
717
718    let rpitit_shim = if let ty::InstanceKind::ReifyShim(..) = instance {
719        tcx.return_position_impl_trait_in_trait_shim_data(def_id)
720    } else {
721        None
722    };
723
724    let sig = tcx.fn_sig(def_id);
725    let sig = sig.map_bound(|sig| tcx.instantiate_bound_regions_with_erased(sig));
726
727    assert_eq!(sig_args.is_some(), !instance.has_polymorphic_mir_body());
728    let mut sig = if let Some(sig_args) = sig_args {
729        sig.instantiate(tcx, &sig_args)
730    } else {
731        sig.instantiate_identity()
732    };
733
734    if let CallKind::Indirect(fnty) = call_kind {
735        // `sig` determines our local decls, and thus the callee type in the `Call` terminator. This
736        // can only be an `FnDef` or `FnPtr`, but currently will be `Self` since the types come from
737        // the implemented `FnX` trait.
738
739        // Apply the opposite adjustment to the MIR input.
740        let mut inputs_and_output = sig.inputs_and_output.to_vec();
741
742        // Initial signature is `fn(&? Self, Args) -> Self::Output` where `Args` is a tuple of the
743        // fn arguments. `Self` may be passed via (im)mutable reference or by-value.
744        assert_eq!(inputs_and_output.len(), 3);
745
746        // `Self` is always the original fn type `ty`. The MIR call terminator is only defined for
747        // `FnDef` and `FnPtr` callees, not the `Self` type param.
748        let self_arg = &mut inputs_and_output[0];
749        *self_arg = match rcvr_adjustment.unwrap() {
750            Adjustment::Identity => fnty,
751            Adjustment::Deref { source } => match source {
752                DerefSource::ImmRef => Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fnty),
753                DerefSource::MutRef => Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, fnty),
754                DerefSource::MutPtr => Ty::new_mut_ptr(tcx, fnty),
755            },
756            Adjustment::RefMut => bug!("`RefMut` is never used with indirect calls: {instance:?}"),
757        };
758        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
759    }
760
761    // FIXME: Avoid having to adjust the signature both here and in
762    // `fn_sig_for_fn_abi`.
763    if let ty::InstanceKind::VTableShim(..) = instance {
764        // Modify fn(self, ...) to fn(self: *mut Self, ...)
765        let mut inputs_and_output = sig.inputs_and_output.to_vec();
766        let self_arg = &mut inputs_and_output[0];
767        debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param);
768        *self_arg = Ty::new_mut_ptr(tcx, *self_arg);
769        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
770    }
771
772    let span = tcx.def_span(def_id);
773
774    debug!(?sig);
775
776    let mut local_decls = local_decls_for_sig(&sig, span);
777    let source_info = SourceInfo::outermost(span);
778
779    let mut destination = Place::return_place();
780    if let Some((rpitit_def_id, fn_args)) = rpitit_shim {
781        let rpitit_args =
782            fn_args.instantiate_identity().extend_to(tcx, rpitit_def_id, |param, _| {
783                match param.kind {
784                    ty::GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
785                    ty::GenericParamDefKind::Type { .. }
786                    | ty::GenericParamDefKind::Const { .. } => {
787                        unreachable!("rpitit should have no addition ty/ct")
788                    }
789                }
790            });
791        let dyn_star_ty = Ty::new_dynamic(
792            tcx,
793            tcx.item_bounds_to_existential_predicates(rpitit_def_id, rpitit_args),
794            tcx.lifetimes.re_erased,
795            ty::DynStar,
796        );
797        destination = local_decls.push(local_decls[RETURN_PLACE].clone()).into();
798        local_decls[RETURN_PLACE].ty = dyn_star_ty;
799        let mut inputs_and_output = sig.inputs_and_output.to_vec();
800        *inputs_and_output.last_mut().unwrap() = dyn_star_ty;
801        sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
802    }
803
804    let rcvr_place = || {
805        assert!(rcvr_adjustment.is_some());
806        Place::from(Local::new(1))
807    };
808    let mut statements = vec![];
809
810    let rcvr = rcvr_adjustment.map(|rcvr_adjustment| match rcvr_adjustment {
811        Adjustment::Identity => Operand::Move(rcvr_place()),
812        Adjustment::Deref { source: _ } => Operand::Move(tcx.mk_place_deref(rcvr_place())),
813        Adjustment::RefMut => {
814            // let rcvr = &mut rcvr;
815            let ref_rcvr = local_decls.push(
816                LocalDecl::new(
817                    Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, sig.inputs()[0]),
818                    span,
819                )
820                .immutable(),
821            );
822            let borrow_kind = BorrowKind::Mut { kind: MutBorrowKind::Default };
823            statements.push(Statement {
824                source_info,
825                kind: StatementKind::Assign(Box::new((
826                    Place::from(ref_rcvr),
827                    Rvalue::Ref(tcx.lifetimes.re_erased, borrow_kind, rcvr_place()),
828                ))),
829            });
830            Operand::Move(Place::from(ref_rcvr))
831        }
832    });
833
834    let (callee, mut args) = match call_kind {
835        // `FnPtr` call has no receiver. Args are untupled below.
836        CallKind::Indirect(_) => (rcvr.unwrap(), vec![]),
837
838        // `FnDef` call with optional receiver.
839        CallKind::Direct(def_id) => {
840            let ty = tcx.type_of(def_id).instantiate_identity();
841            (
842                Operand::Constant(Box::new(ConstOperand {
843                    span,
844                    user_ty: None,
845                    const_: Const::zero_sized(ty),
846                })),
847                rcvr.into_iter().collect::<Vec<_>>(),
848            )
849        }
850    };
851
852    let mut arg_range = 0..sig.inputs().len();
853
854    // Take the `self` ("receiver") argument out of the range (it's adjusted above).
855    if rcvr_adjustment.is_some() {
856        arg_range.start += 1;
857    }
858
859    // Take the last argument, if we need to untuple it (handled below).
860    if untuple_args.is_some() {
861        arg_range.end -= 1;
862    }
863
864    // Pass all of the non-special arguments directly.
865    args.extend(arg_range.map(|i| Operand::Move(Place::from(Local::new(1 + i)))));
866
867    // Untuple the last argument, if we have to.
868    if let Some(untuple_args) = untuple_args {
869        let tuple_arg = Local::new(1 + (sig.inputs().len() - 1));
870        args.extend(untuple_args.iter().enumerate().map(|(i, ity)| {
871            Operand::Move(tcx.mk_place_field(Place::from(tuple_arg), FieldIdx::new(i), *ity))
872        }));
873    }
874
875    let n_blocks = if let Some(Adjustment::RefMut) = rcvr_adjustment { 5 } else { 2 };
876    let mut blocks = IndexVec::with_capacity(n_blocks);
877    let block = |blocks: &mut IndexVec<_, _>, statements, kind, is_cleanup| {
878        blocks.push(BasicBlockData {
879            statements,
880            terminator: Some(Terminator { source_info, kind }),
881            is_cleanup,
882        })
883    };
884
885    // BB #0
886    let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect();
887    block(
888        &mut blocks,
889        statements,
890        TerminatorKind::Call {
891            func: callee,
892            args,
893            destination,
894            target: Some(BasicBlock::new(1)),
895            unwind: if let Some(Adjustment::RefMut) = rcvr_adjustment {
896                UnwindAction::Cleanup(BasicBlock::new(3))
897            } else {
898                UnwindAction::Continue
899            },
900            call_source: CallSource::Misc,
901            fn_span: span,
902        },
903        false,
904    );
905
906    if let Some(Adjustment::RefMut) = rcvr_adjustment {
907        // BB #1 - drop for Self
908        block(
909            &mut blocks,
910            vec![],
911            TerminatorKind::Drop {
912                place: rcvr_place(),
913                target: BasicBlock::new(2),
914                unwind: UnwindAction::Continue,
915                replace: false,
916            },
917            false,
918        );
919    }
920    // BB #1/#2 - return
921    // NOTE: If this is an RPITIT in dyn, we also want to coerce
922    // the return type of the function into a `dyn*`.
923    let stmts = if rpitit_shim.is_some() {
924        vec![Statement {
925            source_info,
926            kind: StatementKind::Assign(Box::new((
927                Place::return_place(),
928                Rvalue::Cast(
929                    CastKind::PointerCoercion(PointerCoercion::DynStar, CoercionSource::Implicit),
930                    Operand::Move(destination),
931                    sig.output(),
932                ),
933            ))),
934        }]
935    } else {
936        vec![]
937    };
938    block(&mut blocks, stmts, TerminatorKind::Return, false);
939    if let Some(Adjustment::RefMut) = rcvr_adjustment {
940        // BB #3 - drop if closure panics
941        block(
942            &mut blocks,
943            vec![],
944            TerminatorKind::Drop {
945                place: rcvr_place(),
946                target: BasicBlock::new(4),
947                unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
948                replace: false,
949            },
950            /* is_cleanup */ true,
951        );
952
953        // BB #4 - resume
954        block(&mut blocks, vec![], TerminatorKind::UnwindResume, true);
955    }
956
957    let mut body =
958        new_body(MirSource::from_instance(instance), blocks, local_decls, sig.inputs().len(), span);
959
960    if let ExternAbi::RustCall = sig.abi {
961        body.spread_arg = Some(Local::new(sig.inputs().len()));
962    }
963
964    body
965}
966
967pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
968    debug_assert!(tcx.is_constructor(ctor_id));
969
970    let typing_env = ty::TypingEnv::post_analysis(tcx, ctor_id);
971
972    // Normalize the sig.
973    let sig = tcx
974        .fn_sig(ctor_id)
975        .instantiate_identity()
976        .no_bound_vars()
977        .expect("LBR in ADT constructor signature");
978    let sig = tcx.normalize_erasing_regions(typing_env, sig);
979
980    let ty::Adt(adt_def, args) = sig.output().kind() else {
981        bug!("unexpected type for ADT ctor {:?}", sig.output());
982    };
983
984    debug!("build_ctor: ctor_id={:?} sig={:?}", ctor_id, sig);
985
986    let span = tcx.def_span(ctor_id);
987
988    let local_decls = local_decls_for_sig(&sig, span);
989
990    let source_info = SourceInfo::outermost(span);
991
992    let variant_index =
993        if adt_def.is_enum() { adt_def.variant_index_with_ctor_id(ctor_id) } else { FIRST_VARIANT };
994
995    // Generate the following MIR:
996    //
997    // (return as Variant).field0 = arg0;
998    // (return as Variant).field1 = arg1;
999    //
1000    // return;
1001    debug!("build_ctor: variant_index={:?}", variant_index);
1002
1003    let kind = AggregateKind::Adt(adt_def.did(), variant_index, args, None, None);
1004    let variant = adt_def.variant(variant_index);
1005    let statement = Statement {
1006        kind: StatementKind::Assign(Box::new((
1007            Place::return_place(),
1008            Rvalue::Aggregate(
1009                Box::new(kind),
1010                (0..variant.fields.len())
1011                    .map(|idx| Operand::Move(Place::from(Local::new(idx + 1))))
1012                    .collect(),
1013            ),
1014        ))),
1015        source_info,
1016    };
1017
1018    let start_block = BasicBlockData {
1019        statements: vec![statement],
1020        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1021        is_cleanup: false,
1022    };
1023
1024    let source = MirSource::item(ctor_id);
1025    let mut body = new_body(
1026        source,
1027        IndexVec::from_elem_n(start_block, 1),
1028        local_decls,
1029        sig.inputs().len(),
1030        span,
1031    );
1032    // A constructor doesn't mention any other items (and we don't run the usual optimization passes
1033    // so this would otherwise not get filled).
1034    body.set_mentioned_items(Vec::new());
1035
1036    crate::pass_manager::dump_mir_for_phase_change(tcx, &body);
1037
1038    body
1039}
1040
1041/// ```ignore (pseudo-impl)
1042/// impl FnPtr for fn(u32) {
1043///     fn addr(self) -> usize {
1044///         self as usize
1045///     }
1046/// }
1047/// ```
1048fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
1049    assert_matches!(self_ty.kind(), ty::FnPtr(..), "expected fn ptr, found {self_ty}");
1050    let span = tcx.def_span(def_id);
1051    let Some(sig) = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]).no_bound_vars() else {
1052        span_bug!(span, "FnPtr::addr with bound vars for `{self_ty}`");
1053    };
1054    let locals = local_decls_for_sig(&sig, span);
1055
1056    let source_info = SourceInfo::outermost(span);
1057    // FIXME: use `expose_provenance` once we figure out whether function pointers have meaningful
1058    // provenance.
1059    let rvalue = Rvalue::Cast(
1060        CastKind::FnPtrToPtr,
1061        Operand::Move(Place::from(Local::new(1))),
1062        Ty::new_imm_ptr(tcx, tcx.types.unit),
1063    );
1064    let stmt = Statement {
1065        source_info,
1066        kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1067    };
1068    let statements = vec![stmt];
1069    let start_block = BasicBlockData {
1070        statements,
1071        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1072        is_cleanup: false,
1073    };
1074    let source = MirSource::from_instance(ty::InstanceKind::FnPtrAddrShim(def_id, self_ty));
1075    new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span)
1076}
1077
1078fn build_construct_coroutine_by_move_shim<'tcx>(
1079    tcx: TyCtxt<'tcx>,
1080    coroutine_closure_def_id: DefId,
1081    receiver_by_ref: bool,
1082) -> Body<'tcx> {
1083    let mut self_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity();
1084    let mut self_local: Place<'tcx> = Local::from_usize(1).into();
1085    let ty::CoroutineClosure(_, args) = *self_ty.kind() else {
1086        bug!();
1087    };
1088
1089    // We use `&Self` here because we only need to emit an ABI-compatible shim body,
1090    // rather than match the signature exactly (which might take `&mut self` instead).
1091    //
1092    // We adjust the `self_local` to be a deref since we want to copy fields out of
1093    // a reference to the closure.
1094    if receiver_by_ref {
1095        self_local = tcx.mk_place_deref(self_local);
1096        self_ty = Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, self_ty);
1097    }
1098
1099    let poly_sig = args.as_coroutine_closure().coroutine_closure_sig().map_bound(|sig| {
1100        tcx.mk_fn_sig(
1101            [self_ty].into_iter().chain(sig.tupled_inputs_ty.tuple_fields()),
1102            sig.to_coroutine_given_kind_and_upvars(
1103                tcx,
1104                args.as_coroutine_closure().parent_args(),
1105                tcx.coroutine_for_closure(coroutine_closure_def_id),
1106                ty::ClosureKind::FnOnce,
1107                tcx.lifetimes.re_erased,
1108                args.as_coroutine_closure().tupled_upvars_ty(),
1109                args.as_coroutine_closure().coroutine_captures_by_ref_ty(),
1110            ),
1111            sig.c_variadic,
1112            sig.safety,
1113            sig.abi,
1114        )
1115    });
1116    let sig = tcx.liberate_late_bound_regions(coroutine_closure_def_id, poly_sig);
1117    let ty::Coroutine(coroutine_def_id, coroutine_args) = *sig.output().kind() else {
1118        bug!();
1119    };
1120
1121    let span = tcx.def_span(coroutine_closure_def_id);
1122    let locals = local_decls_for_sig(&sig, span);
1123
1124    let mut fields = vec![];
1125
1126    // Move all of the closure args.
1127    for idx in 1..sig.inputs().len() {
1128        fields.push(Operand::Move(Local::from_usize(idx + 1).into()));
1129    }
1130
1131    for (idx, ty) in args.as_coroutine_closure().upvar_tys().iter().enumerate() {
1132        if receiver_by_ref {
1133            // The only situation where it's possible is when we capture immuatable references,
1134            // since those don't need to be reborrowed with the closure's env lifetime. Since
1135            // references are always `Copy`, just emit a copy.
1136            if !matches!(ty.kind(), ty::Ref(_, _, hir::Mutability::Not)) {
1137                // This copy is only sound if it's a `&T`. This may be
1138                // reachable e.g. when eagerly computing the `Fn` instance
1139                // of an async closure that doesn't borrowck.
1140                tcx.dcx().delayed_bug(format!(
1141                    "field should be captured by immutable ref if we have \
1142                    an `Fn` instance, but it was: {ty}"
1143                ));
1144            }
1145            fields.push(Operand::Copy(tcx.mk_place_field(
1146                self_local,
1147                FieldIdx::from_usize(idx),
1148                ty,
1149            )));
1150        } else {
1151            fields.push(Operand::Move(tcx.mk_place_field(
1152                self_local,
1153                FieldIdx::from_usize(idx),
1154                ty,
1155            )));
1156        }
1157    }
1158
1159    let source_info = SourceInfo::outermost(span);
1160    let rvalue = Rvalue::Aggregate(
1161        Box::new(AggregateKind::Coroutine(coroutine_def_id, coroutine_args)),
1162        IndexVec::from_raw(fields),
1163    );
1164    let stmt = Statement {
1165        source_info,
1166        kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1167    };
1168    let statements = vec![stmt];
1169    let start_block = BasicBlockData {
1170        statements,
1171        terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
1172        is_cleanup: false,
1173    };
1174
1175    let source = MirSource::from_instance(ty::InstanceKind::ConstructCoroutineInClosureShim {
1176        coroutine_closure_def_id,
1177        receiver_by_ref,
1178    });
1179
1180    let body =
1181        new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span);
1182    dump_mir(
1183        tcx,
1184        false,
1185        if receiver_by_ref { "coroutine_closure_by_ref" } else { "coroutine_closure_by_move" },
1186        &0,
1187        &body,
1188        |_, _| Ok(()),
1189    );
1190
1191    body
1192}
rustc_mir_transform/shim.rs

rustc_mir_transform/
shim.rs
