1use std::assert_matches::assert_matches;
2use std::{fmt, iter};
3
4use rustc_abi::{ExternAbi, FIRST_VARIANT, FieldIdx, VariantIdx};
5use rustc_hir as hir;
6use rustc_hir::def_id::DefId;
7use rustc_hir::lang_items::LangItem;
8use rustc_index::{Idx, IndexVec};
9use rustc_middle::mir::visit::{MutVisitor, PlaceContext};
10use rustc_middle::mir::*;
11use rustc_middle::query::Providers;
12use rustc_middle::ty::{
13 self, CoroutineArgs, CoroutineArgsExt, EarlyBinder, GenericArgs, Ty, TyCtxt,
14};
15use rustc_middle::{bug, span_bug};
16use rustc_span::source_map::{Spanned, dummy_spanned};
17use rustc_span::{DUMMY_SP, Span};
18use tracing::{debug, instrument};
19
20use crate::deref_separator::deref_finder;
21use crate::elaborate_drop::{DropElaborator, DropFlagMode, DropStyle, Unwind, elaborate_drop};
22use crate::patch::MirPatch;
23use crate::{
24 abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, inline, instsimplify,
25 mentioned_items, pass_manager as pm, remove_noop_landing_pads, run_optimization_passes,
26 simplify,
27};
28
29mod async_destructor_ctor;
30
31pub(super) fn provide(providers: &mut Providers) {
32 providers.mir_shims = make_shim;
33}
34
35struct FixProxyFutureDropVisitor<'tcx> {
37 tcx: TyCtxt<'tcx>,
38 replace_to: Local,
39}
40
41impl<'tcx> MutVisitor<'tcx> for FixProxyFutureDropVisitor<'tcx> {
42 fn tcx(&self) -> TyCtxt<'tcx> {
43 self.tcx
44 }
45
46 fn visit_place(
47 &mut self,
48 place: &mut Place<'tcx>,
49 _context: PlaceContext,
50 _location: Location,
51 ) {
52 if place.local == Local::from_u32(1) {
53 if place.projection.len() == 1 {
54 assert!(matches!(
55 place.projection.first(),
56 Some(ProjectionElem::Field(FieldIdx::ZERO, _))
57 ));
58 *place = Place::from(self.replace_to);
59 } else if place.projection.len() == 2 {
60 assert!(matches!(place.projection[0], ProjectionElem::Field(FieldIdx::ZERO, _)));
61 assert!(matches!(place.projection[1], ProjectionElem::Deref));
62 *place =
63 Place::from(self.replace_to).project_deeper(&[ProjectionElem::Deref], self.tcx);
64 }
65 }
66 }
67}
68
69fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<'tcx> {
70 debug!("make_shim({:?})", instance);
71
72 let mut result = match instance {
73 ty::InstanceKind::Item(..) => bug!("item {:?} passed to make_shim", instance),
74 ty::InstanceKind::VTableShim(def_id) => {
75 let adjustment = Adjustment::Deref { source: DerefSource::MutPtr };
76 build_call_shim(tcx, instance, Some(adjustment), CallKind::Direct(def_id))
77 }
78 ty::InstanceKind::FnPtrShim(def_id, ty) => {
79 let trait_ = tcx.parent(def_id);
80 let adjustment = match tcx
82 .fn_trait_kind_from_def_id(trait_)
83 .or_else(|| tcx.async_fn_trait_kind_from_def_id(trait_))
84 {
85 Some(ty::ClosureKind::FnOnce) => Adjustment::Identity,
86 Some(ty::ClosureKind::Fn) => Adjustment::Deref { source: DerefSource::ImmRef },
87 Some(ty::ClosureKind::FnMut) => Adjustment::Deref { source: DerefSource::MutRef },
88 None => bug!("fn pointer {:?} is not an fn", ty),
89 };
90
91 build_call_shim(tcx, instance, Some(adjustment), CallKind::Indirect(ty))
92 }
93 ty::InstanceKind::ReifyShim(def_id, _) => {
99 build_call_shim(tcx, instance, None, CallKind::Direct(def_id))
100 }
101 ty::InstanceKind::ClosureOnceShim { call_once: _, track_caller: _ } => {
102 let fn_mut = tcx.require_lang_item(LangItem::FnMut, DUMMY_SP);
103 let call_mut = tcx
104 .associated_items(fn_mut)
105 .in_definition_order()
106 .find(|it| it.is_fn())
107 .unwrap()
108 .def_id;
109
110 build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut))
111 }
112
113 ty::InstanceKind::ConstructCoroutineInClosureShim {
114 coroutine_closure_def_id,
115 receiver_by_ref,
116 } => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref),
117
118 ty::InstanceKind::DropGlue(def_id, ty) => {
119 if let Some(&ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) {
122 let coroutine_body = tcx.optimized_mir(coroutine_def_id);
123
124 let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind()
125 else {
126 bug!()
127 };
128
129 let body = if id_args.as_coroutine().kind_ty() == args.as_coroutine().kind_ty() {
135 coroutine_body.coroutine_drop().unwrap()
136 } else {
137 assert_eq!(
138 args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(),
139 ty::ClosureKind::FnOnce
140 );
141 tcx.optimized_mir(tcx.coroutine_by_move_body_def_id(coroutine_def_id))
142 .coroutine_drop()
143 .unwrap()
144 };
145
146 let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args);
147 debug!("make_shim({:?}) = {:?}", instance, body);
148
149 pm::run_passes(
150 tcx,
151 &mut body,
152 &[
153 &mentioned_items::MentionedItems,
154 &abort_unwinding_calls::AbortUnwindingCalls,
155 &add_call_guards::CriticalCallEdges,
156 ],
157 Some(MirPhase::Runtime(RuntimePhase::Optimized)),
158 pm::Optimizations::Allowed,
159 );
160
161 return body;
162 }
163
164 build_drop_shim(tcx, def_id, ty)
165 }
166 ty::InstanceKind::ThreadLocalShim(..) => build_thread_local_shim(tcx, instance),
167 ty::InstanceKind::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty),
168 ty::InstanceKind::FnPtrAddrShim(def_id, ty) => build_fn_ptr_addr_shim(tcx, def_id, ty),
169 ty::InstanceKind::FutureDropPollShim(def_id, proxy_ty, impl_ty) => {
170 let mut body =
171 async_destructor_ctor::build_future_drop_poll_shim(tcx, def_id, proxy_ty, impl_ty);
172
173 pm::run_passes(
174 tcx,
175 &mut body,
176 &[
177 &mentioned_items::MentionedItems,
178 &abort_unwinding_calls::AbortUnwindingCalls,
179 &add_call_guards::CriticalCallEdges,
180 ],
181 Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
182 pm::Optimizations::Allowed,
183 );
184 run_optimization_passes(tcx, &mut body);
185 debug!("make_shim({:?}) = {:?}", instance, body);
186 return body;
187 }
188 ty::InstanceKind::AsyncDropGlue(def_id, ty) => {
189 let mut body = async_destructor_ctor::build_async_drop_shim(tcx, def_id, ty);
190
191 pm::run_passes(
195 tcx,
196 &mut body,
197 &[
198 &mentioned_items::MentionedItems,
199 &abort_unwinding_calls::AbortUnwindingCalls,
200 &add_call_guards::CriticalCallEdges,
201 &simplify::SimplifyCfg::MakeShim,
202 &crate::coroutine::StateTransform,
203 ],
204 Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
205 pm::Optimizations::Allowed,
206 );
207 run_optimization_passes(tcx, &mut body);
208 debug!("make_shim({:?}) = {:?}", instance, body);
209 return body;
210 }
211
212 ty::InstanceKind::AsyncDropGlueCtorShim(def_id, ty) => {
213 let body = async_destructor_ctor::build_async_destructor_ctor_shim(tcx, def_id, ty);
214 debug!("make_shim({:?}) = {:?}", instance, body);
215 return body;
216 }
217 ty::InstanceKind::Virtual(..) => {
218 bug!("InstanceKind::Virtual ({:?}) is for direct calls only", instance)
219 }
220 ty::InstanceKind::Intrinsic(_) => {
221 bug!("creating shims from intrinsics ({:?}) is unsupported", instance)
222 }
223 };
224 debug!("make_shim({:?}) = untransformed {:?}", instance, result);
225
226 deref_finder(tcx, &mut result, false);
227
228 pm::run_passes_no_validate(
233 tcx,
234 &mut result,
235 &[
236 &mentioned_items::MentionedItems,
237 &add_moves_for_packed_drops::AddMovesForPackedDrops,
238 &remove_noop_landing_pads::RemoveNoopLandingPads,
239 &simplify::SimplifyCfg::MakeShim,
240 &instsimplify::InstSimplify::BeforeInline,
241 &inline::ForceInline,
243 &abort_unwinding_calls::AbortUnwindingCalls,
244 &add_call_guards::CriticalCallEdges,
245 ],
246 Some(MirPhase::Runtime(RuntimePhase::Optimized)),
247 );
248
249 debug!("make_shim({:?}) = {:?}", instance, result);
250
251 result
252}
253
254#[derive(Copy, Clone, Debug, PartialEq)]
255enum DerefSource {
256 ImmRef,
258 MutRef,
260 MutPtr,
262}
263
264#[derive(Copy, Clone, Debug, PartialEq)]
265enum Adjustment {
266 Identity,
268
269 Deref { source: DerefSource },
274
275 RefMut,
280}
281
282#[derive(Copy, Clone, Debug, PartialEq)]
283enum CallKind<'tcx> {
284 Indirect(Ty<'tcx>),
286
287 Direct(DefId),
289}
290
291fn local_decls_for_sig<'tcx>(
292 sig: &ty::FnSig<'tcx>,
293 span: Span,
294) -> IndexVec<Local, LocalDecl<'tcx>> {
295 iter::once(LocalDecl::new(sig.output(), span))
296 .chain(sig.inputs().iter().map(|ity| LocalDecl::new(*ity, span).immutable()))
297 .collect()
298}
299
300fn dropee_emit_retag<'tcx>(
301 tcx: TyCtxt<'tcx>,
302 body: &mut Body<'tcx>,
303 mut dropee_ptr: Place<'tcx>,
304 span: Span,
305) -> Place<'tcx> {
306 if tcx.sess.opts.unstable_opts.mir_emit_retag {
307 let source_info = SourceInfo::outermost(span);
308 let reborrow = Rvalue::Ref(
317 tcx.lifetimes.re_erased,
318 BorrowKind::Mut { kind: MutBorrowKind::Default },
319 tcx.mk_place_deref(dropee_ptr),
320 );
321 let ref_ty = reborrow.ty(body.local_decls(), tcx);
322 dropee_ptr = body.local_decls.push(LocalDecl::new(ref_ty, span)).into();
323 let new_statements = [
324 StatementKind::Assign(Box::new((dropee_ptr, reborrow))),
325 StatementKind::Retag(RetagKind::FnEntry, Box::new(dropee_ptr)),
326 ];
327 for s in new_statements {
328 body.basic_blocks_mut()[START_BLOCK].statements.push(Statement::new(source_info, s));
329 }
330 }
331 dropee_ptr
332}
333
334fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> {
335 debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty);
336
337 assert!(!matches!(ty, Some(ty) if ty.is_coroutine()));
338
339 let args = if let Some(ty) = ty {
340 tcx.mk_args(&[ty.into()])
341 } else {
342 GenericArgs::identity_for_item(tcx, def_id)
343 };
344 let sig = tcx.fn_sig(def_id).instantiate(tcx, args);
345 let sig = tcx.instantiate_bound_regions_with_erased(sig);
346 let span = tcx.def_span(def_id);
347
348 let source_info = SourceInfo::outermost(span);
349
350 let return_block = BasicBlock::new(1);
351 let mut blocks = IndexVec::with_capacity(2);
352 let block = |blocks: &mut IndexVec<_, _>, kind| {
353 blocks.push(BasicBlockData::new(Some(Terminator { source_info, kind }), false))
354 };
355 block(&mut blocks, TerminatorKind::Goto { target: return_block });
356 block(&mut blocks, TerminatorKind::Return);
357
358 let source = MirSource::from_instance(ty::InstanceKind::DropGlue(def_id, ty));
359 let mut body =
360 new_body(source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span);
361
362 let dropee_ptr = Place::from(Local::new(1 + 0));
364 let dropee_ptr = dropee_emit_retag(tcx, &mut body, dropee_ptr, span);
365
366 if ty.is_some() {
367 let patch = {
368 let typing_env = ty::TypingEnv::post_analysis(tcx, def_id);
369 let mut elaborator = DropShimElaborator {
370 body: &body,
371 patch: MirPatch::new(&body),
372 tcx,
373 typing_env,
374 produce_async_drops: false,
375 };
376 let dropee = tcx.mk_place_deref(dropee_ptr);
377 let resume_block = elaborator.patch.resume_block();
378 elaborate_drop(
379 &mut elaborator,
380 source_info,
381 dropee,
382 (),
383 return_block,
384 Unwind::To(resume_block),
385 START_BLOCK,
386 None,
387 );
388 elaborator.patch
389 };
390 patch.apply(&mut body);
391 }
392
393 body
394}
395
396fn new_body<'tcx>(
397 source: MirSource<'tcx>,
398 basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
399 local_decls: IndexVec<Local, LocalDecl<'tcx>>,
400 arg_count: usize,
401 span: Span,
402) -> Body<'tcx> {
403 let mut body = Body::new(
404 source,
405 basic_blocks,
406 IndexVec::from_elem_n(
407 SourceScopeData {
408 span,
409 parent_scope: None,
410 inlined: None,
411 inlined_parent_scope: None,
412 local_data: ClearCrossCrate::Clear,
413 },
414 1,
415 ),
416 local_decls,
417 IndexVec::new(),
418 arg_count,
419 vec![],
420 span,
421 None,
422 None,
424 );
425 body.set_required_consts(Vec::new());
427 body
428}
429
430pub(super) struct DropShimElaborator<'a, 'tcx> {
431 pub body: &'a Body<'tcx>,
432 pub patch: MirPatch<'tcx>,
433 pub tcx: TyCtxt<'tcx>,
434 pub typing_env: ty::TypingEnv<'tcx>,
435 pub produce_async_drops: bool,
436}
437
438impl fmt::Debug for DropShimElaborator<'_, '_> {
439 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
440 f.debug_struct("DropShimElaborator").finish_non_exhaustive()
441 }
442}
443
444impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> {
445 type Path = ();
446
447 fn patch_ref(&self) -> &MirPatch<'tcx> {
448 &self.patch
449 }
450 fn patch(&mut self) -> &mut MirPatch<'tcx> {
451 &mut self.patch
452 }
453 fn body(&self) -> &'a Body<'tcx> {
454 self.body
455 }
456 fn tcx(&self) -> TyCtxt<'tcx> {
457 self.tcx
458 }
459 fn typing_env(&self) -> ty::TypingEnv<'tcx> {
460 self.typing_env
461 }
462
463 fn terminator_loc(&self, bb: BasicBlock) -> Location {
464 self.patch.terminator_loc(self.body, bb)
465 }
466 fn allow_async_drops(&self) -> bool {
467 self.produce_async_drops
468 }
469
470 fn drop_style(&self, _path: Self::Path, mode: DropFlagMode) -> DropStyle {
471 match mode {
472 DropFlagMode::Shallow => {
473 DropStyle::Static
476 }
477 DropFlagMode::Deep => {
478 DropStyle::Open
481 }
482 }
483 }
484
485 fn get_drop_flag(&mut self, _path: Self::Path) -> Option<Operand<'tcx>> {
486 None
487 }
488
489 fn clear_drop_flag(&mut self, _location: Location, _path: Self::Path, _mode: DropFlagMode) {}
490
491 fn field_subpath(&self, _path: Self::Path, _field: FieldIdx) -> Option<Self::Path> {
492 None
493 }
494 fn deref_subpath(&self, _path: Self::Path) -> Option<Self::Path> {
495 None
496 }
497 fn downcast_subpath(&self, _path: Self::Path, _variant: VariantIdx) -> Option<Self::Path> {
498 Some(())
499 }
500 fn array_subpath(&self, _path: Self::Path, _index: u64, _size: u64) -> Option<Self::Path> {
501 None
502 }
503}
504
505fn build_thread_local_shim<'tcx>(
506 tcx: TyCtxt<'tcx>,
507 instance: ty::InstanceKind<'tcx>,
508) -> Body<'tcx> {
509 let def_id = instance.def_id();
510
511 let span = tcx.def_span(def_id);
512 let source_info = SourceInfo::outermost(span);
513
514 let blocks = IndexVec::from_raw(vec![BasicBlockData::new_stmts(
515 vec![Statement::new(
516 source_info,
517 StatementKind::Assign(Box::new((
518 Place::return_place(),
519 Rvalue::ThreadLocalRef(def_id),
520 ))),
521 )],
522 Some(Terminator { source_info, kind: TerminatorKind::Return }),
523 false,
524 )]);
525
526 new_body(
527 MirSource::from_instance(instance),
528 blocks,
529 IndexVec::from_raw(vec![LocalDecl::new(tcx.thread_local_ptr_ty(def_id), span)]),
530 0,
531 span,
532 )
533}
534
535fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
537 debug!("build_clone_shim(def_id={:?})", def_id);
538
539 let mut builder = CloneShimBuilder::new(tcx, def_id, self_ty);
540
541 let dest = Place::return_place();
542 let src = tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
543
544 match self_ty.kind() {
545 ty::FnDef(..) | ty::FnPtr(..) => builder.copy_shim(),
546 ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()),
547 ty::CoroutineClosure(_, args) => {
548 builder.tuple_like_shim(dest, src, args.as_coroutine_closure().upvar_tys())
549 }
550 ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()),
551 ty::Coroutine(coroutine_def_id, args) => {
552 assert_eq!(tcx.coroutine_movability(*coroutine_def_id), hir::Movability::Movable);
553 builder.coroutine_shim(dest, src, *coroutine_def_id, args.as_coroutine())
554 }
555 _ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty),
556 };
557
558 builder.into_mir()
559}
560
561struct CloneShimBuilder<'tcx> {
562 tcx: TyCtxt<'tcx>,
563 def_id: DefId,
564 local_decls: IndexVec<Local, LocalDecl<'tcx>>,
565 blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
566 span: Span,
567 sig: ty::FnSig<'tcx>,
568}
569
570impl<'tcx> CloneShimBuilder<'tcx> {
571 fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Self {
572 let sig = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]);
576 let sig = tcx.instantiate_bound_regions_with_erased(sig);
577 let span = tcx.def_span(def_id);
578
579 CloneShimBuilder {
580 tcx,
581 def_id,
582 local_decls: local_decls_for_sig(&sig, span),
583 blocks: IndexVec::new(),
584 span,
585 sig,
586 }
587 }
588
589 fn into_mir(self) -> Body<'tcx> {
590 let source = MirSource::from_instance(ty::InstanceKind::CloneShim(
591 self.def_id,
592 self.sig.inputs_and_output[0],
593 ));
594 new_body(source, self.blocks, self.local_decls, self.sig.inputs().len(), self.span)
595 }
596
597 fn source_info(&self) -> SourceInfo {
598 SourceInfo::outermost(self.span)
599 }
600
601 fn block(
602 &mut self,
603 statements: Vec<Statement<'tcx>>,
604 kind: TerminatorKind<'tcx>,
605 is_cleanup: bool,
606 ) -> BasicBlock {
607 let source_info = self.source_info();
608 self.blocks.push(BasicBlockData::new_stmts(
609 statements,
610 Some(Terminator { source_info, kind }),
611 is_cleanup,
612 ))
613 }
614
615 fn block_index_offset(&self, offset: usize) -> BasicBlock {
620 BasicBlock::new(self.blocks.len() + offset)
621 }
622
623 fn make_statement(&self, kind: StatementKind<'tcx>) -> Statement<'tcx> {
624 Statement::new(self.source_info(), kind)
625 }
626
627 fn copy_shim(&mut self) {
628 let rcvr = self.tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
629 let ret_statement = self.make_statement(StatementKind::Assign(Box::new((
630 Place::return_place(),
631 Rvalue::Use(Operand::Copy(rcvr)),
632 ))));
633 self.block(vec![ret_statement], TerminatorKind::Return, false);
634 }
635
636 fn make_place(&mut self, mutability: Mutability, ty: Ty<'tcx>) -> Place<'tcx> {
637 let span = self.span;
638 let mut local = LocalDecl::new(ty, span);
639 if mutability.is_not() {
640 local = local.immutable();
641 }
642 Place::from(self.local_decls.push(local))
643 }
644
645 fn make_clone_call(
646 &mut self,
647 dest: Place<'tcx>,
648 src: Place<'tcx>,
649 ty: Ty<'tcx>,
650 next: BasicBlock,
651 cleanup: BasicBlock,
652 ) {
653 let tcx = self.tcx;
654
655 let func_ty = Ty::new_fn_def(tcx, self.def_id, [ty]);
657 let func = Operand::Constant(Box::new(ConstOperand {
658 span: self.span,
659 user_ty: None,
660 const_: Const::zero_sized(func_ty),
661 }));
662
663 let ref_loc =
664 self.make_place(Mutability::Not, Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, ty));
665
666 let statement = self.make_statement(StatementKind::Assign(Box::new((
668 ref_loc,
669 Rvalue::Ref(tcx.lifetimes.re_erased, BorrowKind::Shared, src),
670 ))));
671
672 self.block(
674 vec![statement],
675 TerminatorKind::Call {
676 func,
677 args: [Spanned { node: Operand::Move(ref_loc), span: DUMMY_SP }].into(),
678 destination: dest,
679 target: Some(next),
680 unwind: UnwindAction::Cleanup(cleanup),
681 call_source: CallSource::Normal,
682 fn_span: self.span,
683 },
684 false,
685 );
686 }
687
688 fn clone_fields<I>(
689 &mut self,
690 dest: Place<'tcx>,
691 src: Place<'tcx>,
692 target: BasicBlock,
693 mut unwind: BasicBlock,
694 tys: I,
695 ) -> BasicBlock
696 where
697 I: IntoIterator<Item = Ty<'tcx>>,
698 {
699 for (i, ity) in tys.into_iter().enumerate() {
701 let field = FieldIdx::new(i);
712 let src_field = self.tcx.mk_place_field(src, field, ity);
713
714 let dest_field = self.tcx.mk_place_field(dest, field, ity);
715
716 let next_unwind = self.block_index_offset(1);
717 let next_block = self.block_index_offset(2);
718 self.make_clone_call(dest_field, src_field, ity, next_block, unwind);
719 self.block(
720 vec![],
721 TerminatorKind::Drop {
722 place: dest_field,
723 target: unwind,
724 unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
725 replace: false,
726 drop: None,
727 async_fut: None,
728 },
729 true,
730 );
731 unwind = next_unwind;
732 }
733 self.block(vec![], TerminatorKind::Goto { target }, false);
735 unwind
736 }
737
738 fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
739 where
740 I: IntoIterator<Item = Ty<'tcx>>,
741 {
742 self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
743 let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
744 let target = self.block(vec![], TerminatorKind::Return, false);
745
746 let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, tys);
747 }
748
749 fn coroutine_shim(
750 &mut self,
751 dest: Place<'tcx>,
752 src: Place<'tcx>,
753 coroutine_def_id: DefId,
754 args: CoroutineArgs<TyCtxt<'tcx>>,
755 ) {
756 self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
757 let unwind = self.block(vec![], TerminatorKind::UnwindResume, true);
758 let switch = self.block(vec![], TerminatorKind::Unreachable, false);
760 let unwind = self.clone_fields(dest, src, switch, unwind, args.upvar_tys());
761 let target = self.block(vec![], TerminatorKind::Return, false);
762 let unreachable = self.block(vec![], TerminatorKind::Unreachable, false);
763 let mut cases = Vec::with_capacity(args.state_tys(coroutine_def_id, self.tcx).count());
764 for (index, state_tys) in args.state_tys(coroutine_def_id, self.tcx).enumerate() {
765 let variant_index = VariantIdx::new(index);
766 let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index);
767 let src = self.tcx.mk_place_downcast_unnamed(src, variant_index);
768 let clone_block = self.block_index_offset(1);
769 let start_block = self.block(
770 vec![self.make_statement(StatementKind::SetDiscriminant {
771 place: Box::new(Place::return_place()),
772 variant_index,
773 })],
774 TerminatorKind::Goto { target: clone_block },
775 false,
776 );
777 cases.push((index as u128, start_block));
778 let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, state_tys);
779 }
780 let discr_ty = args.discr_ty(self.tcx);
781 let temp = self.make_place(Mutability::Mut, discr_ty);
782 let rvalue = Rvalue::Discriminant(src);
783 let statement = self.make_statement(StatementKind::Assign(Box::new((temp, rvalue))));
784 match &mut self.blocks[switch] {
785 BasicBlockData { statements, terminator: Some(Terminator { kind, .. }), .. } => {
786 statements.push(statement);
787 *kind = TerminatorKind::SwitchInt {
788 discr: Operand::Move(temp),
789 targets: SwitchTargets::new(cases.into_iter(), unreachable),
790 };
791 }
792 BasicBlockData { terminator: None, .. } => unreachable!(),
793 }
794 }
795}
796
797#[instrument(level = "debug", skip(tcx), ret)]
800fn build_call_shim<'tcx>(
801 tcx: TyCtxt<'tcx>,
802 instance: ty::InstanceKind<'tcx>,
803 rcvr_adjustment: Option<Adjustment>,
804 call_kind: CallKind<'tcx>,
805) -> Body<'tcx> {
806 let (sig_args, untuple_args) = if let ty::InstanceKind::FnPtrShim(_, ty) = instance {
810 let sig = tcx.instantiate_bound_regions_with_erased(ty.fn_sig(tcx));
811
812 let untuple_args = sig.inputs();
813
814 let arg_tup = Ty::new_tup(tcx, untuple_args);
816
817 (Some([ty.into(), arg_tup.into()]), Some(untuple_args))
818 } else {
819 (None, None)
820 };
821
822 let def_id = instance.def_id();
823
824 let sig = tcx.fn_sig(def_id);
825 let sig = sig.map_bound(|sig| tcx.instantiate_bound_regions_with_erased(sig));
826
827 assert_eq!(sig_args.is_some(), !instance.has_polymorphic_mir_body());
828 let mut sig = if let Some(sig_args) = sig_args {
829 sig.instantiate(tcx, &sig_args)
830 } else {
831 sig.instantiate_identity()
832 };
833
834 if let CallKind::Indirect(fnty) = call_kind {
835 let mut inputs_and_output = sig.inputs_and_output.to_vec();
841
842 assert_eq!(inputs_and_output.len(), 3);
845
846 let self_arg = &mut inputs_and_output[0];
849 *self_arg = match rcvr_adjustment.unwrap() {
850 Adjustment::Identity => fnty,
851 Adjustment::Deref { source } => match source {
852 DerefSource::ImmRef => Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fnty),
853 DerefSource::MutRef => Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, fnty),
854 DerefSource::MutPtr => Ty::new_mut_ptr(tcx, fnty),
855 },
856 Adjustment::RefMut => bug!("`RefMut` is never used with indirect calls: {instance:?}"),
857 };
858 sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
859 }
860
861 if let ty::InstanceKind::VTableShim(..) = instance {
864 let mut inputs_and_output = sig.inputs_and_output.to_vec();
866 let self_arg = &mut inputs_and_output[0];
867 debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param);
868 *self_arg = Ty::new_mut_ptr(tcx, *self_arg);
869 sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output);
870 }
871
872 let span = tcx.def_span(def_id);
873
874 debug!(?sig);
875
876 let mut local_decls = local_decls_for_sig(&sig, span);
877 let source_info = SourceInfo::outermost(span);
878
879 let destination = Place::return_place();
880
881 let rcvr_place = || {
882 assert!(rcvr_adjustment.is_some());
883 Place::from(Local::new(1))
884 };
885 let mut statements = vec![];
886
887 let rcvr = rcvr_adjustment.map(|rcvr_adjustment| match rcvr_adjustment {
888 Adjustment::Identity => Operand::Move(rcvr_place()),
889 Adjustment::Deref { source: _ } => Operand::Move(tcx.mk_place_deref(rcvr_place())),
890 Adjustment::RefMut => {
891 let ref_rcvr = local_decls.push(
893 LocalDecl::new(
894 Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, sig.inputs()[0]),
895 span,
896 )
897 .immutable(),
898 );
899 let borrow_kind = BorrowKind::Mut { kind: MutBorrowKind::Default };
900 statements.push(Statement::new(
901 source_info,
902 StatementKind::Assign(Box::new((
903 Place::from(ref_rcvr),
904 Rvalue::Ref(tcx.lifetimes.re_erased, borrow_kind, rcvr_place()),
905 ))),
906 ));
907 Operand::Move(Place::from(ref_rcvr))
908 }
909 });
910
911 let (callee, mut args) = match call_kind {
912 CallKind::Indirect(_) => (rcvr.unwrap(), vec![]),
914
915 CallKind::Direct(def_id) => {
917 let ty = tcx.type_of(def_id).instantiate_identity();
918 (
919 Operand::Constant(Box::new(ConstOperand {
920 span,
921 user_ty: None,
922 const_: Const::zero_sized(ty),
923 })),
924 rcvr.into_iter().collect::<Vec<_>>(),
925 )
926 }
927 };
928
929 let mut arg_range = 0..sig.inputs().len();
930
931 if rcvr_adjustment.is_some() {
933 arg_range.start += 1;
934 }
935
936 if untuple_args.is_some() {
938 arg_range.end -= 1;
939 }
940
941 args.extend(arg_range.map(|i| Operand::Move(Place::from(Local::new(1 + i)))));
943
944 if let Some(untuple_args) = untuple_args {
946 let tuple_arg = Local::new(1 + (sig.inputs().len() - 1));
947 args.extend(untuple_args.iter().enumerate().map(|(i, ity)| {
948 Operand::Move(tcx.mk_place_field(Place::from(tuple_arg), FieldIdx::new(i), *ity))
949 }));
950 }
951
952 let n_blocks = if let Some(Adjustment::RefMut) = rcvr_adjustment { 5 } else { 2 };
953 let mut blocks = IndexVec::with_capacity(n_blocks);
954 let block = |blocks: &mut IndexVec<_, _>, statements, kind, is_cleanup| {
955 blocks.push(BasicBlockData::new_stmts(
956 statements,
957 Some(Terminator { source_info, kind }),
958 is_cleanup,
959 ))
960 };
961
962 let args = args.into_iter().map(|a| Spanned { node: a, span: DUMMY_SP }).collect();
964 block(
965 &mut blocks,
966 statements,
967 TerminatorKind::Call {
968 func: callee,
969 args,
970 destination,
971 target: Some(BasicBlock::new(1)),
972 unwind: if let Some(Adjustment::RefMut) = rcvr_adjustment {
973 UnwindAction::Cleanup(BasicBlock::new(3))
974 } else {
975 UnwindAction::Continue
976 },
977 call_source: CallSource::Misc,
978 fn_span: span,
979 },
980 false,
981 );
982
983 if let Some(Adjustment::RefMut) = rcvr_adjustment {
984 block(
986 &mut blocks,
987 vec![],
988 TerminatorKind::Drop {
989 place: rcvr_place(),
990 target: BasicBlock::new(2),
991 unwind: UnwindAction::Continue,
992 replace: false,
993 drop: None,
994 async_fut: None,
995 },
996 false,
997 );
998 }
999 let stmts = vec![];
1001 block(&mut blocks, stmts, TerminatorKind::Return, false);
1002 if let Some(Adjustment::RefMut) = rcvr_adjustment {
1003 block(
1005 &mut blocks,
1006 vec![],
1007 TerminatorKind::Drop {
1008 place: rcvr_place(),
1009 target: BasicBlock::new(4),
1010 unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
1011 replace: false,
1012 drop: None,
1013 async_fut: None,
1014 },
1015 true,
1016 );
1017
1018 block(&mut blocks, vec![], TerminatorKind::UnwindResume, true);
1020 }
1021
1022 let mut body =
1023 new_body(MirSource::from_instance(instance), blocks, local_decls, sig.inputs().len(), span);
1024
1025 if let ExternAbi::RustCall = sig.abi {
1026 body.spread_arg = Some(Local::new(sig.inputs().len()));
1027 }
1028
1029 body
1030}
1031
1032pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
1033 debug_assert!(tcx.is_constructor(ctor_id));
1034
1035 let typing_env = ty::TypingEnv::post_analysis(tcx, ctor_id);
1036
1037 let sig = tcx
1039 .fn_sig(ctor_id)
1040 .instantiate_identity()
1041 .no_bound_vars()
1042 .expect("LBR in ADT constructor signature");
1043 let sig = tcx.normalize_erasing_regions(typing_env, sig);
1044
1045 let ty::Adt(adt_def, args) = sig.output().kind() else {
1046 bug!("unexpected type for ADT ctor {:?}", sig.output());
1047 };
1048
1049 debug!("build_ctor: ctor_id={:?} sig={:?}", ctor_id, sig);
1050
1051 let span = tcx.def_span(ctor_id);
1052
1053 let local_decls = local_decls_for_sig(&sig, span);
1054
1055 let source_info = SourceInfo::outermost(span);
1056
1057 let variant_index =
1058 if adt_def.is_enum() { adt_def.variant_index_with_ctor_id(ctor_id) } else { FIRST_VARIANT };
1059
1060 debug!("build_ctor: variant_index={:?}", variant_index);
1067
1068 let kind = AggregateKind::Adt(adt_def.did(), variant_index, args, None, None);
1069 let variant = adt_def.variant(variant_index);
1070 let statement = Statement::new(
1071 source_info,
1072 StatementKind::Assign(Box::new((
1073 Place::return_place(),
1074 Rvalue::Aggregate(
1075 Box::new(kind),
1076 (0..variant.fields.len())
1077 .map(|idx| Operand::Move(Place::from(Local::new(idx + 1))))
1078 .collect(),
1079 ),
1080 ))),
1081 );
1082
1083 let start_block = BasicBlockData::new_stmts(
1084 vec![statement],
1085 Some(Terminator { source_info, kind: TerminatorKind::Return }),
1086 false,
1087 );
1088
1089 let source = MirSource::item(ctor_id);
1090 let mut body = new_body(
1091 source,
1092 IndexVec::from_elem_n(start_block, 1),
1093 local_decls,
1094 sig.inputs().len(),
1095 span,
1096 );
1097 body.set_mentioned_items(Vec::new());
1100
1101 crate::pass_manager::dump_mir_for_phase_change(tcx, &body);
1102
1103 body
1104}
1105
1106fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
1114 assert_matches!(self_ty.kind(), ty::FnPtr(..), "expected fn ptr, found {self_ty}");
1115 let span = tcx.def_span(def_id);
1116 let Some(sig) = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]).no_bound_vars() else {
1117 span_bug!(span, "FnPtr::addr with bound vars for `{self_ty}`");
1118 };
1119 let locals = local_decls_for_sig(&sig, span);
1120
1121 let source_info = SourceInfo::outermost(span);
1122 let rvalue = Rvalue::Cast(
1125 CastKind::FnPtrToPtr,
1126 Operand::Move(Place::from(Local::new(1))),
1127 Ty::new_imm_ptr(tcx, tcx.types.unit),
1128 );
1129 let stmt = Statement::new(
1130 source_info,
1131 StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1132 );
1133 let statements = vec![stmt];
1134 let start_block = BasicBlockData::new_stmts(
1135 statements,
1136 Some(Terminator { source_info, kind: TerminatorKind::Return }),
1137 false,
1138 );
1139 let source = MirSource::from_instance(ty::InstanceKind::FnPtrAddrShim(def_id, self_ty));
1140 new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span)
1141}
1142
1143fn build_construct_coroutine_by_move_shim<'tcx>(
1144 tcx: TyCtxt<'tcx>,
1145 coroutine_closure_def_id: DefId,
1146 receiver_by_ref: bool,
1147) -> Body<'tcx> {
1148 let mut self_ty = tcx.type_of(coroutine_closure_def_id).instantiate_identity();
1149 let mut self_local: Place<'tcx> = Local::from_usize(1).into();
1150 let ty::CoroutineClosure(_, args) = *self_ty.kind() else {
1151 bug!();
1152 };
1153
1154 if receiver_by_ref {
1160 self_local = tcx.mk_place_deref(self_local);
1161 self_ty = Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, self_ty);
1162 }
1163
1164 let poly_sig = args.as_coroutine_closure().coroutine_closure_sig().map_bound(|sig| {
1165 tcx.mk_fn_sig(
1166 [self_ty].into_iter().chain(sig.tupled_inputs_ty.tuple_fields()),
1167 sig.to_coroutine_given_kind_and_upvars(
1168 tcx,
1169 args.as_coroutine_closure().parent_args(),
1170 tcx.coroutine_for_closure(coroutine_closure_def_id),
1171 ty::ClosureKind::FnOnce,
1172 tcx.lifetimes.re_erased,
1173 args.as_coroutine_closure().tupled_upvars_ty(),
1174 args.as_coroutine_closure().coroutine_captures_by_ref_ty(),
1175 ),
1176 sig.c_variadic,
1177 sig.safety,
1178 sig.abi,
1179 )
1180 });
1181 let sig = tcx.liberate_late_bound_regions(coroutine_closure_def_id, poly_sig);
1182 let ty::Coroutine(coroutine_def_id, coroutine_args) = *sig.output().kind() else {
1183 bug!();
1184 };
1185
1186 let span = tcx.def_span(coroutine_closure_def_id);
1187 let locals = local_decls_for_sig(&sig, span);
1188
1189 let mut fields = vec![];
1190
1191 for idx in 1..sig.inputs().len() {
1193 fields.push(Operand::Move(Local::from_usize(idx + 1).into()));
1194 }
1195
1196 for (idx, ty) in args.as_coroutine_closure().upvar_tys().iter().enumerate() {
1197 if receiver_by_ref {
1198 if !matches!(ty.kind(), ty::Ref(_, _, hir::Mutability::Not)) {
1202 tcx.dcx().delayed_bug(format!(
1206 "field should be captured by immutable ref if we have \
1207 an `Fn` instance, but it was: {ty}"
1208 ));
1209 }
1210 fields.push(Operand::Copy(tcx.mk_place_field(
1211 self_local,
1212 FieldIdx::from_usize(idx),
1213 ty,
1214 )));
1215 } else {
1216 fields.push(Operand::Move(tcx.mk_place_field(
1217 self_local,
1218 FieldIdx::from_usize(idx),
1219 ty,
1220 )));
1221 }
1222 }
1223
1224 let source_info = SourceInfo::outermost(span);
1225 let rvalue = Rvalue::Aggregate(
1226 Box::new(AggregateKind::Coroutine(coroutine_def_id, coroutine_args)),
1227 IndexVec::from_raw(fields),
1228 );
1229 let stmt = Statement::new(
1230 source_info,
1231 StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
1232 );
1233 let statements = vec![stmt];
1234 let start_block = BasicBlockData::new_stmts(
1235 statements,
1236 Some(Terminator { source_info, kind: TerminatorKind::Return }),
1237 false,
1238 );
1239
1240 let source = MirSource::from_instance(ty::InstanceKind::ConstructCoroutineInClosureShim {
1241 coroutine_closure_def_id,
1242 receiver_by_ref,
1243 });
1244
1245 let body =
1246 new_body(source, IndexVec::from_elem_n(start_block, 1), locals, sig.inputs().len(), span);
1247
1248 let pass_name =
1249 if receiver_by_ref { "coroutine_closure_by_ref" } else { "coroutine_closure_by_move" };
1250 if let Some(dumper) = MirDumper::new(tcx, pass_name, &body) {
1251 dumper.dump_mir(&body);
1252 }
1253
1254 body
1255}