1#![feature(array_windows)]
3#![feature(assert_matches)]
4#![feature(box_patterns)]
5#![feature(const_type_name)]
6#![feature(cow_is_borrowed)]
7#![feature(file_buffered)]
8#![feature(gen_blocks)]
9#![feature(if_let_guard)]
10#![feature(impl_trait_in_assoc_type)]
11#![feature(try_blocks)]
12#![feature(yeet_expr)]
13use hir::ConstContext;
16use required_consts::RequiredConstsVisitor;
17use rustc_const_eval::check_consts::{self, ConstCx};
18use rustc_const_eval::util;
19use rustc_data_structures::fx::FxIndexSet;
20use rustc_data_structures::steal::Steal;
21use rustc_hir as hir;
22use rustc_hir::def::{CtorKind, DefKind};
23use rustc_hir::def_id::LocalDefId;
24use rustc_index::IndexVec;
25use rustc_middle::mir::{
26 AnalysisPhase, Body, CallSource, ClearCrossCrate, ConstOperand, ConstQualifs, LocalDecl,
27 MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, START_BLOCK,
28 SourceInfo, Statement, StatementKind, TerminatorKind,
29};
30use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
31use rustc_middle::util::Providers;
32use rustc_middle::{bug, query, span_bug};
33use rustc_mir_build::builder::build_mir;
34use rustc_span::source_map::Spanned;
35use rustc_span::{DUMMY_SP, sym};
36use tracing::debug;
37
38#[macro_use]
39mod pass_manager;
40
41use std::sync::LazyLock;
42
43use pass_manager::{self as pm, Lint, MirLint, MirPass, WithMinOptLevel};
44
45mod check_pointers;
46mod cost_checker;
47mod cross_crate_inline;
48mod deduce_param_attrs;
49mod elaborate_drop;
50mod errors;
51mod ffi_unwind_calls;
52mod lint;
53mod lint_tail_expr_drop_order;
54mod patch;
55mod shim;
56mod ssa;
57
58macro_rules! declare_passes {
81 (
82 $(
83 $vis:vis mod $mod_name:ident : $($pass_name:ident $( { $($ident:ident),* } )?),+ $(,)?;
84 )*
85 ) => {
86 $(
87 $vis mod $mod_name;
88 $(
89 #[allow(unused_imports)]
91 use $mod_name::$pass_name as _;
92 )+
93 )*
94
95 static PASS_NAMES: LazyLock<FxIndexSet<&str>> = LazyLock::new(|| [
96 "PreCodegen",
98 $(
99 $(
100 stringify!($pass_name),
101 $(
102 $(
103 $mod_name::$pass_name::$ident.name(),
104 )*
105 )?
106 )+
107 )*
108 ].into_iter().collect());
109 };
110}
111
112declare_passes! {
113 mod abort_unwinding_calls : AbortUnwindingCalls;
114 mod add_call_guards : AddCallGuards { AllCallEdges, CriticalCallEdges };
115 mod add_moves_for_packed_drops : AddMovesForPackedDrops;
116 mod add_retag : AddRetag;
117 mod add_subtyping_projections : Subtyper;
118 mod check_inline : CheckForceInline;
119 mod check_call_recursion : CheckCallRecursion, CheckDropRecursion;
120 mod check_inline_always_target_features: CheckInlineAlwaysTargetFeature;
121 mod check_alignment : CheckAlignment;
122 mod check_enums : CheckEnums;
123 mod check_const_item_mutation : CheckConstItemMutation;
124 mod check_null : CheckNull;
125 mod check_packed_ref : CheckPackedRef;
126 pub mod cleanup_post_borrowck : CleanupPostBorrowck;
128
129 mod copy_prop : CopyProp;
130 mod coroutine : StateTransform;
131 mod coverage : InstrumentCoverage;
132 mod ctfe_limit : CtfeLimit;
133 mod dataflow_const_prop : DataflowConstProp;
134 mod dead_store_elimination : DeadStoreElimination {
135 Initial,
136 Final
137 };
138 mod deref_separator : Derefer;
139 mod dest_prop : DestinationPropagation;
140 pub mod dump_mir : Marker;
141 mod early_otherwise_branch : EarlyOtherwiseBranch;
142 mod elaborate_box_derefs : ElaborateBoxDerefs;
143 mod elaborate_drops : ElaborateDrops;
144 mod function_item_references : FunctionItemReferences;
145 mod gvn : GVN;
146 pub mod inline : Inline, ForceInline;
149 mod impossible_predicates : ImpossiblePredicates;
150 mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg };
151 mod jump_threading : JumpThreading;
152 mod known_panics_lint : KnownPanicsLint;
153 mod large_enums : EnumSizeOpt;
154 mod lower_intrinsics : LowerIntrinsics;
155 mod lower_slice_len : LowerSliceLenCalls;
156 mod match_branches : MatchBranchSimplification;
157 mod mentioned_items : MentionedItems;
158 mod multiple_return_terminators : MultipleReturnTerminators;
159 mod post_drop_elaboration : CheckLiveDrops;
160 mod prettify : ReorderBasicBlocks, ReorderLocals;
161 mod promote_consts : PromoteTemps;
162 mod ref_prop : ReferencePropagation;
163 mod remove_noop_landing_pads : RemoveNoopLandingPads;
164 mod remove_place_mention : RemovePlaceMention;
165 mod remove_storage_markers : RemoveStorageMarkers;
166 mod remove_uninit_drops : RemoveUninitDrops;
167 mod remove_unneeded_drops : RemoveUnneededDrops;
168 mod remove_zsts : RemoveZsts;
169 mod required_consts : RequiredConstsVisitor;
170 mod post_analysis_normalize : PostAnalysisNormalize;
171 mod sanity_check : SanityCheck;
172 pub mod simplify :
174 SimplifyCfg {
175 Initial,
176 PromoteConsts,
177 RemoveFalseEdges,
178 PostAnalysis,
179 PreOptimizations,
180 Final,
181 MakeShim,
182 AfterUnreachableEnumBranching
183 },
184 SimplifyLocals {
185 BeforeConstProp,
186 AfterGVN,
187 Final
188 };
189 mod simplify_branches : SimplifyConstCondition {
190 AfterConstProp,
191 Final
192 };
193 mod simplify_comparison_integral : SimplifyComparisonIntegral;
194 mod single_use_consts : SingleUseConsts;
195 mod sroa : ScalarReplacementOfAggregates;
196 mod strip_debuginfo : StripDebugInfo;
197 mod unreachable_enum_branching : UnreachableEnumBranching;
198 mod unreachable_prop : UnreachablePropagation;
199 mod validate : Validator;
200}
201
202rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
203
204pub fn provide(providers: &mut Providers) {
205 coverage::query::provide(providers);
206 ffi_unwind_calls::provide(providers);
207 shim::provide(providers);
208 cross_crate_inline::provide(providers);
209 providers.queries = query::Providers {
210 mir_keys,
211 mir_built,
212 mir_const_qualif,
213 mir_promoted,
214 mir_drops_elaborated_and_const_checked,
215 mir_for_ctfe,
216 mir_coroutine_witnesses: coroutine::mir_coroutine_witnesses,
217 optimized_mir,
218 is_mir_available,
219 is_ctfe_mir_available: is_mir_available,
220 mir_callgraph_cyclic: inline::cycle::mir_callgraph_cyclic,
221 mir_inliner_callees: inline::cycle::mir_inliner_callees,
222 promoted_mir,
223 deduced_param_attrs: deduce_param_attrs::deduced_param_attrs,
224 coroutine_by_move_body_def_id: coroutine::coroutine_by_move_body_def_id,
225 ..providers.queries
226 };
227}
228
229fn remap_mir_for_const_eval_select<'tcx>(
230 tcx: TyCtxt<'tcx>,
231 mut body: Body<'tcx>,
232 context: hir::Constness,
233) -> Body<'tcx> {
234 for bb in body.basic_blocks.as_mut().iter_mut() {
235 let terminator = bb.terminator.as_mut().expect("invalid terminator");
236 match terminator.kind {
237 TerminatorKind::Call {
238 func: Operand::Constant(box ConstOperand { ref const_, .. }),
239 ref mut args,
240 destination,
241 target,
242 unwind,
243 fn_span,
244 ..
245 } if let ty::FnDef(def_id, _) = *const_.ty().kind()
246 && tcx.is_intrinsic(def_id, sym::const_eval_select) =>
247 {
248 let Ok([tupled_args, called_in_const, called_at_rt]) = take_array(args) else {
249 unreachable!()
250 };
251 let ty = tupled_args.node.ty(&body.local_decls, tcx);
252 let fields = ty.tuple_fields();
253 let num_args = fields.len();
254 let func =
255 if context == hir::Constness::Const { called_in_const } else { called_at_rt };
256 let (method, place): (fn(Place<'tcx>) -> Operand<'tcx>, Place<'tcx>) =
257 match tupled_args.node {
258 Operand::Constant(_) => {
259 let local = body.local_decls.push(LocalDecl::new(ty, fn_span));
263 bb.statements.push(Statement::new(
264 SourceInfo::outermost(fn_span),
265 StatementKind::Assign(Box::new((
266 local.into(),
267 Rvalue::Use(tupled_args.node.clone()),
268 ))),
269 ));
270 (Operand::Move, local.into())
271 }
272 Operand::Move(place) => (Operand::Move, place),
273 Operand::Copy(place) => (Operand::Copy, place),
274 };
275 let place_elems = place.projection;
276 let arguments = (0..num_args)
277 .map(|x| {
278 let mut place_elems = place_elems.to_vec();
279 place_elems.push(ProjectionElem::Field(x.into(), fields[x]));
280 let projection = tcx.mk_place_elems(&place_elems);
281 let place = Place { local: place.local, projection };
282 Spanned { node: method(place), span: DUMMY_SP }
283 })
284 .collect();
285 terminator.kind = TerminatorKind::Call {
286 func: func.node,
287 args: arguments,
288 destination,
289 target,
290 unwind,
291 call_source: CallSource::Misc,
292 fn_span,
293 };
294 }
295 _ => {}
296 }
297 }
298 body
299}
300
301fn take_array<T, const N: usize>(b: &mut Box<[T]>) -> Result<[T; N], Box<[T]>> {
302 let b: Box<[T; N]> = std::mem::take(b).try_into()?;
303 Ok(*b)
304}
305
306fn is_mir_available(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
307 tcx.mir_keys(()).contains(&def_id)
308}
309
310fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> {
313 let mut set: FxIndexSet<_> = tcx.hir_body_owners().collect();
315
316 set.retain(|&def_id| !matches!(tcx.def_kind(def_id), DefKind::GlobalAsm));
319
320 for body_owner in tcx.hir_body_owners() {
323 if let DefKind::Closure = tcx.def_kind(body_owner)
324 && tcx.needs_coroutine_by_move_body_def_id(body_owner.to_def_id())
325 {
326 set.insert(tcx.coroutine_by_move_body_def_id(body_owner).expect_local());
327 }
328 }
329
330 for item in tcx.hir_crate_items(()).free_items() {
333 if let DefKind::Struct | DefKind::Enum = tcx.def_kind(item.owner_id) {
334 for variant in tcx.adt_def(item.owner_id).variants() {
335 if let Some((CtorKind::Fn, ctor_def_id)) = variant.ctor {
336 set.insert(ctor_def_id.expect_local());
337 }
338 }
339 }
340 }
341
342 set
343}
344
345fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs {
346 let body = &tcx.mir_built(def).borrow();
351 let ccx = check_consts::ConstCx::new(tcx, body);
352 match ccx.const_kind {
354 Some(ConstContext::Const { .. } | ConstContext::Static(_) | ConstContext::ConstFn) => {}
355 None => span_bug!(
356 tcx.def_span(def),
357 "`mir_const_qualif` should only be called on const fns and const items"
358 ),
359 }
360
361 if body.return_ty().references_error() {
362 tcx.dcx().span_delayed_bug(body.span, "mir_const_qualif: MIR had errors");
364 return Default::default();
365 }
366
367 let mut validator = check_consts::check::Checker::new(&ccx);
368 validator.check_body();
369
370 validator.qualifs_in_return_place()
373}
374
375fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
376 let mut body = build_mir(tcx, def);
377
378 pass_manager::dump_mir_for_phase_change(tcx, &body);
379
380 pm::run_passes(
381 tcx,
382 &mut body,
383 &[
384 &Lint(check_inline::CheckForceInline),
386 &Lint(check_call_recursion::CheckCallRecursion),
387 &Lint(check_inline_always_target_features::CheckInlineAlwaysTargetFeature),
390 &Lint(check_packed_ref::CheckPackedRef),
391 &Lint(check_const_item_mutation::CheckConstItemMutation),
392 &Lint(function_item_references::FunctionItemReferences),
393 &simplify::SimplifyCfg::Initial,
395 &Lint(sanity_check::SanityCheck),
396 ],
397 None,
398 pm::Optimizations::Allowed,
399 );
400 tcx.alloc_steal_mir(body)
401}
402
403fn mir_promoted(
405 tcx: TyCtxt<'_>,
406 def: LocalDefId,
407) -> (&Steal<Body<'_>>, &Steal<IndexVec<Promoted, Body<'_>>>) {
408 let const_qualifs = match tcx.def_kind(def) {
413 DefKind::Fn | DefKind::AssocFn | DefKind::Closure
414 if tcx.constness(def) == hir::Constness::Const
415 || tcx.is_const_default_method(def.to_def_id()) =>
416 {
417 tcx.mir_const_qualif(def)
418 }
419 DefKind::AssocConst
420 | DefKind::Const
421 | DefKind::Static { .. }
422 | DefKind::InlineConst
423 | DefKind::AnonConst => tcx.mir_const_qualif(def),
424 _ => ConstQualifs::default(),
425 };
426
427 tcx.ensure_done().has_ffi_unwind_calls(def);
429
430 if tcx.needs_coroutine_by_move_body_def_id(def.to_def_id()) {
432 tcx.ensure_done().coroutine_by_move_body_def_id(def);
433 }
434
435 let mut body = tcx.mir_built(def).steal();
436 if let Some(error_reported) = const_qualifs.tainted_by_errors {
437 body.tainted_by_errors = Some(error_reported);
438 }
439
440 RequiredConstsVisitor::compute_required_consts(&mut body);
443
444 let promote_pass = promote_consts::PromoteTemps::default();
446 pm::run_passes(
447 tcx,
448 &mut body,
449 &[&promote_pass, &simplify::SimplifyCfg::PromoteConsts, &coverage::InstrumentCoverage],
450 Some(MirPhase::Analysis(AnalysisPhase::Initial)),
451 pm::Optimizations::Allowed,
452 );
453
454 lint_tail_expr_drop_order::run_lint(tcx, def, &body);
455
456 let promoted = promote_pass.promoted_fragments.into_inner();
457 (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted))
458}
459
460fn mir_for_ctfe(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &Body<'_> {
462 tcx.arena.alloc(inner_mir_for_ctfe(tcx, def_id))
463}
464
465fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> {
466 if tcx.is_constructor(def.to_def_id()) {
468 return shim::build_adt_ctor(tcx, def.to_def_id());
473 }
474
475 let body = tcx.mir_drops_elaborated_and_const_checked(def);
476 let body = match tcx.hir_body_const_context(def) {
477 Some(hir::ConstContext::Const { .. } | hir::ConstContext::Static(_)) => body.steal(),
480 Some(hir::ConstContext::ConstFn) => body.borrow().clone(),
481 None => bug!("`mir_for_ctfe` called on non-const {def:?}"),
482 };
483
484 let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::Const);
485 pm::run_passes(tcx, &mut body, &[&ctfe_limit::CtfeLimit], None, pm::Optimizations::Allowed);
486
487 body
488}
489
490fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
494 if tcx.is_coroutine(def.to_def_id()) {
495 tcx.ensure_done().mir_coroutine_witnesses(def);
496 }
497
498 let tainted_by_errors = if !tcx.is_synthetic_mir(def) {
500 tcx.mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local()).err()
501 } else {
502 None
503 };
504
505 let is_fn_like = tcx.def_kind(def).is_fn_like();
506 if is_fn_like {
507 if pm::should_run_pass(tcx, &inline::Inline, pm::Optimizations::Allowed)
509 || inline::ForceInline::should_run_pass_for_callee(tcx, def.to_def_id())
510 {
511 tcx.ensure_done().mir_inliner_callees(ty::InstanceKind::Item(def.to_def_id()));
512 }
513 }
514
515 let (body, _) = tcx.mir_promoted(def);
516 let mut body = body.steal();
517
518 if let Some(error_reported) = tainted_by_errors {
519 body.tainted_by_errors = Some(error_reported);
520 }
521
522 let root = tcx.typeck_root_def_id(def.to_def_id());
527 match tcx.def_kind(root) {
528 DefKind::Fn
529 | DefKind::AssocFn
530 | DefKind::Static { .. }
531 | DefKind::Const
532 | DefKind::AssocConst => {
533 if let Err(guar) = tcx.ensure_ok().check_well_formed(root.expect_local()) {
534 body.tainted_by_errors = Some(guar);
535 }
536 }
537 _ => {}
538 }
539
540 run_analysis_to_runtime_passes(tcx, &mut body);
541
542 tcx.alloc_steal_mir(body)
543}
544
545pub fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
548 assert!(body.phase == MirPhase::Analysis(AnalysisPhase::Initial));
549 let did = body.source.def_id();
550
551 debug!("analysis_mir_cleanup({:?})", did);
552 run_analysis_cleanup_passes(tcx, body);
553 assert!(body.phase == MirPhase::Analysis(AnalysisPhase::PostCleanup));
554
555 if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, body)) {
557 pm::run_passes(
558 tcx,
559 body,
560 &[
561 &remove_uninit_drops::RemoveUninitDrops,
562 &simplify::SimplifyCfg::RemoveFalseEdges,
563 &Lint(post_drop_elaboration::CheckLiveDrops),
564 ],
565 None,
566 pm::Optimizations::Allowed,
567 );
568 }
569
570 debug!("runtime_mir_lowering({:?})", did);
571 run_runtime_lowering_passes(tcx, body);
572 assert!(body.phase == MirPhase::Runtime(RuntimePhase::Initial));
573
574 debug!("runtime_mir_cleanup({:?})", did);
575 run_runtime_cleanup_passes(tcx, body);
576 assert!(body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup));
577}
578
579fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
583 let passes: &[&dyn MirPass<'tcx>] = &[
584 &impossible_predicates::ImpossiblePredicates,
585 &cleanup_post_borrowck::CleanupPostBorrowck,
586 &remove_noop_landing_pads::RemoveNoopLandingPads,
587 &simplify::SimplifyCfg::PostAnalysis,
588 &deref_separator::Derefer,
589 ];
590
591 pm::run_passes(
592 tcx,
593 body,
594 passes,
595 Some(MirPhase::Analysis(AnalysisPhase::PostCleanup)),
596 pm::Optimizations::Allowed,
597 );
598}
599
600fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
602 let passes: &[&dyn MirPass<'tcx>] = &[
603 &add_call_guards::CriticalCallEdges,
605 &post_analysis_normalize::PostAnalysisNormalize,
607 &add_subtyping_projections::Subtyper,
609 &elaborate_drops::ElaborateDrops,
610 &Lint(check_call_recursion::CheckDropRecursion),
612 &abort_unwinding_calls::AbortUnwindingCalls,
616 &add_moves_for_packed_drops::AddMovesForPackedDrops,
619 &add_retag::AddRetag,
622 &elaborate_box_derefs::ElaborateBoxDerefs,
623 &coroutine::StateTransform,
624 &Lint(known_panics_lint::KnownPanicsLint),
625 ];
626 pm::run_passes_no_validate(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::Initial)));
627}
628
629fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
631 let passes: &[&dyn MirPass<'tcx>] = &[
632 &lower_intrinsics::LowerIntrinsics,
633 &remove_place_mention::RemovePlaceMention,
634 &simplify::SimplifyCfg::PreOptimizations,
635 ];
636
637 pm::run_passes(
638 tcx,
639 body,
640 passes,
641 Some(MirPhase::Runtime(RuntimePhase::PostCleanup)),
642 pm::Optimizations::Allowed,
643 );
644
645 for decl in &mut body.local_decls {
648 decl.local_info = ClearCrossCrate::Clear;
649 }
650}
651
652pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
653 fn o1<T>(x: T) -> WithMinOptLevel<T> {
654 WithMinOptLevel(1, x)
655 }
656
657 let def_id = body.source.def_id();
658 let optimizations = if tcx.def_kind(def_id).has_codegen_attrs()
659 && tcx.codegen_fn_attrs(def_id).optimize.do_not_optimize()
660 {
661 pm::Optimizations::Suppressed
662 } else {
663 pm::Optimizations::Allowed
664 };
665
666 pm::run_passes(
668 tcx,
669 body,
670 &[
671 &check_alignment::CheckAlignment,
673 &check_null::CheckNull,
674 &check_enums::CheckEnums,
675 &lower_slice_len::LowerSliceLenCalls,
680 &instsimplify::InstSimplify::BeforeInline,
683 &inline::ForceInline,
685 &inline::Inline,
687 &remove_storage_markers::RemoveStorageMarkers,
690 &remove_zsts::RemoveZsts,
692 &remove_unneeded_drops::RemoveUnneededDrops,
693 &unreachable_enum_branching::UnreachableEnumBranching,
696 &unreachable_prop::UnreachablePropagation,
697 &o1(simplify::SimplifyCfg::AfterUnreachableEnumBranching),
698 &ref_prop::ReferencePropagation,
701 &sroa::ScalarReplacementOfAggregates,
702 &multiple_return_terminators::MultipleReturnTerminators,
703 &instsimplify::InstSimplify::AfterSimplifyCfg,
706 &simplify::SimplifyLocals::BeforeConstProp,
707 &dead_store_elimination::DeadStoreElimination::Initial,
708 &gvn::GVN,
709 &simplify::SimplifyLocals::AfterGVN,
710 &match_branches::MatchBranchSimplification,
711 &dataflow_const_prop::DataflowConstProp,
712 &single_use_consts::SingleUseConsts,
713 &o1(simplify_branches::SimplifyConstCondition::AfterConstProp),
714 &jump_threading::JumpThreading,
715 &early_otherwise_branch::EarlyOtherwiseBranch,
716 &simplify_comparison_integral::SimplifyComparisonIntegral,
717 &o1(simplify_branches::SimplifyConstCondition::Final),
718 &o1(remove_noop_landing_pads::RemoveNoopLandingPads),
719 &o1(simplify::SimplifyCfg::Final),
720 &strip_debuginfo::StripDebugInfo,
722 ©_prop::CopyProp,
723 &dead_store_elimination::DeadStoreElimination::Final,
724 &dest_prop::DestinationPropagation,
725 &simplify::SimplifyLocals::Final,
726 &multiple_return_terminators::MultipleReturnTerminators,
727 &large_enums::EnumSizeOpt { discrepancy: 128 },
728 &add_call_guards::CriticalCallEdges,
730 &prettify::ReorderBasicBlocks,
732 &prettify::ReorderLocals,
733 &dump_mir::Marker("PreCodegen"),
735 ],
736 Some(MirPhase::Runtime(RuntimePhase::Optimized)),
737 optimizations,
738 );
739}
740
741fn optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> &Body<'_> {
743 tcx.arena.alloc(inner_optimized_mir(tcx, did))
744}
745
746fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> {
747 if tcx.is_constructor(did.to_def_id()) {
748 return shim::build_adt_ctor(tcx, did.to_def_id());
753 }
754
755 match tcx.hir_body_const_context(did) {
756 Some(hir::ConstContext::ConstFn) => tcx.ensure_done().mir_for_ctfe(did),
760 None => {}
761 Some(other) => panic!("do not use `optimized_mir` for constants: {other:?}"),
762 }
763 debug!("about to call mir_drops_elaborated...");
764 let body = tcx.mir_drops_elaborated_and_const_checked(did).steal();
765 let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::NotConst);
766
767 if body.tainted_by_errors.is_some() {
768 return body;
769 }
770
771 mentioned_items::MentionedItems.run_pass(tcx, &mut body);
775
776 if let TerminatorKind::Unreachable = body.basic_blocks[START_BLOCK].terminator().kind
780 && body.basic_blocks[START_BLOCK].statements.is_empty()
781 {
782 return body;
783 }
784
785 run_optimization_passes(tcx, &mut body);
786
787 body
788}
789
790fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_>> {
793 if tcx.is_constructor(def.to_def_id()) {
794 return tcx.arena.alloc(IndexVec::new());
795 }
796
797 if !tcx.is_synthetic_mir(def) {
798 tcx.ensure_done().mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local());
799 }
800 let mut promoted = tcx.mir_promoted(def).1.steal();
801
802 for body in &mut promoted {
803 run_analysis_to_runtime_passes(tcx, body);
804 }
805
806 tcx.arena.alloc(promoted)
807}