Skip to main content

rustc_mir_build/builder/
mod.rs

1//! This module used to be named `build`, but that was causing GitHub's
2//! "Go to file" feature to silently ignore all files in the module, probably
3//! because it assumes that "build" is a build-output directory.
4//! See <https://github.com/rust-lang/rust/pull/134365>.
5//!
6//! ## The `let this = self;` idiom (LET_THIS_SELF)
7//!
8//! Throughout MIR building there are several places where a `Builder` method
9//! needs to borrow `self`, and then re-expose it to a closure as `|this|`.
10//!
11//! In complex builder methods, potentially with multiple levels of nesting, it
12//! would thus become necessary to mentally keep track of whether the builder
13//! is `self` (at the top level) or `this` (nested in a closure), or to replace
14//! one with the other when moving code in or out of a closure.
15//!
16//! (The borrow checker will prevent incorrect usage, but having to go back and
17//! satisfy the borrow checker still creates contributor friction.)
18//!
19//! To reduce that friction, some builder methods therefore start with
20//! `let this = self;` or similar, allowing subsequent code to uniformly refer
21//! to the builder as `this` (and never `self`), even when not nested.
22
23use itertools::{Either, Itertools};
24use rustc_abi::{ExternAbi, FieldIdx};
25use rustc_apfloat::Float;
26use rustc_apfloat::ieee::{Double, Half, Quad, Single};
27use rustc_attr_ir::find_attr;
28use rustc_data_structures::fx::FxHashMap;
29use rustc_data_structures::sorted_map::SortedIndexMultiMap;
30use rustc_errors::ErrorGuaranteed;
31use rustc_hir::def::DefKind;
32use rustc_hir::def_id::LocalDefId;
33use rustc_hir::{self as hir, BindingMode, ByRef, HirId, ItemLocalId, Node};
34use rustc_index::bit_set::GrowableBitSet;
35use rustc_index::{Idx, IndexSlice, IndexVec};
36use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
37use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
38use rustc_middle::middle::region;
39use rustc_middle::mir::*;
40use rustc_middle::thir::{self, ExprId, LocalVarId, Param, ParamId, PatKind, Thir};
41use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt, TypeVisitableExt, TypingMode};
42use rustc_span::{Span, Symbol, bug, span_bug};
43
44use crate::builder::expr::as_place::PlaceBuilder;
45use crate::builder::scope::LintLevel;
46
47pub(crate) fn closure_saved_names_of_captured_variables<'tcx>(
48    tcx: TyCtxt<'tcx>,
49    def_id: LocalDefId,
50) -> IndexVec<FieldIdx, Symbol> {
51    tcx.closure_captures(def_id)
52        .iter()
53        .map(|captured_place| {
54            let name = captured_place.to_symbol();
55            match captured_place.info.capture_kind {
56                ty::UpvarCapture::ByValue | ty::UpvarCapture::ByUse => name,
57                ty::UpvarCapture::ByRef(..) => Symbol::intern(&::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("_ref__{0}", name))
    })format!("_ref__{name}")),
58            }
59        })
60        .collect()
61}
62
63/// Create the MIR for a given `DefId`, including unreachable code.
64///
65/// This is the implementation of hook `build_mir_inner_impl`, which should only
66/// be called by the query `mir_built`.
67pub(crate) fn build_mir_inner_impl<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> Body<'tcx> {
68    if #[allow(non_exhaustive_omitted_patterns)] match tcx.def_kind(def) {
    DefKind::Const | DefKind::AssocConst => true,
    _ => false,
}matches!(tcx.def_kind(def), DefKind::Const | DefKind::AssocConst)
69        && tcx.const_of_item(def).is_some()
70    {
71        return mir_for_direct_const(tcx, def);
72    }
73
74    tcx.ensure_done().thir_abstract_const(def);
75    if let Err(e) = tcx.ensure_result().check_match(def) {
76        return construct_error(tcx, def, e);
77    }
78
79    if let Err(err) = tcx.ensure_result().check_tail_calls(def) {
80        return construct_error(tcx, def, err);
81    }
82
83    let body = match tcx.thir_body(def) {
84        Err(error_reported) => construct_error(tcx, def, error_reported),
85        Ok((thir, expr)) => {
86            let build_mir = |thir: &Thir<'tcx>| match thir.body_type {
87                thir::BodyTy::Fn(fn_sig) => construct_fn(tcx, def, thir, expr, fn_sig),
88                thir::BodyTy::Const(ty) | thir::BodyTy::GlobalAsm(ty) => {
89                    construct_const(tcx, def, thir, expr, ty)
90                }
91            };
92
93            // Don't steal here, instead steal in unsafeck. This is so that
94            // pattern inline constants can be evaluated as part of building the
95            // THIR of the parent function without a cycle.
96            build_mir(&thir.borrow())
97        }
98    };
99
100    // The borrow checker will replace all the regions here with its own
101    // inference variables. There's no point having non-erased regions here.
102    // The exception is `body.user_type_annotations`, which is used unmodified
103    // by borrow checking.
104    if true {
    if !!(body.local_decls.has_free_regions() ||
                                body.basic_blocks.has_free_regions() ||
                            body.var_debug_info.has_free_regions() ||
                        body.yield_ty().has_free_regions()) {
        {
            ::core::panicking::panic_fmt(format_args!("Unexpected free regions in MIR: {0:?}",
                    body));
        }
    };
};debug_assert!(
105        !(body.local_decls.has_free_regions()
106            || body.basic_blocks.has_free_regions()
107            || body.var_debug_info.has_free_regions()
108            || body.yield_ty().has_free_regions()),
109        "Unexpected free regions in MIR: {body:?}",
110    );
111
112    body
113}
114
115///////////////////////////////////////////////////////////////////////////
116// BuildMir -- walks a crate, looking for fn items and methods to build MIR from
117
118#[derive(#[automatically_derived]impl ::core::fmt::Debug for BlockFrame {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        match self {
            Self::Statement { ignores_expr_result: __self_0 } =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "Statement ignores_expr_result", &__self_0),
            Self::TailExpr { info: __self_0 } =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f,
                    "TailExpr info", &__self_0),
            Self::SubExpr => ::core::fmt::Formatter::write_str(f, "SubExpr"),
        }
    }
}Debug, #[automatically_derived]impl ::core::marker::StructuralPartialEq for
    BlockFrame {
}
#[automatically_derived]impl ::core::cmp::PartialEq for BlockFrame {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
                ::core::intrinsics::discriminant_value(other) &&
            match (self, other) {
                (Self::Statement { ignores_expr_result: __self_0 },
                    Self::Statement { ignores_expr_result: __arg1_0 }) =>
                    __self_0 == __arg1_0,
                (Self::TailExpr { info: __self_0 }, Self::TailExpr {
                    info: __arg1_0 }) => __self_0 == __arg1_0,
                _ => true,
            }
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for BlockFrame {
    #[inline]
    #[doc(hidden)]
    #[coverage(off)]
    fn assert_fields_are_eq(&self) {
        let _: ::core::cmp::AssertParamIsEq<bool>;
        let _: ::core::cmp::AssertParamIsEq<BlockTailInfo>;
    }
}Eq)]
119enum BlockFrame {
120    /// Evaluation is currently within a statement.
121    ///
122    /// Examples include:
123    /// 1. `EXPR;`
124    /// 2. `let _ = EXPR;`
125    /// 3. `let x = EXPR;`
126    Statement {
127        /// If true, then statement discards result from evaluating
128        /// the expression (such as examples 1 and 2 above).
129        ignores_expr_result: bool,
130    },
131
132    /// Evaluation is currently within the tail expression of a block.
133    ///
134    /// Example: `{ STMT_1; STMT_2; EXPR }`
135    TailExpr { info: BlockTailInfo },
136
137    /// Generic mark meaning that the block occurred as a subexpression
138    /// where the result might be used.
139    ///
140    /// Examples: `foo(EXPR)`, `match EXPR { ... }`
141    SubExpr,
142}
143
144impl BlockFrame {
145    fn is_tail_expr(&self) -> bool {
146        match *self {
147            BlockFrame::TailExpr { .. } => true,
148
149            BlockFrame::Statement { .. } | BlockFrame::SubExpr => false,
150        }
151    }
152    fn is_statement(&self) -> bool {
153        match *self {
154            BlockFrame::Statement { .. } => true,
155
156            BlockFrame::TailExpr { .. } | BlockFrame::SubExpr => false,
157        }
158    }
159}
160
161#[derive(#[automatically_derived]impl ::core::fmt::Debug for BlockContext {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field1_finish(f, "BlockContext",
            &&self.0)
    }
}Debug)]
162struct BlockContext(Vec<BlockFrame>);
163
164struct Builder<'a, 'tcx> {
165    tcx: TyCtxt<'tcx>,
166    // FIXME(@lcnr): Why does this use an `infcx`, there should be
167    // no shared type inference going on here. I feel like it would
168    // clearer to manually construct one where necessary or to provide
169    // a nice API for non-type inference trait system checks.
170    infcx: InferCtxt<'tcx>,
171    region_scope_tree: &'tcx region::ScopeTree,
172    param_env: ty::ParamEnv<'tcx>,
173
174    thir: &'a Thir<'tcx>,
175    cfg: CFG<'tcx>,
176
177    def_id: LocalDefId,
178    hir_id: HirId,
179    check_overflow: bool,
180    fn_span: Span,
181    arg_count: usize,
182    coroutine: Option<Box<CoroutineInfo<'tcx>>>,
183
184    /// The current set of scopes, updated as we traverse;
185    /// see the `scope` module for more details.
186    scopes: scope::Scopes<'tcx>,
187
188    /// The block-context: each time we build the code within an thir::Block,
189    /// we push a frame here tracking whether we are building a statement or
190    /// if we are pushing the tail expression of the block. This is used to
191    /// embed information in generated temps about whether they were created
192    /// for a block tail expression or not.
193    ///
194    /// It would be great if we could fold this into `self.scopes`
195    /// somehow, but right now I think that is very tightly tied to
196    /// the code generation in ways that we cannot (or should not)
197    /// start just throwing new entries onto that vector in order to
198    /// distinguish the context of EXPR1 from the context of EXPR2 in
199    /// `{ STMTS; EXPR1 } + EXPR2`.
200    block_context: BlockContext,
201
202    /// The vector of all scopes that we have created thus far;
203    /// we track this for debuginfo later.
204    source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>,
205    source_scope: SourceScope,
206
207    /// The guard-context: each time we build the guard expression for
208    /// a match arm, we push onto this stack, and then pop when we
209    /// finish building it.
210    guard_context: Vec<GuardFrame>,
211
212    /// Temporaries with fixed indexes. Used so that if-let guards on arms
213    /// with an or-pattern are only created once.
214    fixed_temps: FxHashMap<ExprId, Local>,
215    /// Scope of temporaries that should be deduplicated using [Self::fixed_temps].
216    fixed_temps_scope: Option<region::Scope>,
217
218    /// Maps `HirId`s of variable bindings to the `Local`s created for them.
219    /// (A match binding can have two locals; the 2nd is for the arm's guard.)
220    var_indices: FxHashMap<LocalVarId, LocalsForNode>,
221    local_decls: IndexVec<Local, LocalDecl<'tcx>>,
222    canonical_user_type_annotations: ty::CanonicalUserTypeAnnotations<'tcx>,
223    upvars: CaptureMap<'tcx>,
224    unit_temp: Option<Place<'tcx>>,
225
226    var_debug_info: Vec<VarDebugInfo<'tcx>>,
227
228    // A cache for `maybe_lint_level_roots_bounded`. That function is called
229    // repeatedly, and each time it effectively traces a path through a tree
230    // structure from a node towards the root, doing an attribute check on each
231    // node along the way. This cache records which nodes trace all the way to
232    // the root (most of them do) and saves us from retracing many sub-paths
233    // many times, and rechecking many nodes.
234    lint_level_roots_cache: GrowableBitSet<hir::ItemLocalId>,
235
236    /// Collects additional coverage information during MIR building.
237    /// Only present if coverage is enabled and this function is eligible.
238    coverage_info: Option<coverageinfo::CoverageInfoBuilder>,
239}
240
241type CaptureMap<'tcx> = SortedIndexMultiMap<usize, ItemLocalId, Capture<'tcx>>;
242
243#[derive(#[automatically_derived]impl<'tcx> ::core::fmt::Debug for Capture<'tcx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field3_finish(f,
            "Capture captured_place use_place mutability",
            &self.captured_place, &self.use_place, &&self.mutability)
    }
}Debug)]
244struct Capture<'tcx> {
245    captured_place: &'tcx ty::CapturedPlace<'tcx>,
246    use_place: Place<'tcx>,
247    mutability: Mutability,
248}
249
250impl<'a, 'tcx> Builder<'a, 'tcx> {
251    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
252        self.infcx.typing_env(self.param_env)
253    }
254
255    fn is_bound_var_in_guard(&self, id: LocalVarId) -> bool {
256        self.guard_context.iter().any(|frame| frame.locals.iter().any(|local| local.id == id))
257    }
258
259    fn var_local_id(&self, id: LocalVarId, for_guard: ForGuard) -> Local {
260        self.var_indices[&id].local_id(for_guard)
261    }
262}
263
264impl BlockContext {
265    fn new() -> Self {
266        BlockContext(::alloc::vec::Vec::new()vec![])
267    }
268    fn push(&mut self, bf: BlockFrame) {
269        self.0.push(bf);
270    }
271    fn pop(&mut self) -> Option<BlockFrame> {
272        self.0.pop()
273    }
274
275    /// Traverses the frames on the `BlockContext`, searching for either
276    /// the first block-tail expression frame with no intervening
277    /// statement frame.
278    ///
279    /// Notably, this skips over `SubExpr` frames; this method is
280    /// meant to be used in the context of understanding the
281    /// relationship of a temp (created within some complicated
282    /// expression) with its containing expression, and whether the
283    /// value of that *containing expression* (not the temp!) is
284    /// ignored.
285    fn currently_in_block_tail(&self) -> Option<BlockTailInfo> {
286        for bf in self.0.iter().rev() {
287            match bf {
288                BlockFrame::SubExpr => continue,
289                BlockFrame::Statement { .. } => break,
290                &BlockFrame::TailExpr { info } => return Some(info),
291            }
292        }
293
294        None
295    }
296
297    /// Looks at the topmost frame on the BlockContext and reports
298    /// whether its one that would discard a block tail result.
299    ///
300    /// Unlike `currently_within_ignored_tail_expression`, this does
301    /// *not* skip over `SubExpr` frames: here, we want to know
302    /// whether the block result itself is discarded.
303    fn currently_ignores_tail_results(&self) -> bool {
304        match self.0.last() {
305            // no context: conservatively assume result is read
306            None => false,
307
308            // sub-expression: block result feeds into some computation
309            Some(BlockFrame::SubExpr) => false,
310
311            // otherwise: use accumulated is_ignored state.
312            Some(
313                BlockFrame::TailExpr { info: BlockTailInfo { tail_result_is_ignored: ign, .. } }
314                | BlockFrame::Statement { ignores_expr_result: ign },
315            ) => *ign,
316        }
317    }
318}
319
320#[derive(#[automatically_derived]impl ::core::fmt::Debug for LocalsForNode {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        match self {
            Self::One(__self_0) =>
                ::core::fmt::Formatter::debug_struct_field1_finish(f, "One",
                    &__self_0),
            Self::ForGuard { ref_for_guard: __self_0, for_arm_body: __self_1 }
                =>
                ::core::fmt::Formatter::debug_struct_field2_finish(f,
                    "ForGuard ref_for_guard for_arm_body", __self_0, &__self_1),
        }
    }
}Debug)]
321enum LocalsForNode {
322    /// In the usual case, a `HirId` for an identifier maps to at most
323    /// one `Local` declaration.
324    One(Local),
325
326    /// The exceptional case is identifiers in a match arm's pattern
327    /// that are referenced in a guard of that match arm. For these,
328    /// we have `2` Locals.
329    ///
330    /// * `for_arm_body` is the Local used in the arm body (which is
331    ///   just like the `One` case above),
332    ///
333    /// * `ref_for_guard` is the Local used in the arm's guard (which
334    ///   is a reference to a temp that is an alias of
335    ///   `for_arm_body`).
336    ForGuard { ref_for_guard: Local, for_arm_body: Local },
337}
338
339#[derive(#[automatically_derived]impl ::core::fmt::Debug for GuardFrameLocal {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field1_finish(f,
            "GuardFrameLocal id", &&self.id)
    }
}Debug)]
340struct GuardFrameLocal {
341    id: LocalVarId,
342}
343
344impl GuardFrameLocal {
345    fn new(id: LocalVarId) -> Self {
346        GuardFrameLocal { id }
347    }
348}
349
350#[derive(#[automatically_derived]impl ::core::fmt::Debug for GuardFrame {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field1_finish(f,
            "GuardFrame locals", &&self.locals)
    }
}Debug)]
351struct GuardFrame {
352    /// These are the id's of names that are bound by patterns of the
353    /// arm of *this* guard.
354    ///
355    /// (Frames higher up the stack will have the id's bound in arms
356    /// further out, such as in a case like:
357    ///
358    /// match E1 {
359    ///      P1(id1) if (... (match E2 { P2(id2) if ... => B2 })) => B1,
360    /// }
361    ///
362    /// here, when building for FIXME.
363    locals: Vec<GuardFrameLocal>,
364}
365
366/// `ForGuard` indicates whether we are talking about:
367///   1. The variable for use outside of guard expressions, or
368///   2. The temp that holds reference to (1.), which is actually what the
369///      guard expressions see.
370#[derive(#[automatically_derived]impl ::core::marker::Copy for ForGuard { }Copy, #[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for ForGuard {
}
#[automatically_derived]impl ::core::clone::Clone for ForGuard {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self { *self }
}Clone, #[automatically_derived]impl ::core::fmt::Debug for ForGuard {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                ForGuard::RefWithinGuard => "RefWithinGuard",
                ForGuard::OutsideGuard => "OutsideGuard",
            })
    }
}Debug, #[automatically_derived]impl ::core::marker::StructuralPartialEq for ForGuard
    {
}
#[automatically_derived]impl ::core::cmp::PartialEq for ForGuard {
    #[inline]
    fn eq(&self, other: &Self) -> bool {
        ::core::intrinsics::discriminant_value(self) ==
            ::core::intrinsics::discriminant_value(other)
    }
}PartialEq, #[automatically_derived]impl ::core::cmp::Eq for ForGuard { }Eq)]
371enum ForGuard {
372    RefWithinGuard,
373    OutsideGuard,
374}
375
376impl LocalsForNode {
377    fn local_id(&self, for_guard: ForGuard) -> Local {
378        match (self, for_guard) {
379            (&LocalsForNode::One(local_id), ForGuard::OutsideGuard)
380            | (
381                &LocalsForNode::ForGuard { ref_for_guard: local_id, .. },
382                ForGuard::RefWithinGuard,
383            )
384            | (&LocalsForNode::ForGuard { for_arm_body: local_id, .. }, ForGuard::OutsideGuard) => {
385                local_id
386            }
387
388            (&LocalsForNode::One(_), ForGuard::RefWithinGuard) => {
389                ::rustc_span::macros::bug_impl(None,
    format_args!("anything with one local should never be within a guard."),
    Location::caller())bug!("anything with one local should never be within a guard.")
390            }
391        }
392    }
393}
394
395#[derive(#[automatically_derived]impl<'tcx> ::core::fmt::Debug for
    UnterminatedBlockData<'tcx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field2_finish(f,
            "UnterminatedBlockData statements is_cleanup", &self.statements,
            &&self.is_cleanup)
    }
}Debug, #[automatically_derived]impl<'tcx> ::core::clone::Clone for
    UnterminatedBlockData<'tcx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        UnterminatedBlockData {
            statements: ::core::clone::Clone::clone(&self.statements),
            is_cleanup: ::core::clone::Clone::clone(&self.is_cleanup),
        }
    }
}Clone)]
396pub(crate) struct UnterminatedBlockData<'tcx> {
397    statements: Vec<Statement<'tcx>>,
398    is_cleanup: bool,
399}
400impl<'tcx> UnterminatedBlockData<'tcx> {
401    fn new(statements: Vec<Statement<'tcx>>, is_cleanup: bool) -> Self {
402        UnterminatedBlockData { statements, is_cleanup }
403    }
404}
405
406pub(crate) type CFGBlock<'tcx> = Either<BasicBlockData<'tcx>, UnterminatedBlockData<'tcx>>;
407
408#[cfg(target_pointer_width = "64")]
409mod size_asserts {
410    use rustc_data_structures::static_assert_size;
411
412    use super::*;
413    // Note: It's important that `CFGBlock` has the same layout as `BasicBlockData`.
414    // because it makes the final transition to `IndexVec<BasicBlock, BasicBlockData<'tcx>>`
415    // cheaper. If those type layouts differ, we get a perf regression.
416    // See benchmarks in https://github.com/rust-lang/rust/pull/165611 for more info.
417    const _: [(); 136] = [(); ::std::mem::size_of::<BasicBlockData<'_>>()];static_assert_size!(BasicBlockData<'_>, 136);
418    const _: [(); 136] = [(); ::std::mem::size_of::<CFGBlock<'_>>()];static_assert_size!(CFGBlock<'_>, 136);
419}
420
421#[derive(#[automatically_derived]impl<'tcx> ::core::fmt::Debug for CFG<'tcx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field1_finish(f,
            "CFG basic_blocks", &&self.basic_blocks)
    }
}Debug, #[automatically_derived]impl<'tcx> ::core::clone::Clone for CFG<'tcx> {
    #[inline]
    #[rustc_trivial_field_reads]
    fn clone(&self) -> Self {
        CFG { basic_blocks: ::core::clone::Clone::clone(&self.basic_blocks) }
    }
}Clone)]
422struct CFG<'tcx> {
423    basic_blocks: IndexVec<BasicBlock, CFGBlock<'tcx>>,
424}
425
426#[derive(#[automatically_derived]impl ::core::fmt::Debug for NeedsTemporary {
    #[inline]
    #[rustc_trivial_field_reads]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                NeedsTemporary::No => "No",
                NeedsTemporary::Maybe => "Maybe",
            })
    }
}Debug)]
427enum NeedsTemporary {
428    /// Use this variant when whatever you are converting with `as_operand`
429    /// is the last thing you are converting. This means that if we introduced
430    /// an intermediate temporary, we'd only read it immediately after, so we can
431    /// also avoid it.
432    No,
433    /// For all cases where you aren't sure or that are too expensive to compute
434    /// for now. It is always safe to fall back to this.
435    Maybe,
436}
437
438/// The `BlockAnd` "monad" packages up the new basic block along with a
439/// produced value (sometimes just unit, of course). The `unpack!`
440/// macro (and methods below) makes working with `BlockAnd` much more
441/// convenient.
442#[must_use = "if you don't use one of these results, you're leaving a dangling edge"]
443struct BlockAnd<T>(BasicBlock, T);
444
445impl BlockAnd<()> {
446    /// Unpacks `BlockAnd<()>` into a [`BasicBlock`].
447    #[must_use]
448    fn into_block(self) -> BasicBlock {
449        let Self(block, ()) = self;
450        block
451    }
452}
453
454trait BlockAndExtension {
455    fn and<T>(self, v: T) -> BlockAnd<T>;
456    fn unit(self) -> BlockAnd<()>;
457}
458
459impl BlockAndExtension for BasicBlock {
460    fn and<T>(self, v: T) -> BlockAnd<T> {
461        BlockAnd(self, v)
462    }
463
464    fn unit(self) -> BlockAnd<()> {
465        BlockAnd(self, ())
466    }
467}
468
469/// Update a block pointer and return the value.
470/// Use it like `let x = unpack!(block = self.foo(block, foo))`.
471macro_rules! unpack {
472    ($x:ident = $c:expr) => {{
473        let BlockAnd(b, v) = $c;
474        $x = b;
475        v
476    }};
477}
478
479/// The main entry point for building MIR for a function.
480fn construct_fn<'tcx>(
481    tcx: TyCtxt<'tcx>,
482    fn_def: LocalDefId,
483    thir: &Thir<'tcx>,
484    expr: ExprId,
485    fn_sig: ty::FnSig<'tcx>,
486) -> Body<'tcx> {
487    let span = tcx.def_span(fn_def);
488    let fn_id = tcx.local_def_id_to_hir_id(fn_def);
489
490    // Figure out what primary body this item has.
491    let body = tcx.hir_body_owned_by(fn_def);
492    let span_with_body = tcx.hir_span_with_body(fn_id);
493    let return_ty_span = tcx
494        .hir_fn_decl_by_hir_id(fn_id)
495        .unwrap_or_else(|| ::rustc_span::macros::bug_impl(Some(span),
    format_args!("can\'t build MIR for {0:?}", fn_def), Location::caller())span_bug!(span, "can't build MIR for {:?}", fn_def))
496        .output
497        .span();
498
499    let mut abi = fn_sig.abi();
500    if let DefKind::Closure = tcx.def_kind(fn_def) {
501        // HACK(eddyb) Avoid having RustCall on closures,
502        // as it adds unnecessary (and wrong) auto-tupling.
503        abi = ExternAbi::Rust;
504    }
505
506    let arguments = &thir.params;
507
508    let return_ty = fn_sig.output();
509
510    if let Some((dialect, phase)) =
511        {
    {
        'done:
            {
            for i in ::rustc_attr_ir::HasAttrs::get_attrs(fn_id, &tcx) {
                #[allow(unused_imports)]
                use ::rustc_attr_ir::AttributeKind::*;
                let i: &::rustc_attr_ir::Attribute = i;
                match i {
                    ::rustc_attr_ir::Attribute::Parsed(CustomMir(dialect,
                        phase)) => {
                        break 'done Some((dialect, phase));
                    }
                    ::rustc_attr_ir::Attribute::Tool(..) |
                        ::rustc_attr_ir::Attribute::DeriveHelper(..) =>
                        {}
                        #[deny(unreachable_patterns)]
                        _ => {}
                }
            }
            None
        }
    }
}find_attr!(tcx, fn_id, CustomMir(dialect, phase) => (dialect, phase))
512    {
513        return custom::build_custom_mir(
514            tcx,
515            fn_def.to_def_id(),
516            fn_id,
517            thir,
518            expr,
519            arguments,
520            return_ty,
521            return_ty_span,
522            span_with_body,
523            dialect.as_ref().map(|(d, _)| *d),
524            phase.as_ref().map(|(p, _)| *p),
525        );
526    }
527
528    let typing_mode = if tcx.use_typing_mode_post_typeck_through_borrowck() {
529        TypingMode::borrowck(tcx, fn_def)
530    } else {
531        // FIXME(#132279): This should be able to reveal opaque
532        // types defined during HIR typeck.
533        TypingMode::non_body_analysis()
534    };
535
536    let infcx = tcx.infer_ctxt().build(typing_mode);
537
538    let defining_ty = tcx.type_of(fn_def).instantiate_identity().skip_normalization();
539    let defining_ty = if infcx.next_trait_solver() {
540        // Closure types come from HIR typeck results, where they were already
541        // normalized during writeback. Wrapping them in an `EarlyBinder`
542        // conservatively makes aliases non-rigid, so restore their rigidness
543        // instead of normalizing them again during MIR build.
544        ty::set_aliases_to_rigid(tcx, defining_ty)
545    } else {
546        defining_ty
547    };
548    let coroutine = match defining_ty.kind() {
549        ty::Coroutine(_, args) => Some(Box::new(CoroutineInfo::initial(
550            tcx.coroutine_kind(fn_def).unwrap(),
551            args.as_coroutine().yield_ty(),
552            args.as_coroutine().resume_ty(),
553        ))),
554        ty::Closure(..) | ty::CoroutineClosure(..) | ty::FnDef(..) => None,
555        ty => ::rustc_span::macros::bug_impl(Some(span_with_body),
    format_args!("unexpected type of body: {0:?}", ty), Location::caller())span_bug!(span_with_body, "unexpected type of body: {ty:?}"),
556    };
557
558    let mut builder = Builder::new(
559        thir,
560        infcx,
561        fn_def,
562        fn_id,
563        span_with_body,
564        arguments.len(),
565        return_ty,
566        return_ty_span,
567        coroutine,
568    );
569
570    let call_site_scope =
571        region::Scope { local_id: body.id().hir_id.local_id, data: region::ScopeData::CallSite };
572    let arg_scope =
573        region::Scope { local_id: body.id().hir_id.local_id, data: region::ScopeData::Arguments };
574    let source_info = builder.source_info(span);
575    let call_site_s = (call_site_scope, source_info);
576    let _: BlockAnd<()> = builder.in_scope(call_site_s, LintLevel::Inherited, |builder| {
577        let arg_scope_s = (arg_scope, source_info);
578        // Attribute epilogue to function's closing brace
579        let fn_end = span_with_body.shrink_to_hi();
580        let return_block = builder
581            .in_breakable_scope(None, Place::return_place(), fn_end, |builder| {
582                Some(builder.in_scope(arg_scope_s, LintLevel::Inherited, |builder| {
583                    builder.args_and_body(START_BLOCK, arguments, arg_scope, expr)
584                }))
585            })
586            .into_block();
587        let source_info = builder.source_info(fn_end);
588        builder.push_coverage_point_for_fn_end(return_block, source_info, fn_id);
589        builder.cfg.terminate(return_block, source_info, TerminatorKind::Return);
590        builder.build_drop_trees();
591        return_block.unit()
592    });
593
594    let mut body = builder.finish();
595
596    body.spread_arg = if abi == ExternAbi::RustCall {
597        // RustCall pseudo-ABI untuples the last argument.
598        // FIXME(splat): splat can untuple any argument, set spread_arg here
599        Some(Local::new(arguments.len()))
600    } else {
601        None
602    };
603
604    body
605}
606
607fn construct_const<'a, 'tcx>(
608    tcx: TyCtxt<'tcx>,
609    def: LocalDefId,
610    thir: &'a Thir<'tcx>,
611    expr: ExprId,
612    const_ty: Ty<'tcx>,
613) -> Body<'tcx> {
614    let hir_id = tcx.local_def_id_to_hir_id(def);
615
616    // Figure out what primary body this item has.
617    let (span, const_ty_span) = match tcx.hir_node(hir_id) {
618        Node::Item(hir::Item {
619            kind: hir::ItemKind::Static(_, _, ty, _) | hir::ItemKind::Const(_, _, ty, _),
620            span,
621            ..
622        })
623        | Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Const(ty, _), span, .. })
624        | Node::TraitItem(hir::TraitItem {
625            kind: hir::TraitItemKind::Const(ty, Some(_)),
626            span,
627            ..
628        }) => (*span, ty.span),
629        Node::AnonConst(ct) => (ct.span, ct.span),
630        Node::ConstBlock(_) => {
631            let span = tcx.def_span(def);
632            (span, span)
633        }
634        Node::Item(hir::Item { kind: hir::ItemKind::GlobalAsm { .. }, span, .. }) => (*span, *span),
635        _ => ::rustc_span::macros::bug_impl(Some(tcx.def_span(def)),
    format_args!("can\'t build MIR for {0:?}", def), Location::caller())span_bug!(tcx.def_span(def), "can't build MIR for {:?}", def),
636    };
637
638    let typing_mode = if tcx.use_typing_mode_post_typeck_through_borrowck() {
639        TypingMode::borrowck(tcx, def)
640    } else {
641        // FIXME(#132279): This should be able to reveal opaque
642        // types defined during HIR typeck.
643        TypingMode::non_body_analysis()
644    };
645
646    let infcx = tcx.infer_ctxt().build(typing_mode);
647    let mut builder =
648        Builder::new(thir, infcx, def, hir_id, span, 0, const_ty, const_ty_span, None);
649
650    let mut block = START_BLOCK;
651    block = builder.expr_into_dest(Place::return_place(), block, expr).into_block();
652
653    let source_info = builder.source_info(span);
654    builder.cfg.terminate(block, source_info, TerminatorKind::Return);
655
656    builder.build_drop_trees();
657    builder.finish()
658}
659
660/// Construct MIR for an item that has had errors in type checking.
661///
662/// This is required because we may still want to run MIR passes on an item
663/// with type errors, but normal MIR construction can't handle that in general.
664fn construct_error(tcx: TyCtxt<'_>, def_id: LocalDefId, guar: ErrorGuaranteed) -> Body<'_> {
665    let span = tcx.def_span(def_id);
666    let hir_id = tcx.local_def_id_to_hir_id(def_id);
667
668    let (inputs, output, coroutine) = match tcx.def_kind(def_id) {
669        DefKind::Const
670        | DefKind::AssocConst
671        | DefKind::AnonConst
672        | DefKind::Static { .. }
673        | DefKind::GlobalAsm => {
674            (::alloc::vec::Vec::new()vec![], tcx.type_of(def_id).instantiate_identity().skip_norm_wip(), None)
675        }
676        DefKind::Ctor(..) | DefKind::Fn | DefKind::AssocFn => {
677            let sig = tcx.liberate_late_bound_regions(
678                def_id.to_def_id(),
679                tcx.fn_sig(def_id).instantiate_identity().skip_norm_wip(),
680            );
681            (sig.inputs().to_vec(), sig.output(), None)
682        }
683        DefKind::Closure => {
684            let closure_ty = tcx.type_of(def_id).instantiate_identity().skip_normalization();
685            let closure_ty = if tcx.next_trait_solver_globally() {
686                // Like `construct_fn`, closure tys were already normalized during writeback and
687                // wrapping them in an `EarlyBinder` makes them non-rigid, so we just need to
688                // restore their rigidness again.
689                ty::set_aliases_to_rigid(tcx, closure_ty)
690            } else {
691                closure_ty
692            };
693            match closure_ty.kind() {
694                ty::Closure(_, args) => {
695                    let args = args.as_closure();
696                    let sig = tcx.liberate_late_bound_regions(def_id.to_def_id(), args.sig());
697                    let self_ty = match args.kind() {
698                        ty::ClosureKind::Fn => {
699                            Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, closure_ty)
700                        }
701                        ty::ClosureKind::FnMut => {
702                            Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, closure_ty)
703                        }
704                        ty::ClosureKind::FnOnce => closure_ty,
705                    };
706                    (
707                        [self_ty].into_iter().chain(sig.inputs()[0].tuple_fields()).collect(),
708                        sig.output(),
709                        None,
710                    )
711                }
712                ty::Coroutine(_, args) => {
713                    let args = args.as_coroutine();
714                    let resume_ty = args.resume_ty();
715                    let yield_ty = args.yield_ty();
716                    let return_ty = args.return_ty();
717                    (
718                        ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [closure_ty, resume_ty]))vec![closure_ty, resume_ty],
719                        return_ty,
720                        Some(Box::new(CoroutineInfo::initial(
721                            tcx.coroutine_kind(def_id).unwrap(),
722                            yield_ty,
723                            resume_ty,
724                        ))),
725                    )
726                }
727                ty::CoroutineClosure(did, args) => {
728                    let args = args.as_coroutine_closure();
729                    let sig = tcx.liberate_late_bound_regions(
730                        def_id.to_def_id(),
731                        args.coroutine_closure_sig(),
732                    );
733                    let self_ty = match args.kind() {
734                        ty::ClosureKind::Fn => {
735                            Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, closure_ty)
736                        }
737                        ty::ClosureKind::FnMut => {
738                            Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, closure_ty)
739                        }
740                        ty::ClosureKind::FnOnce => closure_ty,
741                    };
742                    (
743                        [self_ty].into_iter().chain(sig.tupled_inputs_ty.tuple_fields()).collect(),
744                        sig.to_coroutine(
745                            tcx,
746                            args.parent_args(),
747                            args.kind_ty(),
748                            tcx.coroutine_for_closure(*did),
749                            Ty::new_error(tcx, guar),
750                        ),
751                        None,
752                    )
753                }
754                ty::Error(_) => (::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [closure_ty, closure_ty]))vec![closure_ty, closure_ty], closure_ty, None),
755                kind => {
756                    ::rustc_span::macros::bug_impl(Some(span),
    format_args!("expected type of closure body to be a closure or coroutine, got {0:?}",
        kind), Location::caller());span_bug!(
757                        span,
758                        "expected type of closure body to be a closure or coroutine, got {kind:?}"
759                    );
760                }
761            }
762        }
763        dk => ::rustc_span::macros::bug_impl(Some(span),
    format_args!("{0:?} is not a body: {1:?}", def_id, dk),
    Location::caller())span_bug!(span, "{:?} is not a body: {:?}", def_id, dk),
764    };
765
766    let source_info = SourceInfo { span, scope: OUTERMOST_SOURCE_SCOPE };
767    let local_decls = IndexVec::from_iter(
768        [output].iter().chain(&inputs).map(|ty| LocalDecl::with_source_info(*ty, source_info)),
769    );
770    let mut cfg = CFG { basic_blocks: IndexVec::new() };
771    let mut source_scopes = IndexVec::new();
772
773    cfg.start_new_block();
774    source_scopes.push(SourceScopeData {
775        span,
776        parent_scope: None,
777        inlined: None,
778        inlined_parent_scope: None,
779        local_data: ClearCrossCrate::Set(SourceScopeLocalData { lint_root: hir_id }),
780    });
781
782    cfg.terminate(START_BLOCK, source_info, TerminatorKind::Unreachable);
783
784    Body::new(
785        MirSource::item(def_id.to_def_id()),
786        cfg.into_basic_blocks(),
787        source_scopes,
788        local_decls,
789        IndexVec::new(),
790        inputs.len(),
791        ::alloc::vec::Vec::new()vec![],
792        span,
793        coroutine,
794        Some(guar),
795    )
796}
797
798/// A Const::Unevaluated/mir::UnevaluatedConst to a `gca!` const refers to synthesized MIR that
799/// simply returns a Const::Ty of itself. Evaluation of the Const::Ty then calls typesystem
800/// normalization, which eventually calls const_of_item (among other checks) which is the "actual"
801/// value of the const. It then performs a valtree to const val conversion.
802///
803/// This function produces that synthesized MIR.
804fn mir_for_direct_const<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Body<'tcx> {
805    let span = tcx.def_span(def_id);
806    let hir_id = tcx.local_def_id_to_hir_id(def_id);
807    let source_scope = SourceScopeData {
808        span,
809        parent_scope: None,
810        inlined: None,
811        inlined_parent_scope: None,
812        local_data: ClearCrossCrate::Set(SourceScopeLocalData { lint_root: hir_id }),
813    };
814    let source_scopes = IndexVec::from_elem_n(source_scope, 1);
815    let output = tcx.type_of(def_id).instantiate_identity().skip_norm_wip();
816    let source_info = SourceInfo { span, scope: OUTERMOST_SOURCE_SCOPE };
817    let local_decls = IndexVec::from_elem_n(LocalDecl::with_source_info(output, source_info), 1);
818    let user_type_annotations = IndexVec::new();
819    let arg_count = 0;
820    let var_debug_info = ::alloc::vec::Vec::new()vec![];
821    let coroutine = None;
822    let tainted_by_errors = None;
823
824    // It's possible to fetch const_of_item(def_id) here, and put that in the mir::Const::Ty.
825    // However, doing so would skip things like the ConstArgHasType check on the result of
826    // const_of_item that the trait solver does. So, instead, put an AliasConst to ourselves in the
827    // mir::Const::Ty, so that we don't need to duplicate those checks here, and we have a more
828    // maintainable consistent entry point to typesystem norm const eval.
829    let alias_const = match tcx.def_kind(def_id) {
830        DefKind::Const => {
831            let args = ty::GenericArgs::identity_for_item(tcx, def_id);
832            let kind = ty::AliasConstKind::Free { def_id: def_id.into() };
833            ty::AliasConst::new(tcx, kind, args)
834        }
835        DefKind::AssocConst => match tcx.associated_item(def_id).container {
836            ty::AssocContainer::Trait => {
837                let args = ty::GenericArgs::identity_for_item(tcx, def_id);
838                let kind = ty::AliasConstKind::Projection { def_id: def_id.into() };
839                ty::AliasConst::new(tcx, kind, args)
840            }
841            ty::AssocContainer::TraitImpl(Ok(trait_item_def_id)) => {
842                // HACK(khyperia): The trait solver does not support Projection aliases to impl item
843                // def_ids. So, instead, unresolve the impl back to the trait item's def_id, and
844                // construct a Self arg that will resolve us back to the same impl.
845                //
846                // (Adding support to the trait solver for this would not be overly difficult, it's
847                // merely that this is the lesser hack and is more maintainable)
848                let impl_args = ty::GenericArgs::identity_for_item(tcx, def_id);
849                let parent = tcx.parent(def_id.into());
850                let trait_ref = tcx.impl_trait_ref(parent).instantiate_identity().skip_norm_wip();
851                let args = impl_args.rebase_onto(tcx, parent, trait_ref.args);
852                let kind = ty::AliasConstKind::Projection { def_id: trait_item_def_id };
853                ty::AliasConst::new(tcx, kind, args)
854            }
855            ty::AssocContainer::TraitImpl(Err(guar)) => return construct_error(tcx, def_id, guar),
856            ty::AssocContainer::InherentImpl => {
857                let args = ty::GenericArgs::identity_for_item(tcx, def_id);
858                let kind = ty::AliasConstKind::InherentImpl { def_id: def_id.into() };
859                ty::AliasConst::new(tcx, kind, args)
860            }
861        },
862        def_kind => ::rustc_span::macros::bug_impl(Some(tcx.def_span(def_id)),
    format_args!("unexpected def_kind {0:?}", def_kind), Location::caller())span_bug!(tcx.def_span(def_id), "unexpected def_kind {def_kind:?}"),
863    };
864    let ty_const = ty::Const::new_alias(tcx, ty::IsRigid::No, alias_const);
865    let const_ = Const::Ty(output, ty_const);
866    let value = ConstOperand { span, user_ty: None, const_ };
867    let rvalue = Rvalue::Use(Operand::Constant(Box::new(value)), WithRetag::Yes);
868    let statement_kind = StatementKind::Assign(Box::new((Place::return_place(), rvalue)));
869    let statement = Statement::new(source_info, statement_kind);
870    let terminator = Terminator { source_info, kind: TerminatorKind::Return };
871    let block = BasicBlockData::new_stmts(::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [statement]))vec![statement], terminator, false);
872    let basic_blocks = IndexVec::from_elem_n(block, 1);
873
874    Body::new(
875        MirSource::item(def_id.to_def_id()),
876        basic_blocks,
877        source_scopes,
878        local_decls,
879        user_type_annotations,
880        arg_count,
881        var_debug_info,
882        span,
883        coroutine,
884        tainted_by_errors,
885    )
886}
887
888impl<'a, 'tcx> Builder<'a, 'tcx> {
889    fn new(
890        thir: &'a Thir<'tcx>,
891        infcx: InferCtxt<'tcx>,
892        def: LocalDefId,
893        hir_id: HirId,
894        span: Span,
895        arg_count: usize,
896        return_ty: Ty<'tcx>,
897        return_span: Span,
898        coroutine: Option<Box<CoroutineInfo<'tcx>>>,
899    ) -> Builder<'a, 'tcx> {
900        let tcx = infcx.tcx;
901        // Some functions always have overflow checks enabled,
902        // however, they may not get codegen'd, depending on
903        // the settings for the crate they are codegened in.
904        let mut check_overflow = {
    {
            'done:
                {
                for i in tcx.hir_attrs(hir_id) {
                    #[allow(unused_imports)]
                    use ::rustc_attr_ir::AttributeKind::*;
                    let i: &::rustc_attr_ir::Attribute = i;
                    match i {
                        ::rustc_attr_ir::Attribute::Parsed(RustcInheritOverflowChecks)
                            => {
                            break 'done Some(());
                        }
                        ::rustc_attr_ir::Attribute::Tool(..) |
                            ::rustc_attr_ir::Attribute::DeriveHelper(..) =>
                            {}
                            #[deny(unreachable_patterns)]
                            _ => {}
                    }
                }
                None
            }
        }.is_some()
}find_attr!(tcx.hir_attrs(hir_id), RustcInheritOverflowChecks);
905        // Respect -C overflow-checks.
906        check_overflow |= tcx.sess.overflow_checks();
907        // Constants always need overflow checks.
908        check_overflow |= #[allow(non_exhaustive_omitted_patterns)] match tcx.hir_body_owner_kind(def) {
    hir::BodyOwnerKind::Const { .. } | hir::BodyOwnerKind::Static(_) => true,
    _ => false,
}matches!(
909            tcx.hir_body_owner_kind(def),
910            hir::BodyOwnerKind::Const { .. } | hir::BodyOwnerKind::Static(_)
911        );
912
913        let lint_level = LintLevel::Explicit(hir_id);
914        let param_env = tcx.param_env(def);
915        let mut builder = Builder {
916            thir,
917            tcx,
918            infcx,
919            region_scope_tree: tcx.region_scope_tree(def),
920            param_env,
921            def_id: def,
922            hir_id,
923            check_overflow,
924            cfg: CFG { basic_blocks: IndexVec::new() },
925            fn_span: span,
926            arg_count,
927            coroutine,
928            scopes: scope::Scopes::new(),
929            block_context: BlockContext::new(),
930            source_scopes: IndexVec::new(),
931            source_scope: OUTERMOST_SOURCE_SCOPE,
932            guard_context: ::alloc::vec::Vec::new()vec![],
933            fixed_temps: Default::default(),
934            fixed_temps_scope: None,
935            local_decls: IndexVec::from_elem_n(LocalDecl::new(return_ty, return_span), 1),
936            canonical_user_type_annotations: IndexVec::new(),
937            upvars: CaptureMap::new(),
938            var_indices: Default::default(),
939            unit_temp: None,
940            var_debug_info: ::alloc::vec::Vec::new()vec![],
941            lint_level_roots_cache: GrowableBitSet::new_empty(),
942            coverage_info: coverageinfo::CoverageInfoBuilder::new_if_enabled(tcx, def),
943        };
944
945        {
    match (&builder.cfg.start_new_block(), &START_BLOCK) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(builder.cfg.start_new_block(), START_BLOCK);
946        {
    match (&builder.new_source_scope(span, lint_level),
            &OUTERMOST_SOURCE_SCOPE) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(builder.new_source_scope(span, lint_level), OUTERMOST_SOURCE_SCOPE);
947        builder.source_scopes[OUTERMOST_SOURCE_SCOPE].parent_scope = None;
948
949        builder
950    }
951
952    #[allow(dead_code)]
953    fn dump_for_debugging(&self) {
954        let mut body = Body::new(
955            MirSource::item(self.def_id.to_def_id()),
956            self.cfg.clone().into_basic_blocks(),
957            self.source_scopes.clone(),
958            self.local_decls.clone(),
959            self.canonical_user_type_annotations.clone(),
960            self.arg_count.clone(),
961            self.var_debug_info.clone(),
962            self.fn_span.clone(),
963            self.coroutine.clone(),
964            None,
965        );
966        body.coverage_early_info = self.coverage_info.as_ref().map(|b| b.as_done());
967
968        let writer = pretty::MirWriter::new(self.tcx);
969        writer.write_mir_fn(&body, &mut std::io::stdout()).unwrap();
970    }
971
972    fn finish(self) -> Body<'tcx> {
973        let mut body = Body::new(
974            MirSource::item(self.def_id.to_def_id()),
975            self.cfg.into_basic_blocks(),
976            self.source_scopes,
977            self.local_decls,
978            self.canonical_user_type_annotations,
979            self.arg_count,
980            self.var_debug_info,
981            self.fn_span,
982            self.coroutine,
983            None,
984        );
985        body.coverage_early_info = self.coverage_info.map(|b| b.into_done());
986
987        let writer = pretty::MirWriter::new(self.tcx);
988        for (index, block) in body.basic_blocks.iter().enumerate() {
989            if block.terminator.is_none() {
990                writer.write_mir_fn(&body, &mut std::io::stdout()).unwrap();
991                ::rustc_span::macros::bug_impl(Some(self.fn_span),
    format_args!("no terminator on block {0:?}", index), Location::caller());span_bug!(self.fn_span, "no terminator on block {:?}", index);
992            }
993        }
994
995        body
996    }
997
998    fn insert_upvar_arg(&mut self) {
999        let Some(closure_arg) = self.local_decls.get(ty::CAPTURE_STRUCT_LOCAL) else { return };
1000
1001        let mut closure_ty = closure_arg.ty;
1002        let mut closure_env_projs = ::alloc::vec::Vec::new()vec![];
1003        if let ty::Ref(_, ty, _) = closure_ty.kind() {
1004            closure_env_projs.push(ProjectionElem::Deref);
1005            closure_ty = *ty;
1006        }
1007
1008        let upvar_args = match closure_ty.kind() {
1009            ty::Closure(_, args) => ty::UpvarArgs::Closure(*args),
1010            ty::Coroutine(_, args) => ty::UpvarArgs::Coroutine(*args),
1011            ty::CoroutineClosure(_, args) => ty::UpvarArgs::CoroutineClosure(*args),
1012            _ => return,
1013        };
1014
1015        // In analyze_closure() in upvar.rs we gathered a list of upvars used by an
1016        // indexed closure and we stored in a map called closure_min_captures in TypeckResults
1017        // with the closure's DefId. Here, we run through that vec of UpvarIds for
1018        // the given closure and use the necessary information to create upvar
1019        // debuginfo and to fill `self.upvars`.
1020        let capture_tys = upvar_args.upvar_tys();
1021
1022        let tcx = self.tcx;
1023        let mut upvar_owner = None;
1024        self.upvars = tcx
1025            .closure_captures(self.def_id)
1026            .iter()
1027            .zip_eq(capture_tys)
1028            .enumerate()
1029            .map(|(i, (captured_place, ty))| {
1030                let name = captured_place.to_symbol();
1031
1032                let capture = captured_place.info.capture_kind;
1033                let var_id = match captured_place.place.base {
1034                    HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
1035                    _ => ::rustc_span::macros::bug_impl(None, format_args!("Expected an upvar"),
    Location::caller())bug!("Expected an upvar"),
1036                };
1037                let upvar_base = upvar_owner.get_or_insert(var_id.owner);
1038                {
    match (&*upvar_base, &var_id.owner) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(*upvar_base, var_id.owner);
1039                let var_id = var_id.local_id;
1040
1041                let mutability = captured_place.mutability;
1042
1043                let mut projs = closure_env_projs.clone();
1044                projs.push(ProjectionElem::Field(FieldIdx::new(i), ty));
1045                match capture {
1046                    ty::UpvarCapture::ByValue | ty::UpvarCapture::ByUse => {}
1047                    ty::UpvarCapture::ByRef(..) => {
1048                        projs.push(ProjectionElem::Deref);
1049                    }
1050                };
1051
1052                let use_place = Place {
1053                    local: ty::CAPTURE_STRUCT_LOCAL,
1054                    projection: tcx.mk_place_elems(&projs),
1055                };
1056                self.var_debug_info.push(VarDebugInfo {
1057                    name,
1058                    source_info: SourceInfo::outermost(captured_place.var_ident.span),
1059                    value: VarDebugInfoContents::Place(use_place),
1060                    composite: None,
1061                    argument_index: None,
1062                });
1063
1064                let capture = Capture { captured_place, use_place, mutability };
1065                (var_id, capture)
1066            })
1067            .collect();
1068    }
1069
1070    fn args_and_body(
1071        &mut self,
1072        mut block: BasicBlock,
1073        arguments: &IndexSlice<ParamId, Param<'tcx>>,
1074        argument_scope: region::Scope,
1075        expr_id: ExprId,
1076    ) -> BlockAnd<()> {
1077        let expr_span = self.thir[expr_id].span;
1078        // Allocate locals for the function arguments
1079        for (argument_index, param) in arguments.iter().enumerate() {
1080            let source_info =
1081                SourceInfo::outermost(param.pat.as_ref().map_or(self.fn_span, |pat| pat.span));
1082            let arg_local =
1083                self.local_decls.push(LocalDecl::with_source_info(param.ty, source_info));
1084
1085            // If this is a simple binding pattern, give debuginfo a nice name.
1086            if let Some(ref pat) = param.pat
1087                && let Some(name) = pat.simple_ident()
1088            {
1089                self.var_debug_info.push(VarDebugInfo {
1090                    name,
1091                    source_info,
1092                    value: VarDebugInfoContents::Place(arg_local.into()),
1093                    composite: None,
1094                    argument_index: Some(argument_index as u16 + 1),
1095                });
1096            }
1097        }
1098
1099        self.insert_upvar_arg();
1100
1101        let mut scope = None;
1102        // Bind the argument patterns
1103        for (index, param) in arguments.iter().enumerate() {
1104            // Function arguments always get the first Local indices after the return place
1105            let local = Local::arg(index);
1106            let place = Place::from(local);
1107
1108            // Make sure we drop (parts of) the argument even when not matched on.
1109            self.schedule_drop_value(
1110                param.pat.as_ref().map_or(expr_span, |pat| pat.span),
1111                argument_scope,
1112                local,
1113            );
1114
1115            let Some(ref pat) = param.pat else {
1116                continue;
1117            };
1118            let original_source_scope = self.source_scope;
1119            let span = pat.span;
1120            if let Some(arg_hir_id) = param.hir_id {
1121                self.set_correct_source_scope_for_arg(arg_hir_id, original_source_scope, span);
1122            }
1123            match pat.kind {
1124                // Don't introduce extra copies for simple bindings
1125                PatKind::Binding {
1126                    var,
1127                    mode: BindingMode(ByRef::No, mutability),
1128                    subpattern: None,
1129                    ..
1130                } => {
1131                    self.local_decls[local].mutability = mutability;
1132                    self.local_decls[local].source_info.scope = self.source_scope;
1133                    **self.local_decls[local].local_info.as_mut().unwrap_crate_local() =
1134                        if let Some(kind) = param.self_kind {
1135                            LocalInfo::User(BindingForm::ImplicitSelf(kind))
1136                        } else {
1137                            let binding_mode = BindingMode(ByRef::No, mutability);
1138                            LocalInfo::User(BindingForm::Var(VarBindingForm {
1139                                binding_mode,
1140                                opt_ty_info: param.ty_span,
1141                                opt_match_place: Some((None, span)),
1142                                pat_span: span,
1143                                introductions: ::alloc::boxed::box_assume_init_into_vec_unsafe(::alloc::intrinsics::write_box_via_move(::alloc::boxed::Box::new_uninit(),
        [VarBindingIntroduction { span, is_shorthand: false }]))vec![VarBindingIntroduction {
1144                                    span,
1145                                    is_shorthand: false,
1146                                }],
1147                            }))
1148                        };
1149                    self.var_indices.insert(var, LocalsForNode::One(local));
1150                }
1151                _ => {
1152                    scope = self.declare_bindings(
1153                        scope,
1154                        expr_span,
1155                        &pat,
1156                        None,
1157                        Some((Some(&place), span)),
1158                    );
1159                    let place_builder = PlaceBuilder::from(local);
1160                    block = self.place_into_pattern(block, pat, place_builder, false).into_block();
1161                }
1162            }
1163            self.source_scope = original_source_scope;
1164        }
1165
1166        // Enter the argument pattern bindings source scope, if it exists.
1167        if let Some(source_scope) = scope {
1168            self.source_scope = source_scope;
1169        }
1170
1171        if self.tcx.intrinsic(self.def_id).is_some_and(|i| i.must_be_overridden)
1172            || self.tcx.is_sdylib_interface_build()
1173        {
1174            let source_info = self.source_info(rustc_span::DUMMY_SP);
1175            self.cfg.terminate(block, source_info, TerminatorKind::Unreachable);
1176            self.cfg.start_new_block().unit()
1177        } else {
1178            // Ensure we don't silently codegen functions with fake bodies.
1179            match self.tcx.hir_node(self.hir_id) {
1180                hir::Node::Item(hir::Item {
1181                    kind: hir::ItemKind::Fn { has_body: false, .. },
1182                    ..
1183                }) => {
1184                    self.tcx.dcx().span_delayed_bug(
1185                        expr_span,
1186                        ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("fn item without body has reached MIR building: {0:?}",
                self.def_id))
    })format!("fn item without body has reached MIR building: {:?}", self.def_id),
1187                    );
1188                }
1189                _ => {}
1190            }
1191            self.expr_into_dest(Place::return_place(), block, expr_id)
1192        }
1193    }
1194
1195    fn set_correct_source_scope_for_arg(
1196        &mut self,
1197        arg_hir_id: HirId,
1198        original_source_scope: SourceScope,
1199        pattern_span: Span,
1200    ) {
1201        let parent_id = self.source_scopes[original_source_scope]
1202            .local_data
1203            .as_ref()
1204            .unwrap_crate_local()
1205            .lint_root;
1206        self.maybe_new_source_scope(pattern_span, arg_hir_id, parent_id);
1207    }
1208
1209    fn get_unit_temp(&mut self) -> Place<'tcx> {
1210        match self.unit_temp {
1211            Some(tmp) => tmp,
1212            None => {
1213                let ty = self.tcx.types.unit;
1214                let fn_span = self.fn_span;
1215                let tmp = self.temp(ty, fn_span);
1216                self.unit_temp = Some(tmp);
1217                tmp
1218            }
1219        }
1220    }
1221}
1222
1223fn parse_float_into_constval(num: Symbol, float_ty: ty::FloatTy, neg: bool) -> Option<ConstValue> {
1224    parse_float_into_scalar(num, float_ty, neg).map(|s| ConstValue::Scalar(s.into()))
1225}
1226
1227pub(crate) fn parse_float_into_scalar(
1228    num: Symbol,
1229    float_ty: ty::FloatTy,
1230    neg: bool,
1231) -> Option<ScalarInt> {
1232    let num = num.as_str();
1233    match float_ty {
1234        // FIXME(f16_f128): When available, compare to the library parser as with `f32` and `f64`
1235        ty::FloatTy::F16 => {
1236            let mut f = num.parse::<Half>().ok()?;
1237            if neg {
1238                f = -f;
1239            }
1240            Some(ScalarInt::from(f))
1241        }
1242        ty::FloatTy::F32 => {
1243            let Ok(rust_f) = num.parse::<f32>() else { return None };
1244            let mut f = num
1245                .parse::<Single>()
1246                .unwrap_or_else(|e| {
    ::core::panicking::panic_fmt(format_args!("apfloat::ieee::Single failed to parse `{0}`: {1:?}",
            num, e));
}panic!("apfloat::ieee::Single failed to parse `{num}`: {e:?}"));
1247
1248            if !(u128::from(rust_f.to_bits()) == f.to_bits()) {
    {
        ::core::panicking::panic_fmt(format_args!("apfloat::ieee::Single gave different result for `{0}`: {1}({2:#x}) vs Rust\'s {3}({4:#x})",
                rust_f, f, f.to_bits(),
                Single::from_bits(rust_f.to_bits().into()),
                rust_f.to_bits()));
    }
};assert!(
1249                u128::from(rust_f.to_bits()) == f.to_bits(),
1250                "apfloat::ieee::Single gave different result for `{}`: \
1251                 {}({:#x}) vs Rust's {}({:#x})",
1252                rust_f,
1253                f,
1254                f.to_bits(),
1255                Single::from_bits(rust_f.to_bits().into()),
1256                rust_f.to_bits()
1257            );
1258
1259            if neg {
1260                f = -f;
1261            }
1262
1263            Some(ScalarInt::from(f))
1264        }
1265        ty::FloatTy::F64 => {
1266            let Ok(rust_f) = num.parse::<f64>() else { return None };
1267            let mut f = num
1268                .parse::<Double>()
1269                .unwrap_or_else(|e| {
    ::core::panicking::panic_fmt(format_args!("apfloat::ieee::Double failed to parse `{0}`: {1:?}",
            num, e));
}panic!("apfloat::ieee::Double failed to parse `{num}`: {e:?}"));
1270
1271            if !(u128::from(rust_f.to_bits()) == f.to_bits()) {
    {
        ::core::panicking::panic_fmt(format_args!("apfloat::ieee::Double gave different result for `{0}`: {1}({2:#x}) vs Rust\'s {3}({4:#x})",
                rust_f, f, f.to_bits(),
                Double::from_bits(rust_f.to_bits().into()),
                rust_f.to_bits()));
    }
};assert!(
1272                u128::from(rust_f.to_bits()) == f.to_bits(),
1273                "apfloat::ieee::Double gave different result for `{}`: \
1274                 {}({:#x}) vs Rust's {}({:#x})",
1275                rust_f,
1276                f,
1277                f.to_bits(),
1278                Double::from_bits(rust_f.to_bits().into()),
1279                rust_f.to_bits()
1280            );
1281
1282            if neg {
1283                f = -f;
1284            }
1285
1286            Some(ScalarInt::from(f))
1287        }
1288        // FIXME(f16_f128): When available, compare to the library parser as with `f32` and `f64`
1289        ty::FloatTy::F128 => {
1290            let mut f = num.parse::<Quad>().ok()?;
1291            if neg {
1292                f = -f;
1293            }
1294            Some(ScalarInt::from(f))
1295        }
1296    }
1297}
1298
1299///////////////////////////////////////////////////////////////////////////
1300// Builder methods are broken up into modules, depending on what kind
1301// of thing is being lowered. Note that they use the `unpack` macro
1302// above extensively.
1303
1304mod block;
1305mod cfg;
1306mod coverageinfo;
1307mod custom;
1308mod expr;
1309mod matches;
1310mod misc;
1311mod scope;