rustc_mir_build/thir/cx/

expr.rs

use itertools::Itertools;
use rustc_abi::{FIRST_VARIANT, FieldIdx, Size, VariantIdx};
use rustc_ast::UnsafeBinderCastKind;
use rustc_attr_ir::lang_items::LangItem;
use rustc_attr_ir::{AttributeKind, HasAttrs, UnrollAttr, find_attr};
use rustc_data_structures::thin_vec::ThinVec;
use rustc_hir as hir;
use rustc_hir::HirId;
use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
use rustc_index::Idx;
use rustc_middle::hir::place::{
    Place as HirPlace, PlaceBase as HirPlaceBase, ProjectionKind as HirProjectionKind,
};
use rustc_middle::middle::region;
use rustc_middle::mir::{self, AssignOp, BinOp, BorrowKind, UnOp};
use rustc_middle::thir::*;
use rustc_middle::ty::adjustment::{
    Adjust, Adjustment, AutoBorrow, AutoBorrowMutability, DerefAdjustKind, PointerCoercion,
};
use rustc_middle::ty::{
    self, AdtKind, GenericArgs, InlineConstArgs, InlineConstArgsParts, ScalarInt, SplattedDef, Ty,
    TyCtxt, UpvarArgs,
};
use rustc_span::{DesugaringKind, Span, bug, span_bug};
use tracing::{debug, info, instrument, trace};

use crate::diagnostics::*;
use crate::thir::cx::ThirBuildCx;

/// The receiver of a splatted method, or the expression for a splatted function call.
#[derive(Copy, Clone, Debug)]
enum SplattedFunc<'tcx> {
    /// The expression for a method receiver. Always a FnDef.
    FnDefReceiver(&'tcx hir::Expr<'tcx>),
    /// The expression or path for a function call.
    /// This can be a FnDef or FnPtr.
    FnExpression(&'tcx hir::Expr<'tcx>),
}

impl<'tcx> SplattedFunc<'tcx> {
    fn has_receiver(&self) -> bool {
        matches!(self, SplattedFunc::FnDefReceiver(_))
    }

    fn receiver(&self) -> Option<&'tcx hir::Expr<'tcx>> {
        match self {
            SplattedFunc::FnDefReceiver(receiver) => Some(receiver),
            SplattedFunc::FnExpression(_fn_expression) => None,
        }
    }

    fn fn_expression(&self) -> Option<&'tcx hir::Expr<'tcx>> {
        match self {
            SplattedFunc::FnDefReceiver(_receiver) => None,
            SplattedFunc::FnExpression(fn_expression) => Some(fn_expression),
        }
    }
}

fn filter_loop_hint_attrs(id: HirId, tcx: TyCtxt<'_>) -> ThinVec<UnrollAttr> {
    HasAttrs::get_attrs(id, &tcx)
        .into_iter()
        .filter_map(|attr| match attr {
            rustc_attr_ir::Attribute::Parsed(AttributeKind::Unroll(unroll)) => Some(unroll.clone()),
            _ => None,
        })
        .collect()
}

impl<'tcx> ThirBuildCx<'tcx> {
    pub(crate) fn mirror_exprs(&mut self, exprs: &'tcx [hir::Expr<'tcx>]) -> Box<[ExprId]> {
        exprs.iter().map(|expr| self.mirror_expr(expr)).collect()
    }

    /// Create a THIR expression for the given HIR expression. This expands all
    /// adjustments and directly adds the type information from the
    /// `typeck_results`. See the [dev-guide] for more details.
    ///
    /// (The term "mirror" in this case does not refer to "flipped" or
    /// "reversed".)
    ///
    /// [dev-guide]: https://rustc-dev-guide.rust-lang.org/thir.html
    #[instrument(level = "trace", skip(self, hir_expr))]
    pub(crate) fn mirror_expr(&mut self, hir_expr: &'tcx hir::Expr<'tcx>) -> ExprId {
        let expr_scope =
            region::Scope { local_id: hir_expr.hir_id.local_id, data: region::ScopeData::Node };

        trace!(?hir_expr.hir_id, ?hir_expr.span);

        let mut expr = self.make_mirror_unadjusted(hir_expr);

        trace!(?expr.ty);

        let mut loop_hint_attrs = ThinVec::new();

        if let hir::ExprKind::Loop(_, _, _, span) = hir_expr.kind {
            match span.desugaring_kind() {
                // `for` loop desugaring puts us pretty deep down the HIR tree
                Some(DesugaringKind::ForLoop) => {
                    let arm = self.tcx.parent_hir_node(hir_expr.hir_id).expect_arm();
                    let expr = self.tcx.parent_hir_node(arm.hir_id).expect_expr();
                    std::assert_matches!(expr.kind, hir::ExprKind::Match(..));
                    // ignore async for loops
                    if let hir::Node::Expr(expr) = self.tcx.parent_hir_node(expr.hir_id) {
                        std::assert_matches!(expr.kind, hir::ExprKind::DropTemps(..));
                        loop_hint_attrs = filter_loop_hint_attrs(expr.hir_id, self.tcx)
                    }
                }
                // For loops defined with `loop` and `while`, the expr already has the attrs
                Some(DesugaringKind::WhileLoop) | None => {
                    loop_hint_attrs = filter_loop_hint_attrs(hir_expr.hir_id, self.tcx);
                }
                _ => (),
            }
        }

        // Now apply adjustments, if any.
        if self.apply_adjustments {
            for adjustment in self.typeck_results.expr_adjustments(hir_expr) {
                trace!(?expr, ?adjustment);
                let span = expr.span;
                expr = self.apply_adjustment(hir_expr, expr, adjustment, span);
            }
        }

        trace!(?expr.ty, "after adjustments");

        let ty = expr.ty;
        let value = self.thir.exprs.push(expr);

        if !loop_hint_attrs.is_empty() {
            self.thir.loop_hint_attrs.insert(value, loop_hint_attrs);
        }

        // Finally, wrap this up in the expr's scope.
        //
        // (In addition to marking scope, coverage instrumentation also uses this node
        // to help mark the point in MIR where an expression is about to be evaluated.)
        expr = Expr {
            temp_scope_id: expr_scope.local_id,
            ty,
            span: hir_expr.span,
            kind: ExprKind::Scope { region_scope: expr_scope, value, hir_id: hir_expr.hir_id },
        };

        // OK, all done!
        self.thir.exprs.push(expr)
    }

    #[instrument(level = "trace", skip(self, expr, span))]
    fn apply_adjustment(
        &mut self,
        hir_expr: &'tcx hir::Expr<'tcx>,
        mut expr: Expr<'tcx>,
        adjustment: &Adjustment<'tcx>,
        mut span: Span,
    ) -> Expr<'tcx> {
        let Expr { temp_scope_id, .. } = expr;

        // Adjust the span from the block, to the last expression of the
        // block. This is a better span when returning a mutable reference
        // with too short a lifetime. The error message will use the span
        // from the assignment to the return place, which should only point
        // at the returned value, not the entire function body.
        //
        // fn return_short_lived<'a>(x: &'a mut i32) -> &'static mut i32 {
        //      x
        //   // ^ error message points at this expression.
        // }
        let mut adjust_span = |expr: &mut Expr<'tcx>| {
            if let ExprKind::Block { block } = expr.kind
                && let Some(last_expr) = self.thir[block].expr
            {
                span = self.thir[last_expr].span;
                expr.span = span;
            }
        };

        let kind = match adjustment.kind {
            Adjust::Pointer(cast) => {
                if cast == PointerCoercion::Unsize {
                    adjust_span(&mut expr);
                }

                let is_from_as_cast = if let hir::Node::Expr(hir::Expr {
                    kind: hir::ExprKind::Cast(..),
                    span: cast_span,
                    ..
                }) = self.tcx.parent_hir_node(hir_expr.hir_id)
                {
                    // Use the whole span of the `x as T` expression for the coercion.
                    span = *cast_span;
                    true
                } else {
                    false
                };
                ExprKind::PointerCoercion {
                    cast,
                    source: self.thir.exprs.push(expr),
                    is_from_as_cast,
                }
            }
            Adjust::NeverToAny if adjustment.target.is_never() => return expr,
            Adjust::NeverToAny => ExprKind::NeverToAny { source: self.thir.exprs.push(expr) },
            Adjust::Deref(DerefAdjustKind::Builtin) => {
                adjust_span(&mut expr);
                ExprKind::Deref { arg: self.thir.exprs.push(expr) }
            }
            Adjust::Deref(DerefAdjustKind::Pin) => {
                adjust_span(&mut expr);
                // pointer = ($expr).pointer
                let pin_ty = expr.ty.pinned_ty().expect("Deref(Pin) with non-Pin type");
                let pointer_target = ExprKind::Field {
                    lhs: self.thir.exprs.push(expr),
                    variant_index: FIRST_VARIANT,
                    name: FieldIdx::ZERO,
                };
                let expr = Expr { temp_scope_id, ty: pin_ty, span, kind: pointer_target };
                // expr = *pointer
                ExprKind::Deref { arg: self.thir.exprs.push(expr) }
            }
            Adjust::Deref(DerefAdjustKind::Overloaded(deref)) => {
                // We don't need to do call adjust_span here since
                // deref coercions always start with a built-in deref.
                let call_def_id = deref.method_call(self.tcx);
                let overloaded_callee = self
                    .tcx
                    .type_of(call_def_id)
                    .instantiate(self.tcx, self.tcx.mk_args(&[expr.ty.into()]))
                    .skip_norm_wip();

                expr = Expr {
                    temp_scope_id,
                    ty: Ty::new_ref(self.tcx, self.tcx.lifetimes.re_erased, expr.ty, deref.mutbl),
                    span,
                    kind: ExprKind::Borrow {
                        borrow_kind: deref.mutbl.to_borrow_kind(),
                        arg: self.thir.exprs.push(expr),
                    },
                };

                let expr = Box::new([self.thir.exprs.push(expr)]);

                self.overloaded_place(
                    hir_expr,
                    adjustment.target,
                    Some(overloaded_callee),
                    expr,
                    deref.span,
                )
            }
            Adjust::Borrow(AutoBorrow::Ref(m)) => ExprKind::Borrow {
                borrow_kind: m.to_borrow_kind(),
                arg: self.thir.exprs.push(expr),
            },
            Adjust::Borrow(AutoBorrow::RawPtr(mutability)) => {
                ExprKind::RawBorrow { mutability, arg: self.thir.exprs.push(expr) }
            }
            Adjust::Borrow(AutoBorrow::Pin(mutbl)) => {
                // expr = &pin (mut|const|) arget
                let borrow_kind = match mutbl {
                    hir::Mutability::Mut => BorrowKind::Mut { kind: mir::MutBorrowKind::Default },
                    hir::Mutability::Not => BorrowKind::Shared,
                };
                let new_pin_target =
                    Ty::new_ref(self.tcx, self.tcx.lifetimes.re_erased, expr.ty, mutbl);
                let arg = self.thir.exprs.push(expr);
                let expr = self.thir.exprs.push(Expr {
                    temp_scope_id,
                    ty: new_pin_target,
                    span,
                    kind: ExprKind::Borrow { borrow_kind, arg },
                });

                // kind = Pin { pointer }
                let pin_did = self.tcx.require_lang_item(LangItem::Pin, span);
                let args = self.tcx.mk_args(&[new_pin_target.into()]);
                let kind = ExprKind::Adt(Box::new(AdtExpr {
                    adt_def: self.tcx.adt_def(pin_did),
                    variant_index: FIRST_VARIANT,
                    args,
                    fields: Box::new([FieldExpr { name: FieldIdx::ZERO, expr }]),
                    user_ty: None,
                    base: AdtExprBase::None,
                }));

                debug!(?kind);
                kind
            }
            Adjust::GenericReborrow(mutability) => {
                let expr = self.thir.exprs.push(expr);
                let kind =
                    ExprKind::Reborrow { source: expr, mutability, target: adjustment.target };

                kind
            }
        };

        Expr { temp_scope_id, ty: adjustment.target, span, kind }
    }

    /// Lowers a cast expression.
    ///
    /// Dealing with user type annotations is left to the caller.
    fn mirror_expr_cast(
        &mut self,
        source: &'tcx hir::Expr<'tcx>,
        temp_scope_id: hir::ItemLocalId,
        span: Span,
    ) -> ExprKind<'tcx> {
        let tcx = self.tcx;

        // Check to see if this cast is a "coercion cast", where the cast is actually done
        // using a coercion (or is a no-op).
        if self.typeck_results.is_coercion_cast(source.hir_id) {
            // Convert the lexpr to a vexpr.
            ExprKind::ValueExpr { source: self.mirror_expr(source) }
        } else if self.typeck_results.expr_ty(source).is_ref() {
            // Special cased so that we can type check that the element
            // type of the source matches the pointed to type of the
            // destination.
            ExprKind::PointerCoercion {
                source: self.mirror_expr(source),
                cast: PointerCoercion::ArrayToPointer,
                is_from_as_cast: true,
            }
        } else if let hir::ExprKind::Path(ref qpath) = source.kind
            && let res = self.typeck_results.qpath_res(qpath, source.hir_id)
            && let ty = self.typeck_results.node_type(source.hir_id)
            && let ty::Adt(adt_def, _) = ty.kind()
            && let Res::Def(DefKind::Ctor(CtorOf::Variant, CtorKind::Const), variant_ctor_id) = res
        {
            // Check whether this is casting an enum variant discriminant.
            // To prevent cycles, we refer to the discriminant initializer,
            // which is always an integer and thus doesn't need to know the
            // enum's layout (or its tag type) to compute it during const eval.
            // Example:
            // enum Foo {
            //     A,
            //     B = A as isize + 4,
            // }
            // The correct solution would be to add symbolic computations to miri,
            // so we wouldn't have to compute and store the actual value

            let idx = adt_def.variant_index_with_ctor_id(variant_ctor_id);
            let (discr_did, discr_offset) = adt_def.discriminant_def_for_variant(idx);

            use rustc_middle::ty::util::IntTypeExt;
            let ty = adt_def.repr().discr_type();
            let discr_ty = ty.to_ty(tcx);

            let size = tcx
                .layout_of(self.typing_env.as_query_input(discr_ty))
                .unwrap_or_else(|e| panic!("could not compute layout for {discr_ty:?}: {e:?}"))
                .size;

            let (lit, overflowing) = ScalarInt::truncate_from_uint(discr_offset as u128, size);
            if overflowing {
                // An erroneous enum with too many variants for its repr will emit E0081 and E0370
                self.tcx.dcx().span_delayed_bug(
                    source.span,
                    "overflowing enum wasn't rejected by hir analysis",
                );
            }
            let kind = ExprKind::NonHirLiteral { lit, user_ty: None };
            let offset = self.thir.exprs.push(Expr { temp_scope_id, ty: discr_ty, span, kind });

            let source = match discr_did {
                // in case we are offsetting from a computed discriminant
                // and not the beginning of discriminants (which is always `0`)
                Some(did) => {
                    let args = self.tcx.mk_args(&[]);
                    self.tcx.debug_assert_args_compatible(did, args);
                    let kind = ExprKind::NamedConst { def_id: did, args, user_ty: None };
                    let lhs =
                        self.thir.exprs.push(Expr { temp_scope_id, ty: discr_ty, span, kind });
                    let bin = ExprKind::Binary { op: BinOp::Add, lhs, rhs: offset };
                    self.thir.exprs.push(Expr { temp_scope_id, ty: discr_ty, span, kind: bin })
                }
                None => offset,
            };

            ExprKind::Cast { source }
        } else {
            // Default to `ExprKind::Cast` for all explicit casts.
            // MIR building then picks the right MIR casts based on the types.
            ExprKind::Cast { source: self.mirror_expr(source) }
        }
    }

    #[instrument(level = "debug", skip(self), ret)]
    fn make_mirror_unadjusted(&mut self, expr: &'tcx hir::Expr<'tcx>) -> Expr<'tcx> {
        let tcx = self.tcx;
        let expr_ty = self.typeck_results.expr_ty(expr);
        let mk_expr =
            |kind, ty| Expr { temp_scope_id: expr.hir_id.local_id, span: expr.span, ty, kind };

        let kind = match expr.kind {
            // Here comes the interesting stuff:
            hir::ExprKind::MethodCall(segment, receiver, args, fn_span) => {
                if self.typeck_results.is_splatted_call(expr) {
                    // The callee has a splatted tuple argument.
                    // rewrite `receiver.f(a, u, v)` into `receiver.f(a, #[rustc_splat] (u, v))`
                    self.convert_splatted_callee(
                        expr,
                        fn_span,
                        args,
                        SplattedFunc::FnDefReceiver(receiver),
                    )
                } else {
                    // Rewrite a.b(c) into UFCS form like Trait::b(a, c)
                    let expr = self.method_callee(expr, segment.ident.span, None);
                    info!("Using method span: {:?}", expr.span);

                    // Arg position const args are already part of the callee's generic args,
                    // so we skip them when building runtime arguments.
                    let generics = match *expr.ty.kind() {
                        ty::FnDef(def_id, _) => Some(tcx.generics_of(def_id)),
                        _ => None,
                    };
                    let args = if generics.is_some_and(|g| g.has_arg_pos_consts) {
                        let is_const_arg = |idx: usize| {
                            generics.is_some_and(|g| {
                                g.own_arg_pos_consts().any(|(_, pos)| pos as usize == idx)
                            })
                        };

                        std::iter::once(receiver)
                            .chain(args.iter())
                            .enumerate()
                            .filter(|&(idx, _)| !is_const_arg(idx))
                            .map(|(_, arg)| self.mirror_expr(arg))
                            .collect()
                    } else {
                        std::iter::once(receiver)
                            .chain(args.iter())
                            .map(|arg| self.mirror_expr(arg))
                            .collect()
                    };

                    ExprKind::Call {
                        ty: expr.ty,
                        fun: self.thir.exprs.push(expr),
                        args,
                        from_hir_call: true,
                        fn_span,
                    }
                }
            }

            hir::ExprKind::Call(fun, ref args) => {
                if self.typeck_results.is_method_call(expr) {
                    // The callee is something implementing Fn, FnMut, or FnOnce.
                    // Find the actual method implementation being called and
                    // build the appropriate UFCS call expression with the
                    // callee-object as expr parameter.

                    // rewrite f(u, v) into FnOnce::call_once(f, (u, v))

                    let method = self.method_callee(expr, fun.span, None);

                    let arg_tys = args.iter().map(|e| self.typeck_results.expr_ty_adjusted(e));
                    let tupled_args = Expr {
                        ty: Ty::new_tup_from_iter(tcx, arg_tys),
                        temp_scope_id: expr.hir_id.local_id,
                        span: expr.span,
                        kind: ExprKind::Tuple { fields: self.mirror_exprs(args) },
                    };
                    let tupled_args = self.thir.exprs.push(tupled_args);

                    ExprKind::Call {
                        ty: method.ty,
                        fun: self.thir.exprs.push(method),
                        args: Box::new([self.mirror_expr(fun), tupled_args]),
                        from_hir_call: true,
                        fn_span: expr.span,
                    }
                } else if self.typeck_results.is_splatted_call(expr) {
                    // The callee has a splatted tuple argument.
                    // rewrite `f(a, u, v)` into `f(a, #[rustc_splat] (u, v))`
                    self.convert_splatted_callee(
                        expr,
                        fun.span,
                        args,
                        SplattedFunc::FnExpression(fun),
                    )
                } else {
                    // Tuple-like ADTs are represented as ExprKind::Call. We convert them here.
                    let adt_data = if let hir::ExprKind::Path(ref qpath) = fun.kind
                        && let Some(adt_def) = expr_ty.ty_adt_def()
                    {
                        match qpath {
                            hir::QPath::Resolved(_, path) => match path.res {
                                Res::Def(DefKind::Ctor(_, CtorKind::Fn), ctor_id) => {
                                    Some((adt_def, adt_def.variant_index_with_ctor_id(ctor_id)))
                                }
                                Res::SelfCtor(..) => Some((adt_def, FIRST_VARIANT)),
                                _ => None,
                            },
                            hir::QPath::TypeRelative(_ty, _) => {
                                if let Some((DefKind::Ctor(_, CtorKind::Fn), ctor_id)) =
                                    self.typeck_results.type_dependent_def(fun.hir_id)
                                {
                                    Some((adt_def, adt_def.variant_index_with_ctor_id(ctor_id)))
                                } else {
                                    None
                                }
                            }
                        }
                    } else {
                        None
                    };
                    if let Some((adt_def, index)) = adt_data {
                        let node_args = self.typeck_results.node_args(fun.hir_id);
                        let user_provided_types = self.typeck_results.user_provided_types();
                        let user_ty =
                            user_provided_types.get(fun.hir_id).copied().map(|mut u_ty| {
                                if let ty::UserTypeKind::TypeOf(did, _) = &mut u_ty.value.kind {
                                    *did = adt_def.did();
                                }
                                Box::new(u_ty)
                            });
                        debug!("make_mirror_unadjusted: (call) user_ty={:?}", user_ty);

                        let field_refs = args
                            .iter()
                            .enumerate()
                            .map(|(idx, e)| FieldExpr {
                                name: FieldIdx::new(idx),
                                expr: self.mirror_expr(e),
                            })
                            .collect();
                        ExprKind::Adt(Box::new(AdtExpr {
                            adt_def,
                            args: node_args,
                            variant_index: index,
                            fields: field_refs,
                            user_ty,
                            base: AdtExprBase::None,
                        }))
                    } else {
                        // Arg position const args are already part of the callee's generic args,
                        // so we skip them when building runtime arguments.
                        let fn_ty = self.typeck_results.node_type(fun.hir_id);
                        let generics = match *fn_ty.kind() {
                            ty::FnDef(def_id, _) => Some(tcx.generics_of(def_id)),
                            _ => None,
                        };
                        let args = if generics.is_some_and(|g| g.has_arg_pos_consts) {
                            let is_const_arg = |idx: usize| {
                                generics.is_some_and(|g| {
                                    g.own_arg_pos_consts().any(|(_, pos)| pos as usize == idx)
                                })
                            };
                            args.iter()
                                .enumerate()
                                .filter(|&(idx, _)| !is_const_arg(idx))
                                .map(|(_, arg)| self.mirror_expr(arg))
                                .collect()
                        } else {
                            args.iter().map(|arg| self.mirror_expr(arg)).collect()
                        };

                        ExprKind::Call {
                            ty: fn_ty,
                            fun: self.mirror_expr(fun),
                            args,
                            from_hir_call: true,
                            fn_span: expr.span,
                        }
                    }
                }
            }

            hir::ExprKind::Use(expr, span) => {
                ExprKind::ByUse { expr: self.mirror_expr(expr), span }
            }

            hir::ExprKind::AddrOf(hir::BorrowKind::Ref, mutbl, arg) => {
                ExprKind::Borrow { borrow_kind: mutbl.to_borrow_kind(), arg: self.mirror_expr(arg) }
            }

            hir::ExprKind::AddrOf(hir::BorrowKind::Raw, mutability, arg) => {
                ExprKind::RawBorrow { mutability, arg: self.mirror_expr(arg) }
            }

            // Make `&pin mut $expr` and `&pin const $expr` into
            // `Pin { __pointer: &mut { $expr } }` and `Pin { __pointer: &$expr }`.
            hir::ExprKind::AddrOf(hir::BorrowKind::Pin, mutbl, arg_expr) => match expr_ty.kind() {
                &ty::Adt(adt_def, args) if tcx.is_lang_item(adt_def.did(), LangItem::Pin) => {
                    let ty = args.type_at(0);
                    let arg_ty = self.typeck_results.expr_ty(arg_expr);
                    let mut arg = self.mirror_expr(arg_expr);
                    // For `&pin mut $place` where `$place` is not `Unpin`, move the place
                    // `$place` to ensure it will not be used afterwards.
                    if mutbl.is_mut() && !arg_ty.is_unpin(self.tcx, self.typing_env) {
                        let block = self.thir.blocks.push(Block {
                            targeted_by_break: false,
                            region_scope: region::Scope {
                                local_id: arg_expr.hir_id.local_id,
                                data: region::ScopeData::Node,
                            },
                            span: arg_expr.span,
                            stmts: Box::new([]),
                            expr: Some(arg),
                            safety_mode: BlockSafety::Safe,
                        });
                        arg = self.thir.exprs.push(Expr {
                            temp_scope_id: arg_expr.hir_id.local_id,
                            ty: arg_ty,
                            span: arg_expr.span,
                            kind: ExprKind::Block { block },
                        });
                    }
                    let expr = self.thir.exprs.push(Expr {
                        temp_scope_id: expr.hir_id.local_id,
                        ty,
                        span: expr.span,
                        kind: ExprKind::Borrow { borrow_kind: mutbl.to_borrow_kind(), arg },
                    });
                    ExprKind::Adt(Box::new(AdtExpr {
                        adt_def,
                        variant_index: FIRST_VARIANT,
                        args,
                        fields: Box::new([FieldExpr { name: FieldIdx::from(0u32), expr }]),
                        user_ty: None,
                        base: AdtExprBase::None,
                    }))
                }
                _ => span_bug!(expr.span, "unexpected type for pinned borrow: {:?}", expr_ty),
            },

            hir::ExprKind::Block(blk, _) => ExprKind::Block { block: self.mirror_block(blk) },

            hir::ExprKind::Assign(lhs, rhs, _) => {
                ExprKind::Assign { lhs: self.mirror_expr(lhs), rhs: self.mirror_expr(rhs) }
            }

            hir::ExprKind::AssignOp(op, lhs, rhs) => {
                if self.typeck_results.is_method_call(expr) {
                    let lhs = self.mirror_expr(lhs);
                    let rhs = self.mirror_expr(rhs);
                    self.overloaded_operator(expr, Box::new([lhs, rhs]))
                } else {
                    ExprKind::AssignOp {
                        op: assign_op(op.node),
                        lhs: self.mirror_expr(lhs),
                        rhs: self.mirror_expr(rhs),
                    }
                }
            }

            hir::ExprKind::Lit(lit) => ExprKind::Literal { lit, neg: false },

            hir::ExprKind::Binary(op, lhs, rhs) => {
                if self.typeck_results.is_method_call(expr) {
                    let lhs = self.mirror_expr(lhs);
                    let rhs = self.mirror_expr(rhs);
                    self.overloaded_operator(expr, Box::new([lhs, rhs]))
                } else {
                    match op.node {
                        hir::BinOpKind::And => ExprKind::LogicalOp {
                            op: LogicalOp::And,
                            lhs: self.mirror_expr(lhs),
                            rhs: self.mirror_expr(rhs),
                        },
                        hir::BinOpKind::Or => ExprKind::LogicalOp {
                            op: LogicalOp::Or,
                            lhs: self.mirror_expr(lhs),
                            rhs: self.mirror_expr(rhs),
                        },
                        _ => {
                            let op = bin_op(op.node);
                            ExprKind::Binary {
                                op,
                                lhs: self.mirror_expr(lhs),
                                rhs: self.mirror_expr(rhs),
                            }
                        }
                    }
                }
            }

            hir::ExprKind::Index(lhs, index, brackets_span) => {
                if self.typeck_results.is_method_call(expr) {
                    let lhs = self.mirror_expr(lhs);
                    let index = self.mirror_expr(index);
                    self.overloaded_place(
                        expr,
                        expr_ty,
                        None,
                        Box::new([lhs, index]),
                        brackets_span,
                    )
                } else {
                    ExprKind::Index { lhs: self.mirror_expr(lhs), index: self.mirror_expr(index) }
                }
            }

            hir::ExprKind::Unary(hir::UnOp::Deref, arg) => {
                if self.typeck_results.is_method_call(expr) {
                    let arg = self.mirror_expr(arg);
                    self.overloaded_place(expr, expr_ty, None, Box::new([arg]), expr.span)
                } else {
                    ExprKind::Deref { arg: self.mirror_expr(arg) }
                }
            }

            hir::ExprKind::Unary(hir::UnOp::Not, arg) => {
                if self.typeck_results.is_method_call(expr) {
                    let arg = self.mirror_expr(arg);
                    self.overloaded_operator(expr, Box::new([arg]))
                } else {
                    ExprKind::Unary { op: UnOp::Not, arg: self.mirror_expr(arg) }
                }
            }

            hir::ExprKind::Unary(hir::UnOp::Neg, arg) => {
                if self.typeck_results.is_method_call(expr) {
                    let arg = self.mirror_expr(arg);
                    self.overloaded_operator(expr, Box::new([arg]))
                } else if let hir::ExprKind::Lit(lit) = arg.kind {
                    ExprKind::Literal { lit, neg: true }
                } else {
                    ExprKind::Unary { op: UnOp::Neg, arg: self.mirror_expr(arg) }
                }
            }

            hir::ExprKind::Struct(qpath, fields, ref base) => match expr_ty.kind() {
                ty::Adt(adt, args) => match adt.adt_kind() {
                    AdtKind::Struct | AdtKind::Union => {
                        let user_provided_types = self.typeck_results.user_provided_types();
                        let user_ty = user_provided_types.get(expr.hir_id).copied().map(Box::new);
                        debug!("make_mirror_unadjusted: (struct/union) user_ty={:?}", user_ty);
                        ExprKind::Adt(Box::new(AdtExpr {
                            adt_def: *adt,
                            variant_index: FIRST_VARIANT,
                            args,
                            user_ty,
                            fields: self.field_refs(fields),
                            base: match base {
                                hir::StructTailExpr::Base(base) => AdtExprBase::Base(FruInfo {
                                    base: self.mirror_expr(base),
                                    field_types: self.typeck_results.fru_field_types()[expr.hir_id]
                                        .iter()
                                        .copied()
                                        .collect(),
                                }),
                                hir::StructTailExpr::DefaultFields(_) => {
                                    AdtExprBase::DefaultFields(
                                        self.typeck_results.fru_field_types()[expr.hir_id]
                                            .iter()
                                            .copied()
                                            .collect(),
                                    )
                                }
                                hir::StructTailExpr::None
                                | hir::StructTailExpr::NoneWithError(_) => AdtExprBase::None,
                            },
                        }))
                    }
                    AdtKind::Enum => {
                        let res = self.typeck_results.qpath_res(qpath, expr.hir_id);
                        match res {
                            Res::Def(DefKind::Variant, variant_id) => {
                                assert!(matches!(
                                    base,
                                    hir::StructTailExpr::None
                                        | hir::StructTailExpr::DefaultFields(_)
                                        | hir::StructTailExpr::NoneWithError(_)
                                ));

                                let index = adt.variant_index_with_id(variant_id);
                                let user_provided_types = self.typeck_results.user_provided_types();
                                let user_ty =
                                    user_provided_types.get(expr.hir_id).copied().map(Box::new);
                                debug!("make_mirror_unadjusted: (variant) user_ty={:?}", user_ty);
                                ExprKind::Adt(Box::new(AdtExpr {
                                    adt_def: *adt,
                                    variant_index: index,
                                    args,
                                    user_ty,
                                    fields: self.field_refs(fields),
                                    base: match base {
                                        hir::StructTailExpr::DefaultFields(_) => {
                                            AdtExprBase::DefaultFields(
                                                self.typeck_results.fru_field_types()[expr.hir_id]
                                                    .iter()
                                                    .copied()
                                                    .collect(),
                                            )
                                        }
                                        hir::StructTailExpr::Base(base) => {
                                            span_bug!(base.span, "unexpected res: {:?}", res);
                                        }
                                        hir::StructTailExpr::None
                                        | hir::StructTailExpr::NoneWithError(_) => {
                                            AdtExprBase::None
                                        }
                                    },
                                }))
                            }
                            _ => {
                                span_bug!(expr.span, "unexpected res: {:?}", res);
                            }
                        }
                    }
                },
                _ => {
                    span_bug!(expr.span, "unexpected type for struct literal: {:?}", expr_ty);
                }
            },

            hir::ExprKind::Closure(hir::Closure { .. }) => {
                let closure_ty = self.typeck_results.expr_ty(expr);
                let (def_id, args, movability) = match *closure_ty.kind() {
                    ty::Closure(def_id, args) => (def_id, UpvarArgs::Closure(args), None),
                    ty::Coroutine(def_id, args) => {
                        (def_id, UpvarArgs::Coroutine(args), Some(tcx.coroutine_movability(def_id)))
                    }
                    ty::CoroutineClosure(def_id, args) => {
                        (def_id, UpvarArgs::CoroutineClosure(args), None)
                    }
                    _ => {
                        span_bug!(expr.span, "closure expr w/o closure type: {:?}", closure_ty);
                    }
                };
                let def_id = def_id.expect_local();

                let upvars = self
                    .tcx
                    .closure_captures(def_id)
                    .iter()
                    .zip_eq(args.upvar_tys())
                    .map(|(captured_place, ty)| {
                        let upvars = self.capture_upvar(expr, captured_place, ty);
                        self.thir.exprs.push(upvars)
                    })
                    .collect();

                // Convert the closure fake reads, if any, from hir `Place` to ExprRef
                let fake_reads = match self.typeck_results.closure_fake_reads.get(&def_id) {
                    Some(fake_reads) => fake_reads
                        .iter()
                        .map(|(place, cause, hir_id)| {
                            let expr = self.convert_captured_hir_place(expr, place.clone());
                            (self.thir.exprs.push(expr), *cause, *hir_id)
                        })
                        .collect(),
                    None => Vec::new(),
                };

                ExprKind::Closure(Box::new(ClosureExpr {
                    closure_id: def_id,
                    args,
                    upvars,
                    movability,
                    fake_reads,
                }))
            }

            hir::ExprKind::Path(ref qpath) => {
                let res = self.typeck_results.qpath_res(qpath, expr.hir_id);
                self.convert_path_expr(expr, res)
            }

            hir::ExprKind::InlineAsm(asm) => ExprKind::InlineAsm(Box::new(InlineAsmExpr {
                asm_macro: asm.asm_macro,
                template: asm.template,
                operands: asm
                    .operands
                    .iter()
                    .map(|(op, _op_sp)| match *op {
                        hir::InlineAsmOperand::In { reg, expr } => InlineAsmOperand::In {
                            reg: reg.as_target(),
                            expr: self.mirror_expr(expr),
                        },
                        hir::InlineAsmOperand::Out { reg, late, ref expr } => {
                            InlineAsmOperand::Out {
                                reg: reg.as_target(),
                                late,
                                expr: expr.map(|expr| self.mirror_expr(expr)),
                            }
                        }
                        hir::InlineAsmOperand::InOut { reg, late, expr } => {
                            InlineAsmOperand::InOut {
                                reg: reg.as_target(),
                                late,
                                expr: self.mirror_expr(expr),
                            }
                        }
                        hir::InlineAsmOperand::SplitInOut { reg, late, in_expr, ref out_expr } => {
                            InlineAsmOperand::SplitInOut {
                                reg: reg.as_target(),
                                late,
                                in_expr: self.mirror_expr(in_expr),
                                out_expr: out_expr.map(|expr| self.mirror_expr(expr)),
                            }
                        }
                        hir::InlineAsmOperand::Const { ref anon_const } => {
                            let ty = self.typeck_results.node_type(anon_const.hir_id);
                            let did = anon_const.def_id;
                            let typeck_root_def_id = tcx.typeck_root_def_id_local(did);
                            let parent_args = tcx.erase_and_anonymize_regions(
                                GenericArgs::identity_for_item(tcx, typeck_root_def_id),
                            );
                            let args =
                                InlineConstArgs::new(tcx, InlineConstArgsParts { parent_args, ty })
                                    .args;

                            let uneval = mir::UnevaluatedConst::new(did.to_def_id(), args);
                            let value = mir::Const::Unevaluated(uneval, ty);
                            InlineAsmOperand::Const { value, span: tcx.def_span(did) }
                        }
                        hir::InlineAsmOperand::SymFn { expr } => {
                            InlineAsmOperand::SymFn { value: self.mirror_expr(expr) }
                        }
                        hir::InlineAsmOperand::SymStatic { path: _, def_id } => {
                            InlineAsmOperand::SymStatic { def_id }
                        }
                        hir::InlineAsmOperand::Label { block } => {
                            InlineAsmOperand::Label { block: self.mirror_block(block) }
                        }
                    })
                    .collect(),
                options: asm.options,
                line_spans: asm.line_spans,
            })),

            hir::ExprKind::OffsetOf(_, _) => {
                let offset_of_intrinsic = tcx.require_lang_item(LangItem::OffsetOf, expr.span);
                let mk_u32_kind = |val: u32| ExprKind::NonHirLiteral {
                    lit: ScalarInt::try_from_uint(val, Size::from_bits(32)).unwrap(),
                    user_ty: None,
                };
                let mk_usize_kind = |val: u64| ExprKind::NonHirLiteral {
                    lit: ScalarInt::try_from_target_usize(val, tcx).unwrap(),
                    user_ty: None,
                };
                let mk_call =
                    |thir: &mut Thir<'tcx>, ty: Ty<'tcx>, variant: VariantIdx, field: FieldIdx| {
                        let fun_ty = tcx
                            .type_of(offset_of_intrinsic)
                            .instantiate(tcx, &[ty.into()])
                            .skip_norm_wip();
                        let fun = thir
                            .exprs
                            .push(mk_expr(ExprKind::ZstLiteral { user_ty: None }, fun_ty));
                        let variant =
                            thir.exprs.push(mk_expr(mk_u32_kind(variant.as_u32()), tcx.types.u32));
                        let field =
                            thir.exprs.push(mk_expr(mk_u32_kind(field.as_u32()), tcx.types.u32));
                        let args = Box::new([variant, field]);
                        ExprKind::Call {
                            ty: fun_ty,
                            fun,
                            args,
                            from_hir_call: false,
                            fn_span: expr.span,
                        }
                    };

                let indices = self.typeck_results.offset_of_data().get(expr.hir_id).unwrap();
                let mut expr = None::<ExprKind<'_>>;

                for &(container, variant, field) in indices.iter() {
                    let next = mk_call(&mut self.thir, container, variant, field);
                    expr = Some(match expr.take() {
                        None => next,
                        Some(last) => {
                            let last = self.thir.exprs.push(mk_expr(last, tcx.types.usize));
                            let next = self.thir.exprs.push(mk_expr(next, tcx.types.usize));
                            ExprKind::Binary { op: BinOp::Add, lhs: last, rhs: next }
                        }
                    });
                }

                expr.unwrap_or_else(|| mk_usize_kind(0))
            }

            hir::ExprKind::ConstBlock(ref anon_const) => {
                let ty = self.typeck_results.node_type(anon_const.hir_id);
                let did = anon_const.def_id;
                let typeck_root_def_id = tcx.typeck_root_def_id_local(did);
                let parent_args = tcx.erase_and_anonymize_regions(GenericArgs::identity_for_item(
                    tcx,
                    typeck_root_def_id,
                ));
                let args = InlineConstArgs::new(tcx, InlineConstArgsParts { parent_args, ty }).args;

                ExprKind::ConstBlock { did: did.to_def_id(), args }
            }
            // Now comes the rote stuff:
            hir::ExprKind::Repeat(v, _) => {
                let ty = self.typeck_results.expr_ty(expr);
                let ty::Array(_, count) = ty.kind() else {
                    span_bug!(expr.span, "unexpected repeat expr ty: {:?}", ty);
                };

                ExprKind::Repeat { value: self.mirror_expr(v), count: *count }
            }
            hir::ExprKind::Ret(v) => ExprKind::Return { value: v.map(|v| self.mirror_expr(v)) },
            hir::ExprKind::Become(call) => ExprKind::Become { value: self.mirror_expr(call) },
            hir::ExprKind::Break(dest, ref value) => {
                if find_attr!(self.tcx, expr.hir_id, ConstContinue(_)) {
                    match dest.target_id {
                        Ok(target_id) => {
                            let (Some(value), Some(_)) = (value, dest.label) else {
                                let span = expr.span;
                                self.tcx.dcx().emit_fatal(ConstContinueMissingLabelOrValue { span })
                            };

                            ExprKind::ConstContinue {
                                label: region::Scope {
                                    local_id: target_id.local_id,
                                    data: region::ScopeData::Node,
                                },
                                value: self.mirror_expr(value),
                            }
                        }
                        Err(err) => bug!("invalid loop id for break: {}", err),
                    }
                } else {
                    match dest.target_id {
                        Ok(target_id) => ExprKind::Break {
                            label: region::Scope {
                                local_id: target_id.local_id,
                                data: region::ScopeData::Node,
                            },
                            value: value.map(|value| self.mirror_expr(value)),
                        },
                        Err(err) => bug!("invalid loop id for break: {}", err),
                    }
                }
            }
            hir::ExprKind::Continue(dest) => match dest.target_id {
                Ok(loop_id) => ExprKind::Continue {
                    label: region::Scope {
                        local_id: loop_id.local_id,
                        data: region::ScopeData::Node,
                    },
                },
                Err(err) => bug!("invalid loop id for continue: {}", err),
            },
            hir::ExprKind::Let(let_expr) => ExprKind::Let {
                expr: self.mirror_expr(let_expr.init),
                pat: self.pattern_from_hir(let_expr.pat),
            },
            hir::ExprKind::If(cond, then, else_opt) => ExprKind::If {
                if_then_scope: region::Scope {
                    local_id: then.hir_id.local_id,
                    data: {
                        if expr.span.at_least_rust_2024() {
                            region::ScopeData::IfThenRescope
                        } else {
                            region::ScopeData::IfThen
                        }
                    },
                },
                cond: self.mirror_expr(cond),
                then: self.mirror_expr(then),
                else_opt: else_opt.map(|el| self.mirror_expr(el)),
            },
            hir::ExprKind::Match(discr, arms, match_source) => ExprKind::Match {
                scrutinee: self.mirror_expr(discr),
                arms: arms.iter().map(|a| self.convert_arm(a)).collect(),
                match_source,
            },
            hir::ExprKind::Loop(body, ..) => {
                if find_attr!(self.tcx, expr.hir_id, LoopMatch(_)) {
                    let dcx = self.tcx.dcx();

                    // Accept either `state = expr` or `state = expr;`.
                    let loop_body_expr = match body.stmts {
                        [] => match body.expr {
                            Some(expr) => expr,
                            None => dcx.emit_fatal(LoopMatchMissingAssignment { span: body.span }),
                        },
                        [single] if body.expr.is_none() => match single.kind {
                            hir::StmtKind::Expr(expr) | hir::StmtKind::Semi(expr) => expr,
                            _ => dcx.emit_fatal(LoopMatchMissingAssignment { span: body.span }),
                        },
                        [first @ last] | [first, .., last] => dcx
                            .emit_fatal(LoopMatchBadStatements { span: first.span.to(last.span) }),
                    };

                    let hir::ExprKind::Assign(state, rhs_expr, _) = loop_body_expr.kind else {
                        dcx.emit_fatal(LoopMatchMissingAssignment { span: loop_body_expr.span })
                    };

                    let hir::ExprKind::Block(block_body, _) = rhs_expr.kind else {
                        dcx.emit_fatal(LoopMatchBadRhs { span: rhs_expr.span })
                    };

                    // The labeled block should contain one match expression, but defining items is
                    // allowed.
                    for stmt in block_body.stmts {
                        if !matches!(stmt.kind, rustc_hir::StmtKind::Item(_)) {
                            dcx.emit_fatal(LoopMatchBadStatements { span: stmt.span })
                        }
                    }

                    let Some(block_body_expr) = block_body.expr else {
                        dcx.emit_fatal(LoopMatchBadRhs { span: block_body.span })
                    };

                    let hir::ExprKind::Match(scrutinee, arms, _match_source) = block_body_expr.kind
                    else {
                        dcx.emit_fatal(LoopMatchBadRhs { span: block_body_expr.span })
                    };

                    fn local(
                        cx: &mut ThirBuildCx<'_>,
                        expr: &rustc_hir::Expr<'_>,
                    ) -> Option<hir::HirId> {
                        if let hir::ExprKind::Path(hir::QPath::Resolved(_, path)) = expr.kind
                            && let Res::Local(hir_id) = path.res
                            && !cx.is_upvar(hir_id)
                        {
                            return Some(hir_id);
                        }

                        None
                    }

                    let Some(scrutinee_hir_id) = local(self, scrutinee) else {
                        dcx.emit_fatal(LoopMatchInvalidMatch { span: scrutinee.span })
                    };

                    if local(self, state) != Some(scrutinee_hir_id) {
                        dcx.emit_fatal(LoopMatchInvalidUpdate {
                            scrutinee: scrutinee.span,
                            lhs: state.span,
                        })
                    }

                    ExprKind::LoopMatch {
                        state: self.mirror_expr(state),
                        region_scope: region::Scope {
                            local_id: block_body.hir_id.local_id,
                            data: region::ScopeData::Node,
                        },

                        match_data: Box::new(LoopMatchMatchData {
                            scrutinee: self.mirror_expr(scrutinee),
                            arms: arms.iter().map(|a| self.convert_arm(a)).collect(),
                            span: block_body_expr.span,
                        }),
                    }
                } else {
                    let block_ty = self.typeck_results.node_type(body.hir_id);
                    let block = self.mirror_block(body);
                    let body = self.thir.exprs.push(Expr {
                        ty: block_ty,
                        temp_scope_id: body.hir_id.local_id,
                        span: self.thir[block].span,
                        kind: ExprKind::Block { block },
                    });
                    ExprKind::Loop { body }
                }
            }
            hir::ExprKind::Field(source, ..) => ExprKind::Field {
                lhs: self.mirror_expr(source),
                variant_index: FIRST_VARIANT,
                name: self.typeck_results.field_index(expr.hir_id),
            },
            hir::ExprKind::Cast(source, cast_ty) => {
                // Check for a user-given type annotation on this `cast`
                let user_provided_types = self.typeck_results.user_provided_types();
                let user_ty = user_provided_types.get(cast_ty.hir_id);

                debug!(
                    "cast({:?}) has ty w/ hir_id {:?} and user provided ty {:?}",
                    expr, cast_ty.hir_id, user_ty,
                );

                let cast = self.mirror_expr_cast(source, expr.hir_id.local_id, expr.span);

                if let Some(user_ty) = user_ty {
                    // NOTE: Creating a new Expr and wrapping a Cast inside of it may be
                    //       inefficient, revisit this when performance becomes an issue.
                    let cast_expr = self.thir.exprs.push(Expr {
                        temp_scope_id: expr.hir_id.local_id,
                        ty: expr_ty,
                        span: expr.span,
                        kind: cast,
                    });
                    debug!("make_mirror_unadjusted: (cast) user_ty={:?}", user_ty);

                    ExprKind::ValueTypeAscription {
                        source: cast_expr,
                        user_ty: Some(Box::new(*user_ty)),
                        user_ty_span: cast_ty.span,
                    }
                } else {
                    cast
                }
            }
            hir::ExprKind::Type(source, ty) => {
                let user_provided_types = self.typeck_results.user_provided_types();
                let user_ty = user_provided_types.get(ty.hir_id).copied().map(Box::new);
                debug!("make_mirror_unadjusted: (type) user_ty={:?}", user_ty);
                let mirrored = self.mirror_expr(source);
                if source.is_syntactic_place_expr() {
                    ExprKind::PlaceTypeAscription {
                        source: mirrored,
                        user_ty,
                        user_ty_span: ty.span,
                    }
                } else {
                    ExprKind::ValueTypeAscription {
                        source: mirrored,
                        user_ty,
                        user_ty_span: ty.span,
                    }
                }
            }

            hir::ExprKind::UnsafeBinderCast(UnsafeBinderCastKind::Unwrap, source, _ty) => {
                // FIXME(unsafe_binders): Take into account the ascribed type, too.
                let mirrored = self.mirror_expr(source);
                if source.is_syntactic_place_expr() {
                    ExprKind::PlaceUnwrapUnsafeBinder { source: mirrored }
                } else {
                    ExprKind::ValueUnwrapUnsafeBinder { source: mirrored }
                }
            }
            hir::ExprKind::UnsafeBinderCast(UnsafeBinderCastKind::Wrap, source, _ty) => {
                // FIXME(unsafe_binders): Take into account the ascribed type, too.
                let mirrored = self.mirror_expr(source);
                ExprKind::WrapUnsafeBinder { source: mirrored }
            }

            hir::ExprKind::DropTemps(source) => {
                ExprKind::ValueExpr { source: self.mirror_expr(source) }
            }
            hir::ExprKind::Array(fields) => ExprKind::Array { fields: self.mirror_exprs(fields) },
            hir::ExprKind::Tup(fields) => ExprKind::Tuple { fields: self.mirror_exprs(fields) },

            hir::ExprKind::Yield(v, _) => ExprKind::Yield { value: self.mirror_expr(v) },
            hir::ExprKind::Err(_) => unreachable!("cannot lower a `hir::ExprKind::Err` to THIR"),
        };

        mk_expr(kind, expr_ty)
    }

    fn user_args_applied_to_res(
        &mut self,
        hir_id: hir::HirId,
        res: Res,
    ) -> Option<Box<ty::CanonicalUserType<'tcx>>> {
        debug!("user_args_applied_to_res: res={:?}", res);
        let user_provided_type = match res {
            // A reference to something callable -- e.g., a fn, method, or
            // a tuple-struct or tuple-variant. This has the type of a
            // `Fn` but with the user-given generic parameters.
            Res::Def(DefKind::Fn, _)
            | Res::Def(DefKind::AssocFn, _)
            | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
            | Res::Def(DefKind::Const, _)
            | Res::Def(DefKind::AssocConst, _) => {
                self.typeck_results.user_provided_types().get(hir_id).copied().map(Box::new)
            }

            // A unit struct/variant which is used as a value (e.g.,
            // `None`). This has the type of the enum/struct that defines
            // this variant -- but with the generic parameters given by the
            // user.
            Res::Def(DefKind::Ctor(_, CtorKind::Const), _) => {
                self.user_args_applied_to_ty_of_hir_id(hir_id).map(Box::new)
            }

            // `Self` is used in expression as a tuple struct constructor or a unit struct constructor
            Res::SelfCtor(_) => self.user_args_applied_to_ty_of_hir_id(hir_id).map(Box::new),

            _ => bug!("user_args_applied_to_res: unexpected res {:?} at {:?}", res, hir_id),
        };
        debug!("user_args_applied_to_res: user_provided_type={:?}", user_provided_type);
        user_provided_type
    }

    fn method_callee(
        &mut self,
        expr: &hir::Expr<'_>,
        span: Span,
        overloaded_callee: Option<Ty<'tcx>>,
    ) -> Expr<'tcx> {
        let (ty, user_ty) = match overloaded_callee {
            Some(fn_def) => (fn_def, None),
            None => {
                let (kind, def_id) =
                    self.typeck_results.type_dependent_def(expr.hir_id).unwrap_or_else(|| {
                        span_bug!(expr.span, "no type-dependent def for method callee")
                    });
                let user_ty = self.user_args_applied_to_res(expr.hir_id, Res::Def(kind, def_id));
                debug!("method_callee: user_ty={:?}", user_ty);
                (
                    self.tcx
                        .type_of(def_id)
                        .instantiate(self.tcx, self.typeck_results.node_args(expr.hir_id))
                        .skip_norm_wip(),
                    user_ty,
                )
            }
        };
        Expr {
            temp_scope_id: expr.hir_id.local_id,
            ty,
            span,
            kind: ExprKind::ZstLiteral { user_ty },
        }
    }

    /// The callee has a splatted tuple argument.
    /// Rewrite a splatted call `receiver.f(a, u, v)` into `receiver.f(a, #[rustc_splat] (u, v))`.
    /// The receiver is optional.
    fn convert_splatted_callee(
        &mut self,
        call_expr: &'tcx hir::Expr<'_>,
        fn_span: Span,
        args: &'tcx [hir::Expr<'tcx>],
        receiver_or_func: SplattedFunc<'tcx>,
    ) -> ExprKind<'tcx> {
        let tcx = self.tcx;

        // Look up the typeck results
        let splatted_def =
            self.typeck_results.splatted_def(call_expr.hir_id).unwrap_or_else(|| {
                span_bug!(call_expr.span, "no splatted def for function or method callee")
            });

        let tupled_arg_index = usize::from(splatted_def.arg_index());
        let tupled_args_count = usize::from(splatted_def.arg_count());

        // Splatting an empty tuple is permitted: `a.f() -> Trait::f(a, #[rustc_splat] ())`.
        // In that case, the tupled arg index is one past the end of the args.
        if tupled_arg_index + tupled_args_count > args.len() {
            span_bug!(
                call_expr.span,
                "splatted arg index out of bounds of function args: {:?} + {:?} > {:?} for function call: {:?}, args {:?}",
                tupled_arg_index,
                tupled_args_count,
                args.len(),
                receiver_or_func,
                args,
            );
        }

        debug!("Using splatted function span: {:?}", fn_span);

        // Split into non-tupled and tupled arguments
        let initial_non_tupled_args =
            args.iter().take(tupled_arg_index).map(|e| self.mirror_expr(e)).collect_vec();
        let tupled_args = if tupled_arg_index == args.len() || tupled_args_count == 0 {
            // Splatting an empty tuple, in the ABI this gets ignored
            Default::default()
        } else {
            &args[tupled_arg_index..(tupled_arg_index + tupled_args_count)]
        };
        let final_non_tupled_args = args
            .iter()
            .skip(tupled_arg_index + tupled_args_count)
            .map(|e| self.mirror_expr(e))
            .collect_vec();

        let tupled_arg_tys = tupled_args.iter().map(|e| self.typeck_results.expr_ty_adjusted(e));

        // We need the tupled arguments in HIR/MIR for type checking
        // FIXME(splat): de-tuple args in codegen for performance
        let tupled_args = Expr {
            ty: Ty::new_tup_from_iter(tcx, tupled_arg_tys),
            temp_scope_id: call_expr.hir_id.local_id,
            span: call_expr.span,
            kind: ExprKind::Tuple { fields: self.mirror_exprs(tupled_args) },
        };

        let tupled_args = self.thir.exprs.push(tupled_args);

        // Handle the receiver as the first arg, if present
        let mut args = Vec::with_capacity(
            usize::from(receiver_or_func.has_receiver())
                + initial_non_tupled_args.len()
                + 1
                + final_non_tupled_args.len(),
        );
        if let Some(receiver) = receiver_or_func.receiver() {
            args.push(self.mirror_expr(receiver));
        }
        args.extend(initial_non_tupled_args);
        args.push(tupled_args);
        args.extend(final_non_tupled_args);

        let fn_span = if receiver_or_func.has_receiver() { fn_span } else { call_expr.span };

        let (fn_ty, fun_expr) = match (splatted_def, receiver_or_func.fn_expression()) {
            // Create a FnDef shim for user-provided types
            (SplattedDef::FnDef { def_id, arg_index, arg_count }, _) => {
                // We're calling a function via a FnDef, and its possibly generic type
                // This is effectively `self.method_callee(call_expr, fn_span, None)`,
                // applied to `splatted_def` instead of `type_dependent_def`.
                let def_kind = self.tcx.def_kind(def_id);
                let user_ty =
                    self.user_args_applied_to_res(call_expr.hir_id, Res::Def(def_kind, def_id));
                debug!(
                    "splatted_callee FnDef: user_ty={:?} def_kind={:?} def_id={:?} arg_index={:?} arg_count={:?}",
                    user_ty, def_kind, def_id, arg_index, arg_count,
                );

                // Create a new FnDef expression with user-provided type applied
                let callee_expr = Expr {
                    temp_scope_id: call_expr.hir_id.local_id,
                    ty: self
                        .tcx
                        .type_of(def_id)
                        .instantiate(self.tcx, self.typeck_results.node_args(call_expr.hir_id))
                        .skip_norm_wip(),
                    span: fn_span,
                    kind: ExprKind::ZstLiteral { user_ty },
                };
                (callee_expr.ty, self.thir.exprs.push(callee_expr))
            }

            // We're calling a function via a FnPtr and its type
            // FIXME(splat): do we need to populate and apply user_provided_types() ?
            (SplattedDef::FnPtr { fn_ptr_type, arg_index, arg_count }, Some(fn_expression)) => {
                debug!(
                    "splatted_callee FnPtr: fn_ty={:?} arg_index={:?} arg_count={:?}",
                    fn_ptr_type, arg_index, arg_count,
                );

                if !fn_ptr_type.is_fn() {
                    span_bug!(
                        call_expr.span,
                        "splatted FnPtr side-tables were not populated correctly, non-fn type received: {:?}",
                        fn_ptr_type
                    )
                }

                // Pass through the FnPtr type and the mirrored function path
                (fn_ptr_type, self.mirror_expr(fn_expression))
            }
            // FnPtrs must have a function expression (and they never have method receivers)
            (SplattedDef::FnPtr { .. }, None) => {
                span_bug!(
                    call_expr.span,
                    "convert_splatted_callee: FnPtr without fn expression (or with receiver) is invalid: splatted_def={:?}, receiver_or_func={:?}",
                    splatted_def,
                    receiver_or_func,
                );
            }
        };

        ExprKind::Call {
            ty: fn_ty,
            fun: fun_expr,
            args: args.into_boxed_slice(),
            from_hir_call: true,
            fn_span,
        }
    }

    fn convert_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) -> ArmId {
        let arm = Arm {
            pattern: self.pattern_from_hir(&arm.pat),
            guard: arm.guard.as_ref().map(|g| self.mirror_expr(g)),
            body: self.mirror_expr(arm.body),
            hir_id: arm.hir_id,
            scope: region::Scope { local_id: arm.hir_id.local_id, data: region::ScopeData::Node },
            span: arm.span,
        };
        self.thir.arms.push(arm)
    }

    fn convert_path_expr(&mut self, expr: &'tcx hir::Expr<'tcx>, res: Res) -> ExprKind<'tcx> {
        let args = self.typeck_results.node_args(expr.hir_id);
        match res {
            // A regular function, constructor function or a constant.
            Res::Def(DefKind::Fn, _)
            | Res::Def(DefKind::AssocFn, _)
            | Res::Def(DefKind::Ctor(_, CtorKind::Fn), _)
            | Res::SelfCtor(_) => {
                let user_ty = self.user_args_applied_to_res(expr.hir_id, res);
                ExprKind::ZstLiteral { user_ty }
            }

            Res::Def(DefKind::ConstParam, def_id) => {
                let hir_id = self.tcx.local_def_id_to_hir_id(def_id.expect_local());
                let generics = self.tcx.generics_of(hir_id.owner);
                let Some(&index) = generics.param_def_id_to_index.get(&def_id) else {
                    span_bug!(
                        expr.span,
                        "Should have already errored about late bound consts: {def_id:?}"
                    );
                };
                let name = self.tcx.hir_name(hir_id);
                let param = ty::ParamConst::new(index, name);

                ExprKind::ConstParam { param, def_id }
            }

            Res::Def(DefKind::Const, def_id) | Res::Def(DefKind::AssocConst, def_id) => {
                let user_ty = self.user_args_applied_to_res(expr.hir_id, res);
                ExprKind::NamedConst { def_id, args, user_ty }
            }

            Res::Def(DefKind::Ctor(_, CtorKind::Const), def_id) => {
                let user_provided_types = self.typeck_results.user_provided_types();
                let user_ty = user_provided_types.get(expr.hir_id).copied().map(Box::new);
                debug!("convert_path_expr: user_ty={:?}", user_ty);
                let ty = self.typeck_results.node_type(expr.hir_id);
                match ty.kind() {
                    // A unit struct/variant which is used as a value.
                    // We return a completely different ExprKind here to account for this special case.
                    ty::Adt(adt_def, args) => ExprKind::Adt(Box::new(AdtExpr {
                        adt_def: *adt_def,
                        variant_index: adt_def.variant_index_with_ctor_id(def_id),
                        args,
                        user_ty,
                        fields: Box::new([]),
                        base: AdtExprBase::None,
                    })),
                    _ => bug!("unexpected ty: {:?}", ty),
                }
            }

            // A source Rust `path::to::STATIC` is a place expr like *&ident is.
            // In THIR, we make them exactly equivalent by inserting the implied *& or *&raw,
            // but distinguish between &STATIC and &THREAD_LOCAL as they have different semantics
            Res::Def(DefKind::Static { .. }, id) => {
                // this is &raw for extern static or static mut, and & for other statics
                let ty = self.tcx.static_ptr_ty(id, self.typing_env);
                let kind = if self.tcx.is_thread_local_static(id) {
                    ExprKind::ThreadLocalRef(id)
                } else {
                    let alloc_id = self.tcx.reserve_and_set_static_alloc(id);
                    ExprKind::StaticRef { alloc_id, ty, def_id: id }
                };
                ExprKind::Deref {
                    arg: self.thir.exprs.push(Expr {
                        ty,
                        temp_scope_id: expr.hir_id.local_id,
                        span: expr.span,
                        kind,
                    }),
                }
            }

            Res::Local(var_hir_id) => self.convert_var(var_hir_id),

            _ => span_bug!(expr.span, "res `{:?}` not yet implemented", res),
        }
    }

    fn convert_var(&mut self, var_hir_id: hir::HirId) -> ExprKind<'tcx> {
        // We want upvars here not captures.
        // Captures will be handled in MIR.
        let is_upvar = self.is_upvar(var_hir_id);

        debug!(
            "convert_var({:?}): is_upvar={}, body_owner={:?}",
            var_hir_id, is_upvar, self.body_owner
        );

        if is_upvar {
            ExprKind::UpvarRef {
                closure_def_id: self.body_owner,
                var_hir_id: LocalVarId(var_hir_id),
            }
        } else {
            ExprKind::VarRef { id: LocalVarId(var_hir_id) }
        }
    }

    fn overloaded_operator(
        &mut self,
        expr: &'tcx hir::Expr<'tcx>,
        args: Box<[ExprId]>,
    ) -> ExprKind<'tcx> {
        let fun = self.method_callee(expr, expr.span, None);
        let fun = self.thir.exprs.push(fun);
        ExprKind::Call {
            ty: self.thir[fun].ty,
            fun,
            args,
            from_hir_call: false,
            fn_span: expr.span,
        }
    }

    fn overloaded_place(
        &mut self,
        expr: &'tcx hir::Expr<'tcx>,
        place_ty: Ty<'tcx>,
        overloaded_callee: Option<Ty<'tcx>>,
        args: Box<[ExprId]>,
        span: Span,
    ) -> ExprKind<'tcx> {
        // For an overloaded *x or x[y] expression of type T, the method
        // call returns an &T and we must add the deref so that the types
        // line up (this is because `*x` and `x[y]` represent places):

        // Reconstruct the output assuming it's a reference with the
        // same region and mutability as the receiver. This holds for
        // `Deref(Mut)::deref(_mut)` and `Index(Mut)::index(_mut)`.
        let ty::Ref(region, _, mutbl) = *self.thir[args[0]].ty.kind() else {
            span_bug!(span, "overloaded_place: receiver is not a reference");
        };
        let ref_ty = Ty::new_ref(self.tcx, region, place_ty, mutbl);

        // construct the complete expression `foo()` for the overloaded call,
        // which will yield the &T type
        let fun = self.method_callee(expr, span, overloaded_callee);
        let fun = self.thir.exprs.push(fun);
        let fun_ty = self.thir[fun].ty;
        let ref_expr = self.thir.exprs.push(Expr {
            temp_scope_id: expr.hir_id.local_id,
            ty: ref_ty,
            span,
            kind: ExprKind::Call { ty: fun_ty, fun, args, from_hir_call: false, fn_span: span },
        });

        // construct and return a deref wrapper `*foo()`
        ExprKind::Deref { arg: ref_expr }
    }

    fn convert_captured_hir_place(
        &mut self,
        closure_expr: &'tcx hir::Expr<'tcx>,
        place: HirPlace<'tcx>,
    ) -> Expr<'tcx> {
        let temp_scope_id = closure_expr.hir_id.local_id;
        let var_ty = place.base_ty;

        // The result of capture analysis in `rustc_hir_typeck/src/upvar.rs` represents a captured path
        // as it's seen for use within the closure and not at the time of closure creation.
        //
        // That is we see expect to see it start from a captured upvar and not something that is local
        // to the closure's parent.
        let var_hir_id = match place.base {
            HirPlaceBase::Upvar(upvar_id) => upvar_id.var_path.hir_id,
            base => bug!("Expected an upvar, found {:?}", base),
        };

        let mut captured_place_expr = Expr {
            temp_scope_id,
            ty: var_ty,
            span: closure_expr.span,
            kind: self.convert_var(var_hir_id),
        };

        for proj in place.projections.iter() {
            let kind = match proj.kind {
                HirProjectionKind::Deref => {
                    ExprKind::Deref { arg: self.thir.exprs.push(captured_place_expr) }
                }
                HirProjectionKind::Field(field, variant_index) => ExprKind::Field {
                    lhs: self.thir.exprs.push(captured_place_expr),
                    variant_index,
                    name: field,
                },
                HirProjectionKind::OpaqueCast => {
                    ExprKind::ValueExpr { source: self.thir.exprs.push(captured_place_expr) }
                }
                HirProjectionKind::UnwrapUnsafeBinder => ExprKind::PlaceUnwrapUnsafeBinder {
                    source: self.thir.exprs.push(captured_place_expr),
                },
                HirProjectionKind::Index | HirProjectionKind::Subslice => {
                    // We don't capture these projections, so we can ignore them here
                    continue;
                }
            };

            captured_place_expr =
                Expr { temp_scope_id, ty: proj.ty, span: closure_expr.span, kind };
        }

        captured_place_expr
    }

    fn capture_upvar(
        &mut self,
        closure_expr: &'tcx hir::Expr<'tcx>,
        captured_place: &'tcx ty::CapturedPlace<'tcx>,
        upvar_ty: Ty<'tcx>,
    ) -> Expr<'tcx> {
        let upvar_capture = captured_place.info.capture_kind;
        let captured_place_expr =
            self.convert_captured_hir_place(closure_expr, captured_place.place.clone());
        let temp_scope_id = closure_expr.hir_id.local_id;

        match upvar_capture {
            ty::UpvarCapture::ByValue => captured_place_expr,
            ty::UpvarCapture::ByUse => {
                let span = captured_place_expr.span;
                let expr_id = self.thir.exprs.push(captured_place_expr);

                Expr {
                    temp_scope_id,
                    ty: upvar_ty,
                    span: closure_expr.span,
                    kind: ExprKind::ByUse { expr: expr_id, span },
                }
            }
            ty::UpvarCapture::ByRef(upvar_borrow) => {
                let borrow_kind = match upvar_borrow {
                    ty::BorrowKind::Immutable => BorrowKind::Shared,
                    ty::BorrowKind::UniqueImmutable => {
                        BorrowKind::Mut { kind: mir::MutBorrowKind::ClosureCapture }
                    }
                    ty::BorrowKind::Mutable => {
                        BorrowKind::Mut { kind: mir::MutBorrowKind::Default }
                    }
                };
                Expr {
                    temp_scope_id,
                    ty: upvar_ty,
                    span: closure_expr.span,
                    kind: ExprKind::Borrow {
                        borrow_kind,
                        arg: self.thir.exprs.push(captured_place_expr),
                    },
                }
            }
        }
    }

    fn is_upvar(&mut self, var_hir_id: hir::HirId) -> bool {
        self.tcx
            .upvars_mentioned(self.body_owner)
            .is_some_and(|upvars| upvars.contains_key(&var_hir_id))
    }

    /// Converts a list of named fields (i.e., for struct-like struct/enum ADTs) into FieldExpr.
    fn field_refs(&mut self, fields: &'tcx [hir::ExprField<'tcx>]) -> Box<[FieldExpr]> {
        fields
            .iter()
            .map(|field| FieldExpr {
                name: self.typeck_results.field_index(field.hir_id),
                expr: self.mirror_expr(field.expr),
            })
            .collect()
    }
}

trait ToBorrowKind {
    fn to_borrow_kind(&self) -> BorrowKind;
}

impl ToBorrowKind for AutoBorrowMutability {
    fn to_borrow_kind(&self) -> BorrowKind {
        use rustc_middle::ty::adjustment::AllowTwoPhase;
        match *self {
            AutoBorrowMutability::Mut { allow_two_phase_borrow } => BorrowKind::Mut {
                kind: match allow_two_phase_borrow {
                    AllowTwoPhase::Yes => mir::MutBorrowKind::TwoPhaseBorrow,
                    AllowTwoPhase::No => mir::MutBorrowKind::Default,
                },
            },
            AutoBorrowMutability::Not => BorrowKind::Shared,
        }
    }
}

impl ToBorrowKind for hir::Mutability {
    fn to_borrow_kind(&self) -> BorrowKind {
        match *self {
            hir::Mutability::Mut => BorrowKind::Mut { kind: mir::MutBorrowKind::Default },
            hir::Mutability::Not => BorrowKind::Shared,
        }
    }
}

fn bin_op(op: hir::BinOpKind) -> BinOp {
    match op {
        hir::BinOpKind::Add => BinOp::Add,
        hir::BinOpKind::Sub => BinOp::Sub,
        hir::BinOpKind::Mul => BinOp::Mul,
        hir::BinOpKind::Div => BinOp::Div,
        hir::BinOpKind::Rem => BinOp::Rem,
        hir::BinOpKind::BitXor => BinOp::BitXor,
        hir::BinOpKind::BitAnd => BinOp::BitAnd,
        hir::BinOpKind::BitOr => BinOp::BitOr,
        hir::BinOpKind::Shl => BinOp::Shl,
        hir::BinOpKind::Shr => BinOp::Shr,
        hir::BinOpKind::Eq => BinOp::Eq,
        hir::BinOpKind::Lt => BinOp::Lt,
        hir::BinOpKind::Le => BinOp::Le,
        hir::BinOpKind::Ne => BinOp::Ne,
        hir::BinOpKind::Ge => BinOp::Ge,
        hir::BinOpKind::Gt => BinOp::Gt,
        _ => bug!("no equivalent for ast binop {:?}", op),
    }
}

fn assign_op(op: hir::AssignOpKind) -> AssignOp {
    match op {
        hir::AssignOpKind::AddAssign => AssignOp::AddAssign,
        hir::AssignOpKind::SubAssign => AssignOp::SubAssign,
        hir::AssignOpKind::MulAssign => AssignOp::MulAssign,
        hir::AssignOpKind::DivAssign => AssignOp::DivAssign,
        hir::AssignOpKind::RemAssign => AssignOp::RemAssign,
        hir::AssignOpKind::BitXorAssign => AssignOp::BitXorAssign,
        hir::AssignOpKind::BitAndAssign => AssignOp::BitAndAssign,
        hir::AssignOpKind::BitOrAssign => AssignOp::BitOrAssign,
        hir::AssignOpKind::ShlAssign => AssignOp::ShlAssign,
        hir::AssignOpKind::ShrAssign => AssignOp::ShrAssign,
    }
}