rustc_const_eval/const_eval/
eval_queries.rs

Help
1use std::sync::atomic::Ordering::Relaxed;
2
3use either::{Left, Right};
4use rustc_abi::{self as abi, BackendRepr};
5use rustc_hir::def::DefKind;
6use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo};
7use rustc_middle::mir::{self, ConstAlloc, ConstValue};
8use rustc_middle::query::TyCtxtAt;
9use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
10use rustc_middle::ty::print::with_no_trimmed_paths;
11use rustc_middle::ty::{self, Ty, TyCtxt};
12use rustc_middle::{bug, throw_inval};
13use rustc_span::def_id::LocalDefId;
14use rustc_span::{DUMMY_SP, Span};
15use tracing::{debug, instrument, trace};
16
17use super::{CanAccessMutGlobal, CompileTimeInterpCx, CompileTimeMachine};
18use crate::const_eval::CheckAlignment;
19use crate::interpret::{
20    CtfeValidationMode, GlobalId, Immediate, InternKind, InternResult, InterpCx, InterpErrorKind,
21    InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, StackPopCleanup, create_static_alloc,
22    eval_nullary_intrinsic, intern_const_alloc_recursive, interp_ok, throw_exhaust,
23};
24use crate::{CTRL_C_RECEIVED, errors};
25
26// Returns a pointer to where the result lives
27#[instrument(level = "trace", skip(ecx, body))]
28fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
29    ecx: &mut CompileTimeInterpCx<'tcx>,
30    cid: GlobalId<'tcx>,
31    body: &'tcx mir::Body<'tcx>,
32) -> InterpResult<'tcx, R> {
33    let tcx = *ecx.tcx;
34    assert!(
35        cid.promoted.is_some()
36            || matches!(
37                ecx.tcx.def_kind(cid.instance.def_id()),
38                DefKind::Const
39                    | DefKind::Static { .. }
40                    | DefKind::ConstParam
41                    | DefKind::AnonConst
42                    | DefKind::InlineConst
43                    | DefKind::AssocConst
44            ),
45        "Unexpected DefKind: {:?}",
46        ecx.tcx.def_kind(cid.instance.def_id())
47    );
48    let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
49    assert!(layout.is_sized());
50
51    let intern_kind = if cid.promoted.is_some() {
52        InternKind::Promoted
53    } else {
54        match tcx.static_mutability(cid.instance.def_id()) {
55            Some(m) => InternKind::Static(m),
56            None => InternKind::Constant,
57        }
58    };
59
60    let ret = if let InternKind::Static(_) = intern_kind {
61        create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
62    } else {
63        ecx.allocate(layout, MemoryKind::Stack)?
64    };
65
66    trace!(
67        "eval_body_using_ecx: pushing stack frame for global: {}{}",
68        with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
69        cid.promoted.map_or_else(String::new, |p| format!("::{p:?}"))
70    );
71
72    // This can't use `init_stack_frame` since `body` is not a function,
73    // so computing its ABI would fail. It's also not worth it since there are no arguments to pass.
74    ecx.push_stack_frame_raw(cid.instance, body, &ret, StackPopCleanup::Root { cleanup: false })?;
75    ecx.storage_live_for_always_live_locals()?;
76
77    // The main interpreter loop.
78    while ecx.step()? {
79        if CTRL_C_RECEIVED.load(Relaxed) {
80            throw_exhaust!(Interrupted);
81        }
82    }
83
84    // Intern the result
85    let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
86
87    // Since evaluation had no errors, validate the resulting constant.
88    const_validate_mplace(ecx, &ret, cid)?;
89
90    // Only report this after validation, as validaiton produces much better diagnostics.
91    // FIXME: ensure validation always reports this and stop making interning care about it.
92
93    match intern_result {
94        Ok(()) => {}
95        Err(InternResult::FoundDanglingPointer) => {
96            throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
97                ecx.tcx
98                    .dcx()
99                    .emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
100            )));
101        }
102        Err(InternResult::FoundBadMutablePointer) => {
103            throw_inval!(AlreadyReported(ReportedErrorInfo::non_const_eval_error(
104                ecx.tcx
105                    .dcx()
106                    .emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind }),
107            )));
108        }
109    }
110
111    interp_ok(R::make_result(ret, ecx))
112}
113
114/// The `InterpCx` is only meant to be used to do field and index projections into constants for
115/// `simd_shuffle` and const patterns in match arms.
116///
117/// This should *not* be used to do any actual interpretation. In particular, alignment checks are
118/// turned off!
119///
120/// The function containing the `match` that is currently being analyzed may have generic bounds
121/// that inform us about the generic bounds of the constant. E.g., using an associated constant
122/// of a function's generic parameter will require knowledge about the bounds on the generic
123/// parameter. These bounds are passed to `mk_eval_cx` via the `ParamEnv` argument.
124pub(crate) fn mk_eval_cx_to_read_const_val<'tcx>(
125    tcx: TyCtxt<'tcx>,
126    root_span: Span,
127    typing_env: ty::TypingEnv<'tcx>,
128    can_access_mut_global: CanAccessMutGlobal,
129) -> CompileTimeInterpCx<'tcx> {
130    debug!("mk_eval_cx: {:?}", typing_env);
131    InterpCx::new(
132        tcx,
133        root_span,
134        typing_env,
135        CompileTimeMachine::new(can_access_mut_global, CheckAlignment::No),
136    )
137}
138
139/// Create an interpreter context to inspect the given `ConstValue`.
140/// Returns both the context and an `OpTy` that represents the constant.
141pub fn mk_eval_cx_for_const_val<'tcx>(
142    tcx: TyCtxtAt<'tcx>,
143    typing_env: ty::TypingEnv<'tcx>,
144    val: mir::ConstValue<'tcx>,
145    ty: Ty<'tcx>,
146) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
147    let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, typing_env, CanAccessMutGlobal::No);
148    // FIXME: is it a problem to discard the error here?
149    let op = ecx.const_val_to_op(val, ty, None).discard_err()?;
150    Some((ecx, op))
151}
152
153/// This function converts an interpreter value into a MIR constant.
154///
155/// The `for_diagnostics` flag turns the usual rules for returning `ConstValue::Scalar` into a
156/// best-effort attempt. This is not okay for use in const-eval sine it breaks invariants rustc
157/// relies on, but it is okay for diagnostics which will just give up gracefully when they
158/// encounter an `Indirect` they cannot handle.
159#[instrument(skip(ecx), level = "debug")]
160pub(super) fn op_to_const<'tcx>(
161    ecx: &CompileTimeInterpCx<'tcx>,
162    op: &OpTy<'tcx>,
163    for_diagnostics: bool,
164) -> ConstValue<'tcx> {
165    // Handle ZST consistently and early.
166    if op.layout.is_zst() {
167        return ConstValue::ZeroSized;
168    }
169
170    // All scalar types should be stored as `ConstValue::Scalar`. This is needed to make
171    // `ConstValue::try_to_scalar` efficient; we want that to work for *all* constants of scalar
172    // type (it's used throughout the compiler and having it work just on literals is not enough)
173    // and we want it to be fast (i.e., don't go to an `Allocation` and reconstruct the `Scalar`
174    // from its byte-serialized form).
175    let force_as_immediate = match op.layout.backend_repr {
176        BackendRepr::Scalar(abi::Scalar::Initialized { .. }) => true,
177        // We don't *force* `ConstValue::Slice` for `ScalarPair`. This has the advantage that if the
178        // input `op` is a place, then turning it into a `ConstValue` and back into a `OpTy` will
179        // not have to generate any duplicate allocations (we preserve the original `AllocId` in
180        // `ConstValue::Indirect`). It means accessing the contents of a slice can be slow (since
181        // they can be stored as `ConstValue::Indirect`), but that's not relevant since we barely
182        // ever have to do this. (`try_get_slice_bytes_for_diagnostics` exists to provide this
183        // functionality.)
184        _ => false,
185    };
186    let immediate = if force_as_immediate {
187        match ecx.read_immediate(op).report_err() {
188            Ok(imm) => Right(imm),
189            Err(err) => {
190                if for_diagnostics {
191                    // This discard the error, but for diagnostics that's okay.
192                    op.as_mplace_or_imm()
193                } else {
194                    panic!("normalization works on validated constants: {err:?}")
195                }
196            }
197        }
198    } else {
199        op.as_mplace_or_imm()
200    };
201
202    debug!(?immediate);
203
204    match immediate {
205        Left(ref mplace) => {
206            // We know `offset` is relative to the allocation, so we can use `into_parts`.
207            let (prov, offset) = mplace.ptr().into_parts();
208            let alloc_id = prov.expect("cannot have `fake` place for non-ZST type").alloc_id();
209            ConstValue::Indirect { alloc_id, offset }
210        }
211        // see comment on `let force_as_immediate` above
212        Right(imm) => match *imm {
213            Immediate::Scalar(x) => ConstValue::Scalar(x),
214            Immediate::ScalarPair(a, b) => {
215                debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
216                // This codepath solely exists for `valtree_to_const_value` to not need to generate
217                // a `ConstValue::Indirect` for wide references, so it is tightly restricted to just
218                // that case.
219                let pointee_ty = imm.layout.ty.builtin_deref(false).unwrap(); // `false` = no raw ptrs
220                debug_assert!(
221                    matches!(
222                        ecx.tcx.struct_tail_for_codegen(pointee_ty, ecx.typing_env()).kind(),
223                        ty::Str | ty::Slice(..),
224                    ),
225                    "`ConstValue::Slice` is for slice-tailed types only, but got {}",
226                    imm.layout.ty,
227                );
228                let msg = "`op_to_const` on an immediate scalar pair must only be used on slice references to the beginning of an actual allocation";
229                // We know `offset` is relative to the allocation, so we can use `into_parts`.
230                let (prov, offset) = a.to_pointer(ecx).expect(msg).into_parts();
231                let alloc_id = prov.expect(msg).alloc_id();
232                let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
233                assert!(offset == abi::Size::ZERO, "{}", msg);
234                let meta = b.to_target_usize(ecx).expect(msg);
235                ConstValue::Slice { data, meta }
236            }
237            Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
238        },
239    }
240}
241
242#[instrument(skip(tcx), level = "debug", ret)]
243pub(crate) fn turn_into_const_value<'tcx>(
244    tcx: TyCtxt<'tcx>,
245    constant: ConstAlloc<'tcx>,
246    key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
247) -> ConstValue<'tcx> {
248    let cid = key.value;
249    let def_id = cid.instance.def.def_id();
250    let is_static = tcx.is_static(def_id);
251    // This is just accessing an already computed constant, so no need to check alignment here.
252    let ecx = mk_eval_cx_to_read_const_val(
253        tcx,
254        tcx.def_span(key.value.instance.def_id()),
255        key.typing_env,
256        CanAccessMutGlobal::from(is_static),
257    );
258
259    let mplace = ecx.raw_const_to_mplace(constant).expect(
260        "can only fail if layout computation failed, \
261        which should have given a good error before ever invoking this function",
262    );
263    assert!(
264        !is_static || cid.promoted.is_some(),
265        "the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead"
266    );
267
268    // Turn this into a proper constant.
269    op_to_const(&ecx, &mplace.into(), /* for diagnostics */ false)
270}
271
272#[instrument(skip(tcx), level = "debug")]
273pub fn eval_to_const_value_raw_provider<'tcx>(
274    tcx: TyCtxt<'tcx>,
275    key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
276) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> {
277    // We call `const_eval` for zero arg intrinsics, too, in order to cache their value.
278    // Catch such calls and evaluate them instead of trying to load a constant's MIR.
279    if let ty::InstanceKind::Intrinsic(def_id) = key.value.instance.def {
280        let ty = key.value.instance.ty(tcx, key.typing_env);
281        let ty::FnDef(_, args) = ty.kind() else {
282            bug!("intrinsic with type {:?}", ty);
283        };
284        return eval_nullary_intrinsic(tcx, key.typing_env, def_id, args).report_err().map_err(
285            |error| {
286                let span = tcx.def_span(def_id);
287
288                super::report(
289                    tcx,
290                    error.into_kind(),
291                    span,
292                    || (span, vec![]),
293                    |span, _| errors::NullaryIntrinsicError { span },
294                )
295            },
296        );
297    }
298
299    tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
300}
301
302#[instrument(skip(tcx), level = "debug")]
303pub fn eval_static_initializer_provider<'tcx>(
304    tcx: TyCtxt<'tcx>,
305    def_id: LocalDefId,
306) -> ::rustc_middle::mir::interpret::EvalStaticInitializerRawResult<'tcx> {
307    assert!(tcx.is_static(def_id.to_def_id()));
308
309    let instance = ty::Instance::mono(tcx, def_id.to_def_id());
310    let cid = rustc_middle::mir::interpret::GlobalId { instance, promoted: None };
311    eval_in_interpreter(tcx, cid, ty::TypingEnv::fully_monomorphized())
312}
313
314pub trait InterpretationResult<'tcx> {
315    /// This function takes the place where the result of the evaluation is stored
316    /// and prepares it for returning it in the appropriate format needed by the specific
317    /// evaluation query.
318    fn make_result(
319        mplace: MPlaceTy<'tcx>,
320        ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
321    ) -> Self;
322}
323
324impl<'tcx> InterpretationResult<'tcx> for ConstAlloc<'tcx> {
325    fn make_result(
326        mplace: MPlaceTy<'tcx>,
327        _ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
328    ) -> Self {
329        ConstAlloc { alloc_id: mplace.ptr().provenance.unwrap().alloc_id(), ty: mplace.layout.ty }
330    }
331}
332
333#[instrument(skip(tcx), level = "debug")]
334pub fn eval_to_allocation_raw_provider<'tcx>(
335    tcx: TyCtxt<'tcx>,
336    key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
337) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
338    // This shouldn't be used for statics, since statics are conceptually places,
339    // not values -- so what we do here could break pointer identity.
340    assert!(key.value.promoted.is_some() || !tcx.is_static(key.value.instance.def_id()));
341    // Const eval always happens in PostAnalysis mode . See the comment in
342    // `InterpCx::new` for more details.
343    debug_assert_eq!(key.typing_env.typing_mode, ty::TypingMode::PostAnalysis);
344    if cfg!(debug_assertions) {
345        // Make sure we format the instance even if we do not print it.
346        // This serves as a regression test against an ICE on printing.
347        // The next two lines concatenated contain some discussion:
348        // https://rust-lang.zulipchat.com/#narrow/stream/146212-t-compiler.2Fconst-eval/
349        // subject/anon_const_instance_printing/near/135980032
350        let instance = with_no_trimmed_paths!(key.value.instance.to_string());
351        trace!("const eval: {:?} ({})", key, instance);
352    }
353
354    eval_in_interpreter(tcx, key.value, key.typing_env)
355}
356
357fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
358    tcx: TyCtxt<'tcx>,
359    cid: GlobalId<'tcx>,
360    typing_env: ty::TypingEnv<'tcx>,
361) -> Result<R, ErrorHandled> {
362    let def = cid.instance.def.def_id();
363    let is_static = tcx.is_static(def);
364
365    let mut ecx = InterpCx::new(
366        tcx,
367        tcx.def_span(def),
368        typing_env,
369        // Statics (and promoteds inside statics) may access mutable global memory, because unlike consts
370        // they do not have to behave "as if" they were evaluated at runtime.
371        // For consts however we want to ensure they behave "as if" they were evaluated at runtime,
372        // so we have to reject reading mutable global memory.
373        CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
374    );
375    let res = ecx.load_mir(cid.instance.def, cid.promoted);
376    res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
377        .report_err()
378        .map_err(|error| report_eval_error(&ecx, cid, error))
379}
380
381#[inline(always)]
382fn const_validate_mplace<'tcx>(
383    ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
384    mplace: &MPlaceTy<'tcx>,
385    cid: GlobalId<'tcx>,
386) -> Result<(), ErrorHandled> {
387    let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
388    let mut ref_tracking = RefTracking::new(mplace.clone());
389    let mut inner = false;
390    while let Some((mplace, path)) = ref_tracking.next() {
391        let mode = match ecx.tcx.static_mutability(cid.instance.def_id()) {
392            _ if cid.promoted.is_some() => CtfeValidationMode::Promoted,
393            Some(mutbl) => CtfeValidationMode::Static { mutbl }, // a `static`
394            None => {
395                // This is a normal `const` (not promoted).
396                // The outermost allocation is always only copied, so having `UnsafeCell` in there
397                // is okay despite them being in immutable memory.
398                CtfeValidationMode::Const { allow_immutable_unsafe_cell: !inner }
399            }
400        };
401        ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)
402            .report_err()
403            // Instead of just reporting the `InterpError` via the usual machinery, we give a more targeted
404            // error about the validation failure.
405            .map_err(|error| report_validation_error(&ecx, cid, error, alloc_id))?;
406        inner = true;
407    }
408
409    Ok(())
410}
411
412#[inline(never)]
413fn report_eval_error<'tcx>(
414    ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
415    cid: GlobalId<'tcx>,
416    error: InterpErrorInfo<'tcx>,
417) -> ErrorHandled {
418    let (error, backtrace) = error.into_parts();
419    backtrace.print_backtrace();
420
421    let (kind, instance) = if ecx.tcx.is_static(cid.instance.def_id()) {
422        ("static", String::new())
423    } else {
424        // If the current item has generics, we'd like to enrich the message with the
425        // instance and its args: to show the actual compile-time values, in addition to
426        // the expression, leading to the const eval error.
427        let instance = &cid.instance;
428        if !instance.args.is_empty() {
429            let instance = with_no_trimmed_paths!(instance.to_string());
430            ("const_with_path", instance)
431        } else {
432            ("const", String::new())
433        }
434    };
435
436    super::report(
437        *ecx.tcx,
438        error,
439        DUMMY_SP,
440        || super::get_span_and_frames(ecx.tcx, ecx.stack()),
441        |span, frames| errors::ConstEvalError {
442            span,
443            error_kind: kind,
444            instance,
445            frame_notes: frames,
446        },
447    )
448}
449
450#[inline(never)]
451fn report_validation_error<'tcx>(
452    ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
453    cid: GlobalId<'tcx>,
454    error: InterpErrorInfo<'tcx>,
455    alloc_id: AllocId,
456) -> ErrorHandled {
457    if !matches!(error.kind(), InterpErrorKind::UndefinedBehavior(_)) {
458        // Some other error happened during validation, e.g. an unsupported operation.
459        return report_eval_error(ecx, cid, error);
460    }
461
462    let (error, backtrace) = error.into_parts();
463    backtrace.print_backtrace();
464
465    let bytes = ecx.print_alloc_bytes_for_diagnostics(alloc_id);
466    let info = ecx.get_alloc_info(alloc_id);
467    let raw_bytes =
468        errors::RawBytesNote { size: info.size.bytes(), align: info.align.bytes(), bytes };
469
470    crate::const_eval::report(
471        *ecx.tcx,
472        error,
473        DUMMY_SP,
474        || crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()),
475        move |span, frames| errors::ValidationFailure { span, ub_note: (), frames, raw_bytes },
476    )
477}
rustc_const_eval/const_eval/eval_queries.rs

rustc_const_eval/const_eval/
eval_queries.rs
