use std::sync::atomic::Ordering::Relaxed;
use either::{Left, Right};
use rustc_abi::{self as abi, BackendRepr};
use rustc_hir::def::DefKind;
use rustc_middle::bug;
use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo};
use rustc_middle::mir::{self, ConstAlloc, ConstValue};
use rustc_middle::query::TyCtxtAt;
use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
use rustc_middle::ty::print::with_no_trimmed_paths;
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_span::def_id::LocalDefId;
use rustc_span::{DUMMY_SP, Span};
use tracing::{debug, instrument, trace};
use super::{CanAccessMutGlobal, CompileTimeInterpCx, CompileTimeMachine};
use crate::const_eval::CheckAlignment;
use crate::interpret::{
CtfeValidationMode, GlobalId, Immediate, InternKind, InternResult, InterpCx, InterpErrorKind,
InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, StackPopCleanup, create_static_alloc,
eval_nullary_intrinsic, intern_const_alloc_recursive, interp_ok, throw_exhaust,
};
use crate::{CTRL_C_RECEIVED, errors};
#[instrument(level = "trace", skip(ecx, body))]
fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
ecx: &mut CompileTimeInterpCx<'tcx>,
cid: GlobalId<'tcx>,
body: &'tcx mir::Body<'tcx>,
) -> InterpResult<'tcx, R> {
let tcx = *ecx.tcx;
assert!(
cid.promoted.is_some()
|| matches!(
ecx.tcx.def_kind(cid.instance.def_id()),
DefKind::Const
| DefKind::Static { .. }
| DefKind::ConstParam
| DefKind::AnonConst
| DefKind::InlineConst
| DefKind::AssocConst
),
"Unexpected DefKind: {:?}",
ecx.tcx.def_kind(cid.instance.def_id())
);
let layout = ecx.layout_of(body.bound_return_ty().instantiate(tcx, cid.instance.args))?;
assert!(layout.is_sized());
let intern_kind = if cid.promoted.is_some() {
InternKind::Promoted
} else {
match tcx.static_mutability(cid.instance.def_id()) {
Some(m) => InternKind::Static(m),
None => InternKind::Constant,
}
};
let ret = if let InternKind::Static(_) = intern_kind {
create_static_alloc(ecx, cid.instance.def_id().expect_local(), layout)?
} else {
ecx.allocate(layout, MemoryKind::Stack)?
};
trace!(
"eval_body_using_ecx: pushing stack frame for global: {}{}",
with_no_trimmed_paths!(ecx.tcx.def_path_str(cid.instance.def_id())),
cid.promoted.map_or_else(String::new, |p| format!("::{p:?}"))
);
ecx.push_stack_frame_raw(cid.instance, body, &ret, StackPopCleanup::Root { cleanup: false })?;
ecx.storage_live_for_always_live_locals()?;
while ecx.step()? {
if CTRL_C_RECEIVED.load(Relaxed) {
throw_exhaust!(Interrupted);
}
}
let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
const_validate_mplace(ecx, &ret, cid)?;
match intern_result {
Ok(()) => {}
Err(InternResult::FoundDanglingPointer) => {
return Err(ecx
.tcx
.dcx()
.emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind }))
.into();
}
Err(InternResult::FoundBadMutablePointer) => {
return Err(ecx
.tcx
.dcx()
.emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind }))
.into();
}
}
interp_ok(R::make_result(ret, ecx))
}
pub(crate) fn mk_eval_cx_to_read_const_val<'tcx>(
tcx: TyCtxt<'tcx>,
root_span: Span,
typing_env: ty::TypingEnv<'tcx>,
can_access_mut_global: CanAccessMutGlobal,
) -> CompileTimeInterpCx<'tcx> {
debug!("mk_eval_cx: {:?}", typing_env);
InterpCx::new(
tcx,
root_span,
typing_env,
CompileTimeMachine::new(can_access_mut_global, CheckAlignment::No),
)
}
pub fn mk_eval_cx_for_const_val<'tcx>(
tcx: TyCtxtAt<'tcx>,
typing_env: ty::TypingEnv<'tcx>,
val: mir::ConstValue<'tcx>,
ty: Ty<'tcx>,
) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, typing_env, CanAccessMutGlobal::No);
let op = ecx.const_val_to_op(val, ty, None).discard_err()?;
Some((ecx, op))
}
#[instrument(skip(ecx), level = "debug")]
pub(super) fn op_to_const<'tcx>(
ecx: &CompileTimeInterpCx<'tcx>,
op: &OpTy<'tcx>,
for_diagnostics: bool,
) -> ConstValue<'tcx> {
if op.layout.is_zst() {
return ConstValue::ZeroSized;
}
let force_as_immediate = match op.layout.backend_repr {
BackendRepr::Scalar(abi::Scalar::Initialized { .. }) => true,
_ => false,
};
let immediate = if force_as_immediate {
match ecx.read_immediate(op).report_err() {
Ok(imm) => Right(imm),
Err(err) => {
if for_diagnostics {
op.as_mplace_or_imm()
} else {
panic!("normalization works on validated constants: {err:?}")
}
}
}
} else {
op.as_mplace_or_imm()
};
debug!(?immediate);
match immediate {
Left(ref mplace) => {
let (prov, offset) = mplace.ptr().into_parts();
let alloc_id = prov.expect("cannot have `fake` place for non-ZST type").alloc_id();
ConstValue::Indirect { alloc_id, offset }
}
Right(imm) => match *imm {
Immediate::Scalar(x) => ConstValue::Scalar(x),
Immediate::ScalarPair(a, b) => {
debug!("ScalarPair(a: {:?}, b: {:?})", a, b);
let pointee_ty = imm.layout.ty.builtin_deref(false).unwrap(); debug_assert!(
matches!(
ecx.tcx.struct_tail_for_codegen(pointee_ty, ecx.typing_env()).kind(),
ty::Str | ty::Slice(..),
),
"`ConstValue::Slice` is for slice-tailed types only, but got {}",
imm.layout.ty,
);
let msg = "`op_to_const` on an immediate scalar pair must only be used on slice references to the beginning of an actual allocation";
let (prov, offset) = a.to_pointer(ecx).expect(msg).into_parts();
let alloc_id = prov.expect(msg).alloc_id();
let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
assert!(offset == abi::Size::ZERO, "{}", msg);
let meta = b.to_target_usize(ecx).expect(msg);
ConstValue::Slice { data, meta }
}
Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
},
}
}
#[instrument(skip(tcx), level = "debug", ret)]
pub(crate) fn turn_into_const_value<'tcx>(
tcx: TyCtxt<'tcx>,
constant: ConstAlloc<'tcx>,
key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
) -> ConstValue<'tcx> {
let cid = key.value;
let def_id = cid.instance.def.def_id();
let is_static = tcx.is_static(def_id);
let ecx = mk_eval_cx_to_read_const_val(
tcx,
tcx.def_span(key.value.instance.def_id()),
key.typing_env,
CanAccessMutGlobal::from(is_static),
);
let mplace = ecx.raw_const_to_mplace(constant).expect(
"can only fail if layout computation failed, \
which should have given a good error before ever invoking this function",
);
assert!(
!is_static || cid.promoted.is_some(),
"the `eval_to_const_value_raw` query should not be used for statics, use `eval_to_allocation` instead"
);
op_to_const(&ecx, &mplace.into(), false)
}
#[instrument(skip(tcx), level = "debug")]
pub fn eval_to_const_value_raw_provider<'tcx>(
tcx: TyCtxt<'tcx>,
key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
) -> ::rustc_middle::mir::interpret::EvalToConstValueResult<'tcx> {
if let ty::InstanceKind::Intrinsic(def_id) = key.value.instance.def {
let ty = key.value.instance.ty(tcx, key.typing_env);
let ty::FnDef(_, args) = ty.kind() else {
bug!("intrinsic with type {:?}", ty);
};
return eval_nullary_intrinsic(tcx, key.typing_env, def_id, args).report_err().map_err(
|error| {
let span = tcx.def_span(def_id);
super::report(
tcx,
error.into_kind(),
span,
|| (span, vec![]),
|span, _| errors::NullaryIntrinsicError { span },
)
},
);
}
tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
}
#[instrument(skip(tcx), level = "debug")]
pub fn eval_static_initializer_provider<'tcx>(
tcx: TyCtxt<'tcx>,
def_id: LocalDefId,
) -> ::rustc_middle::mir::interpret::EvalStaticInitializerRawResult<'tcx> {
assert!(tcx.is_static(def_id.to_def_id()));
let instance = ty::Instance::mono(tcx, def_id.to_def_id());
let cid = rustc_middle::mir::interpret::GlobalId { instance, promoted: None };
eval_in_interpreter(tcx, cid, ty::TypingEnv::fully_monomorphized())
}
pub trait InterpretationResult<'tcx> {
fn make_result(
mplace: MPlaceTy<'tcx>,
ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
) -> Self;
}
impl<'tcx> InterpretationResult<'tcx> for ConstAlloc<'tcx> {
fn make_result(
mplace: MPlaceTy<'tcx>,
_ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
) -> Self {
ConstAlloc { alloc_id: mplace.ptr().provenance.unwrap().alloc_id(), ty: mplace.layout.ty }
}
}
#[instrument(skip(tcx), level = "debug")]
pub fn eval_to_allocation_raw_provider<'tcx>(
tcx: TyCtxt<'tcx>,
key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
assert!(key.value.promoted.is_some() || !tcx.is_static(key.value.instance.def_id()));
debug_assert_eq!(key.typing_env.typing_mode, ty::TypingMode::PostAnalysis);
if cfg!(debug_assertions) {
let instance = with_no_trimmed_paths!(key.value.instance.to_string());
trace!("const eval: {:?} ({})", key, instance);
}
eval_in_interpreter(tcx, key.value, key.typing_env)
}
fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
tcx: TyCtxt<'tcx>,
cid: GlobalId<'tcx>,
typing_env: ty::TypingEnv<'tcx>,
) -> Result<R, ErrorHandled> {
let def = cid.instance.def.def_id();
let is_static = tcx.is_static(def);
let mut ecx = InterpCx::new(
tcx,
tcx.def_span(def),
typing_env,
CompileTimeMachine::new(CanAccessMutGlobal::from(is_static), CheckAlignment::Error),
);
let res = ecx.load_mir(cid.instance.def, cid.promoted);
res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
.report_err()
.map_err(|error| report_eval_error(&ecx, cid, error))
}
#[inline(always)]
fn const_validate_mplace<'tcx>(
ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
mplace: &MPlaceTy<'tcx>,
cid: GlobalId<'tcx>,
) -> Result<(), ErrorHandled> {
let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
let mut ref_tracking = RefTracking::new(mplace.clone());
let mut inner = false;
while let Some((mplace, path)) = ref_tracking.next() {
let mode = match ecx.tcx.static_mutability(cid.instance.def_id()) {
_ if cid.promoted.is_some() => CtfeValidationMode::Promoted,
Some(mutbl) => CtfeValidationMode::Static { mutbl }, None => {
CtfeValidationMode::Const { allow_immutable_unsafe_cell: !inner }
}
};
ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)
.report_err()
.map_err(|error| report_validation_error(&ecx, cid, error, alloc_id))?;
inner = true;
}
Ok(())
}
#[inline(never)]
fn report_eval_error<'tcx>(
ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
cid: GlobalId<'tcx>,
error: InterpErrorInfo<'tcx>,
) -> ErrorHandled {
let (error, backtrace) = error.into_parts();
backtrace.print_backtrace();
let (kind, instance) = if ecx.tcx.is_static(cid.instance.def_id()) {
("static", String::new())
} else {
let instance = &cid.instance;
if !instance.args.is_empty() {
let instance = with_no_trimmed_paths!(instance.to_string());
("const_with_path", instance)
} else {
("const", String::new())
}
};
super::report(
*ecx.tcx,
error,
DUMMY_SP,
|| super::get_span_and_frames(ecx.tcx, ecx.stack()),
|span, frames| errors::ConstEvalError {
span,
error_kind: kind,
instance,
frame_notes: frames,
},
)
}
#[inline(never)]
fn report_validation_error<'tcx>(
ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
cid: GlobalId<'tcx>,
error: InterpErrorInfo<'tcx>,
alloc_id: AllocId,
) -> ErrorHandled {
if !matches!(error.kind(), InterpErrorKind::UndefinedBehavior(_)) {
return report_eval_error(ecx, cid, error);
}
let (error, backtrace) = error.into_parts();
backtrace.print_backtrace();
let bytes = ecx.print_alloc_bytes_for_diagnostics(alloc_id);
let info = ecx.get_alloc_info(alloc_id);
let raw_bytes =
errors::RawBytesNote { size: info.size.bytes(), align: info.align.bytes(), bytes };
crate::const_eval::report(
*ecx.tcx,
error,
DUMMY_SP,
|| crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()),
move |span, frames| errors::ValidationFailure { span, ub_note: (), frames, raw_bytes },
)
}