rustc_middle/query/
erase.rs

1//! To improve compile times and code size for the compiler itself, query
2//! values are "erased" in some contexts (e.g. inside in-memory cache types),
3//! to reduce the number of generic instantiations created during codegen.
4//!
5//! See <https://github.com/rust-lang/rust/pull/151715> for some bootstrap-time
6//! and performance benchmarks.
7
8use std::ffi::OsStr;
9use std::intrinsics::transmute_unchecked;
10use std::mem::MaybeUninit;
11
12use rustc_ast::tokenstream::TokenStream;
13use rustc_span::ErrorGuaranteed;
14use rustc_span::source_map::Spanned;
15
16use crate::mir::interpret::EvalToValTreeResult;
17use crate::mir::mono::{MonoItem, NormalizationErrorInMono};
18use crate::query::plumbing::CyclePlaceholder;
19use crate::traits::solve;
20use crate::ty::adjustment::CoerceUnsizedInfo;
21use crate::ty::{self, Ty, TyCtxt};
22use crate::{mir, traits};
23
24/// Internal implementation detail of [`Erased`].
25#[derive(#[automatically_derived]
impl<Storage: ::core::marker::Copy + Copy> ::core::marker::Copy for
    ErasedData<Storage> {
}Copy, #[automatically_derived]
impl<Storage: ::core::clone::Clone + Copy> ::core::clone::Clone for
    ErasedData<Storage> {
    #[inline]
    fn clone(&self) -> ErasedData<Storage> {
        ErasedData { data: ::core::clone::Clone::clone(&self.data) }
    }
}Clone)]
26pub struct ErasedData<Storage: Copy> {
27    /// We use `MaybeUninit` here to make sure it's legal to store a transmuted
28    /// value that isn't actually of type `Storage`.
29    data: MaybeUninit<Storage>,
30}
31
32/// Trait for types that can be erased into [`Erased<Self>`].
33///
34/// Erasing and unerasing values is performed by [`erase_val`] and [`restore_val`].
35///
36/// FIXME: This whole trait could potentially be replaced by `T: Copy` and the
37/// storage type `[u8; size_of::<T>()]` when support for that is more mature.
38pub trait Erasable: Copy {
39    /// Storage type to used for erased values of this type.
40    /// Should be `[u8; N]`, where N is equal to `size_of::<Self>`.
41    ///
42    /// [`ErasedData`] wraps this storage type in `MaybeUninit` to ensure that
43    /// transmutes to/from erased storage are well-defined.
44    type Storage: Copy;
45}
46
47/// A value of `T` that has been "erased" into some opaque storage type.
48///
49/// This is helpful for reducing the number of concrete instantiations needed
50/// during codegen when building the compiler.
51///
52/// Using an opaque type alias allows the type checker to enforce that
53/// `Erased<T>` and `Erased<U>` are still distinct types, while allowing
54/// monomorphization to see that they might actually use the same storage type.
55pub type Erased<T: Erasable> = ErasedData<impl Copy>;
56
57/// Erases a value of type `T` into `Erased<T>`.
58///
59/// `Erased<T>` and `Erased<U>` are type-checked as distinct types, but codegen
60/// can see whether they actually have the same storage type.
61///
62/// FIXME: This might have soundness issues with erasable types that don't
63/// implement the same auto-traits as `[u8; _]`; see
64/// <https://github.com/rust-lang/rust/pull/151715#discussion_r2740113250>
65#[inline(always)]
66#[define_opaque(Erased)]
67pub fn erase_val<T: Erasable>(value: T) -> Erased<T> {
68    // Ensure the sizes match
69    const {
70        if size_of::<T>() != size_of::<T::Storage>() {
71            {
    ::core::panicking::panic_fmt(format_args!("size of T must match erased type <T as Erasable>::Storage"));
}panic!("size of T must match erased type <T as Erasable>::Storage")
72        }
73    };
74
75    ErasedData::<<T as Erasable>::Storage> {
76        // `transmute_unchecked` is needed here because it does not have `transmute`'s size check
77        // (and thus allows to transmute between `T` and `MaybeUninit<T::Storage>`) (we do the size
78        // check ourselves in the `const` block above).
79        //
80        // `transmute_copy` is also commonly used for this (and it would work here since
81        // `Erasable: Copy`), but `transmute_unchecked` better explains the intent.
82        //
83        // SAFETY: It is safe to transmute to MaybeUninit for types with the same sizes.
84        data: unsafe { transmute_unchecked::<T, MaybeUninit<T::Storage>>(value) },
85    }
86}
87
88/// Restores an erased value to its real type.
89///
90/// This relies on the fact that `Erased<T>` and `Erased<U>` are type-checked
91/// as distinct types, even if they use the same storage type.
92#[inline(always)]
93#[define_opaque(Erased)]
94pub fn restore_val<T: Erasable>(erased_value: Erased<T>) -> T {
95    let ErasedData { data }: ErasedData<<T as Erasable>::Storage> = erased_value;
96    // See comment in `erase_val` for why we use `transmute_unchecked`.
97    //
98    // SAFETY: Due to the use of impl Trait in `Erased` the only way to safely create an instance
99    // of `Erased` is to call `erase_val`, so we know that `erased_value.data` is a valid instance
100    // of `T` of the right size.
101    unsafe { transmute_unchecked::<MaybeUninit<T::Storage>, T>(data) }
102}
103
104// FIXME(#151565): Using `T: ?Sized` here should let us remove the separate
105// impls for fat reference types.
106impl<T> Erasable for &'_ T {
107    type Storage = [u8; size_of::<&'static ()>()];
108}
109
110impl<T> Erasable for &'_ [T] {
111    type Storage = [u8; size_of::<&'static [()]>()];
112}
113
114impl Erasable for &'_ OsStr {
115    type Storage = [u8; size_of::<&'static OsStr>()];
116}
117
118impl<T> Erasable for &'_ ty::List<T> {
119    type Storage = [u8; size_of::<&'static ty::List<()>>()];
120}
121
122impl<T> Erasable for &'_ ty::ListWithCachedTypeInfo<T> {
123    type Storage = [u8; size_of::<&'static ty::ListWithCachedTypeInfo<()>>()];
124}
125
126impl<I: rustc_index::Idx, T> Erasable for &'_ rustc_index::IndexSlice<I, T> {
127    type Storage = [u8; size_of::<&'static rustc_index::IndexSlice<u32, ()>>()];
128}
129
130impl<T> Erasable for Result<&'_ T, traits::query::NoSolution> {
131    type Storage = [u8; size_of::<Result<&'static (), traits::query::NoSolution>>()];
132}
133
134impl<T> Erasable for Result<&'_ [T], traits::query::NoSolution> {
135    type Storage = [u8; size_of::<Result<&'static [()], traits::query::NoSolution>>()];
136}
137
138impl<T> Erasable for Result<&'_ T, rustc_errors::ErrorGuaranteed> {
139    type Storage = [u8; size_of::<Result<&'static (), rustc_errors::ErrorGuaranteed>>()];
140}
141
142impl<T> Erasable for Result<&'_ [T], rustc_errors::ErrorGuaranteed> {
143    type Storage = [u8; size_of::<Result<&'static [()], rustc_errors::ErrorGuaranteed>>()];
144}
145
146impl<T> Erasable for Result<&'_ T, traits::CodegenObligationError> {
147    type Storage = [u8; size_of::<Result<&'static (), traits::CodegenObligationError>>()];
148}
149
150impl<T> Erasable for Result<&'_ T, &'_ ty::layout::FnAbiError<'_>> {
151    type Storage = [u8; size_of::<Result<&'static (), &'static ty::layout::FnAbiError<'static>>>()];
152}
153
154impl<T> Erasable for Result<(&'_ T, crate::thir::ExprId), rustc_errors::ErrorGuaranteed> {
155    type Storage = [u8; size_of::<
156        Result<(&'static (), crate::thir::ExprId), rustc_errors::ErrorGuaranteed>,
157    >()];
158}
159
160impl Erasable for Result<Option<ty::Instance<'_>>, rustc_errors::ErrorGuaranteed> {
161    type Storage =
162        [u8; size_of::<Result<Option<ty::Instance<'static>>, rustc_errors::ErrorGuaranteed>>()];
163}
164
165impl Erasable for Result<CoerceUnsizedInfo, rustc_errors::ErrorGuaranteed> {
166    type Storage = [u8; size_of::<Result<CoerceUnsizedInfo, rustc_errors::ErrorGuaranteed>>()];
167}
168
169impl Erasable
170    for Result<Option<ty::EarlyBinder<'_, ty::Const<'_>>>, rustc_errors::ErrorGuaranteed>
171{
172    type Storage = [u8; size_of::<
173        Result<Option<ty::EarlyBinder<'static, ty::Const<'static>>>, rustc_errors::ErrorGuaranteed>,
174    >()];
175}
176
177impl Erasable for Result<ty::GenericArg<'_>, traits::query::NoSolution> {
178    type Storage = [u8; size_of::<Result<ty::GenericArg<'static>, traits::query::NoSolution>>()];
179}
180
181impl Erasable for Result<bool, &ty::layout::LayoutError<'_>> {
182    type Storage = [u8; size_of::<Result<bool, &'static ty::layout::LayoutError<'static>>>()];
183}
184
185impl Erasable for Result<rustc_abi::TyAndLayout<'_, Ty<'_>>, &ty::layout::LayoutError<'_>> {
186    type Storage = [u8; size_of::<
187        Result<
188            rustc_abi::TyAndLayout<'static, Ty<'static>>,
189            &'static ty::layout::LayoutError<'static>,
190        >,
191    >()];
192}
193
194impl Erasable for Result<mir::ConstAlloc<'_>, mir::interpret::ErrorHandled> {
195    type Storage =
196        [u8; size_of::<Result<mir::ConstAlloc<'static>, mir::interpret::ErrorHandled>>()];
197}
198
199impl Erasable for Result<mir::ConstValue, mir::interpret::ErrorHandled> {
200    type Storage = [u8; size_of::<Result<mir::ConstValue, mir::interpret::ErrorHandled>>()];
201}
202
203impl Erasable for Option<(mir::ConstValue, Ty<'_>)> {
204    type Storage = [u8; size_of::<Option<(mir::ConstValue, Ty<'_>)>>()];
205}
206
207impl Erasable for EvalToValTreeResult<'_> {
208    type Storage = [u8; size_of::<EvalToValTreeResult<'static>>()];
209}
210
211impl Erasable for Result<&'_ ty::List<Ty<'_>>, ty::util::AlwaysRequiresDrop> {
212    type Storage =
213        [u8; size_of::<Result<&'static ty::List<Ty<'static>>, ty::util::AlwaysRequiresDrop>>()];
214}
215
216impl Erasable for Result<ty::EarlyBinder<'_, Ty<'_>>, CyclePlaceholder> {
217    type Storage = [u8; size_of::<Result<ty::EarlyBinder<'static, Ty<'_>>, CyclePlaceholder>>()];
218}
219
220impl Erasable
221    for Result<(&'_ [Spanned<MonoItem<'_>>], &'_ [Spanned<MonoItem<'_>>]), NormalizationErrorInMono>
222{
223    type Storage = [u8; size_of::<
224        Result<
225            (&'static [Spanned<MonoItem<'static>>], &'static [Spanned<MonoItem<'static>>]),
226            NormalizationErrorInMono,
227        >,
228    >()];
229}
230
231impl Erasable for Result<&'_ TokenStream, ()> {
232    type Storage = [u8; size_of::<Result<&'static TokenStream, ()>>()];
233}
234
235impl<T> Erasable for Option<&'_ T> {
236    type Storage = [u8; size_of::<Option<&'static ()>>()];
237}
238
239impl<T> Erasable for Option<&'_ [T]> {
240    type Storage = [u8; size_of::<Option<&'static [()]>>()];
241}
242
243impl Erasable for Option<&'_ OsStr> {
244    type Storage = [u8; size_of::<Option<&'static OsStr>>()];
245}
246
247impl Erasable for Option<mir::DestructuredConstant<'_>> {
248    type Storage = [u8; size_of::<Option<mir::DestructuredConstant<'static>>>()];
249}
250
251impl Erasable for ty::ImplTraitHeader<'_> {
252    type Storage = [u8; size_of::<ty::ImplTraitHeader<'static>>()];
253}
254
255impl Erasable for Option<ty::EarlyBinder<'_, Ty<'_>>> {
256    type Storage = [u8; size_of::<Option<ty::EarlyBinder<'static, Ty<'static>>>>()];
257}
258
259impl Erasable for Option<ty::Value<'_>> {
260    type Storage = [u8; size_of::<Option<ty::Value<'static>>>()];
261}
262
263impl Erasable for rustc_hir::MaybeOwner<'_> {
264    type Storage = [u8; size_of::<rustc_hir::MaybeOwner<'static>>()];
265}
266
267impl<T: Erasable> Erasable for ty::EarlyBinder<'_, T> {
268    type Storage = T::Storage;
269}
270
271impl Erasable for ty::Binder<'_, ty::FnSig<'_>> {
272    type Storage = [u8; size_of::<ty::Binder<'static, ty::FnSig<'static>>>()];
273}
274
275impl Erasable for ty::Binder<'_, ty::CoroutineWitnessTypes<TyCtxt<'_>>> {
276    type Storage =
277        [u8; size_of::<ty::Binder<'static, ty::CoroutineWitnessTypes<TyCtxt<'static>>>>()];
278}
279
280impl Erasable for ty::Binder<'_, &'_ ty::List<Ty<'_>>> {
281    type Storage = [u8; size_of::<ty::Binder<'static, &'static ty::List<Ty<'static>>>>()];
282}
283
284impl<T0, T1> Erasable for (&'_ T0, &'_ T1) {
285    type Storage = [u8; size_of::<(&'static (), &'static ())>()];
286}
287
288impl<T0> Erasable for (solve::QueryResult<'_>, &'_ T0) {
289    type Storage = [u8; size_of::<(solve::QueryResult<'static>, &'static ())>()];
290}
291
292impl<T0, T1> Erasable for (&'_ T0, &'_ [T1]) {
293    type Storage = [u8; size_of::<(&'static (), &'static [()])>()];
294}
295
296impl<T0, T1> Erasable for (&'_ [T0], &'_ [T1]) {
297    type Storage = [u8; size_of::<(&'static [()], &'static [()])>()];
298}
299
300impl<T0> Erasable for (&'_ T0, Result<(), ErrorGuaranteed>) {
301    type Storage = [u8; size_of::<(&'static (), Result<(), ErrorGuaranteed>)>()];
302}
303
304macro_rules! impl_erasable_for_simple_types {
305    ($($ty:ty),+ $(,)?) => {
306        $(
307            impl Erasable for $ty {
308                type Storage = [u8; size_of::<$ty>()];
309            }
310        )*
311    }
312}
313
314// For concrete types with no lifetimes, the erased storage for `Foo` is
315// `[u8; size_of::<Foo>()]`.
316impl Erasable for usize {
    type Storage = [u8; size_of::<usize>()];
}impl_erasable_for_simple_types! {
317    // FIXME(#151565): Add `tidy-alphabetical-{start,end}` and sort this.
318    (),
319    bool,
320    Option<(rustc_span::def_id::DefId, rustc_session::config::EntryFnType)>,
321    Option<rustc_ast::expand::allocator::AllocatorKind>,
322    Option<rustc_hir::ConstStability>,
323    Option<rustc_hir::DefaultBodyStability>,
324    Option<rustc_hir::Stability>,
325    Option<rustc_data_structures::svh::Svh>,
326    Option<rustc_hir::def::DefKind>,
327    Option<rustc_hir::CoroutineKind>,
328    Option<rustc_hir::HirId>,
329    Option<rustc_middle::middle::stability::DeprecationEntry>,
330    Option<rustc_middle::ty::AsyncDestructor>,
331    Option<rustc_middle::ty::Destructor>,
332    Option<rustc_middle::ty::ImplTraitInTraitData>,
333    Option<rustc_middle::ty::ScalarInt>,
334    Option<rustc_span::def_id::CrateNum>,
335    Option<rustc_span::def_id::DefId>,
336    Option<rustc_span::def_id::LocalDefId>,
337    Option<rustc_span::Span>,
338    Option<rustc_abi::FieldIdx>,
339    Option<rustc_target::spec::PanicStrategy>,
340    Option<usize>,
341    Option<rustc_middle::ty::IntrinsicDef>,
342    Option<rustc_abi::Align>,
343    Result<(), rustc_errors::ErrorGuaranteed>,
344    Result<(), rustc_middle::traits::query::NoSolution>,
345    Result<rustc_middle::traits::EvaluationResult, rustc_middle::traits::OverflowError>,
346    rustc_abi::ReprOptions,
347    rustc_ast::expand::allocator::AllocatorKind,
348    rustc_hir::DefaultBodyStability,
349    rustc_hir::attrs::Deprecation,
350    rustc_hir::attrs::EiiDecl,
351    rustc_hir::attrs::EiiImpl,
352    rustc_data_structures::svh::Svh,
353    rustc_errors::ErrorGuaranteed,
354    rustc_hir::Constness,
355    rustc_hir::ConstStability,
356    rustc_hir::def_id::DefId,
357    rustc_hir::def_id::DefIndex,
358    rustc_hir::def_id::LocalDefId,
359    rustc_hir::def_id::LocalModDefId,
360    rustc_hir::def::DefKind,
361    rustc_hir::Defaultness,
362    rustc_hir::definitions::DefKey,
363    rustc_hir::CoroutineKind,
364    rustc_hir::HirId,
365    rustc_hir::IsAsync,
366    rustc_hir::ItemLocalId,
367    rustc_hir::LangItem,
368    rustc_hir::OpaqueTyOrigin<rustc_hir::def_id::DefId>,
369    rustc_hir::OwnerId,
370    rustc_hir::Stability,
371    rustc_hir::Upvar,
372    rustc_index::bit_set::FiniteBitSet<u32>,
373    rustc_middle::middle::deduced_param_attrs::DeducedParamAttrs,
374    rustc_middle::middle::dependency_format::Linkage,
375    rustc_middle::middle::exported_symbols::SymbolExportInfo,
376    rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
377    rustc_middle::middle::resolve_bound_vars::ResolvedArg,
378    rustc_middle::middle::stability::DeprecationEntry,
379    rustc_middle::mir::ConstQualifs,
380    rustc_middle::mir::ConstValue,
381    rustc_middle::mir::interpret::AllocId,
382    rustc_middle::mir::interpret::CtfeProvenance,
383    rustc_middle::mir::interpret::ErrorHandled,
384    rustc_middle::thir::ExprId,
385    rustc_middle::traits::CodegenObligationError,
386    rustc_middle::traits::EvaluationResult,
387    rustc_middle::traits::OverflowError,
388    rustc_middle::traits::query::NoSolution,
389    rustc_middle::traits::WellFormedLoc,
390    rustc_middle::ty::adjustment::CoerceUnsizedInfo,
391    rustc_middle::ty::AssocItem,
392    rustc_middle::ty::AssocContainer,
393    rustc_middle::ty::Asyncness,
394    rustc_middle::ty::AsyncDestructor,
395    rustc_middle::ty::AnonConstKind,
396    rustc_middle::ty::Destructor,
397    rustc_middle::ty::fast_reject::SimplifiedType,
398    rustc_middle::ty::ImplPolarity,
399    rustc_middle::ty::Representability,
400    rustc_middle::ty::UnusedGenericParams,
401    rustc_middle::ty::util::AlwaysRequiresDrop,
402    rustc_middle::ty::Visibility<rustc_span::def_id::DefId>,
403    rustc_middle::middle::codegen_fn_attrs::SanitizerFnAttrs,
404    rustc_session::config::CrateType,
405    rustc_session::config::EntryFnType,
406    rustc_session::config::OptLevel,
407    rustc_session::config::SymbolManglingVersion,
408    rustc_session::cstore::CrateDepKind,
409    rustc_session::cstore::ExternCrate,
410    rustc_session::cstore::LinkagePreference,
411    rustc_session::Limits,
412    rustc_session::lint::LintExpectationId,
413    rustc_span::def_id::CrateNum,
414    rustc_span::def_id::DefPathHash,
415    rustc_span::ExpnHash,
416    rustc_span::ExpnId,
417    rustc_span::Span,
418    rustc_span::Symbol,
419    rustc_span::Ident,
420    rustc_target::spec::PanicStrategy,
421    rustc_type_ir::Variance,
422    u32,
423    usize,
424}
425
426macro_rules! impl_erasable_for_single_lifetime_types {
427    ($($($fake_path:ident)::+),+ $(,)?) => {
428        $(
429            impl<'tcx> Erasable for $($fake_path)::+<'tcx> {
430                type Storage = [u8; size_of::<$($fake_path)::+<'static>>()];
431            }
432        )*
433    }
434}
435
436// For types containing a single lifetime and no other generics, e.g.
437// `Foo<'tcx>`, the erased storage is `[u8; size_of::<Foo<'static>>()]`.
438//
439// FIXME(#151565): Some of the hand-written impls above that only use one
440// lifetime can probably be migrated here.
441impl<'tcx> Erasable for rustc_middle::ty::VtblEntry<'tcx> {
    type Storage = [u8; size_of::<rustc_middle::ty::VtblEntry<'static>>()];
}impl_erasable_for_single_lifetime_types! {
442    // FIXME(#151565): Add `tidy-alphabetical-{start,end}` and sort this.
443    rustc_middle::middle::exported_symbols::ExportedSymbol,
444    rustc_middle::mir::Const,
445    rustc_middle::mir::DestructuredConstant,
446    rustc_middle::mir::ConstAlloc,
447    rustc_middle::mir::interpret::GlobalId,
448    rustc_middle::mir::interpret::EvalStaticInitializerRawResult,
449    rustc_middle::mir::mono::MonoItemPartitions,
450    rustc_middle::traits::query::MethodAutoderefStepsResult,
451    rustc_middle::traits::query::type_op::AscribeUserType,
452    rustc_middle::traits::query::type_op::Eq,
453    rustc_middle::traits::query::type_op::ProvePredicate,
454    rustc_middle::traits::query::type_op::Subtype,
455    rustc_middle::ty::AdtDef,
456    rustc_middle::ty::AliasTy,
457    rustc_middle::ty::ClauseKind,
458    rustc_middle::ty::ClosureTypeInfo,
459    rustc_middle::ty::Const,
460    rustc_middle::ty::DestructuredAdtConst,
461    rustc_middle::ty::ExistentialTraitRef,
462    rustc_middle::ty::FnSig,
463    rustc_middle::ty::GenericArg,
464    rustc_middle::ty::GenericPredicates,
465    rustc_middle::ty::ConstConditions,
466    rustc_middle::ty::inhabitedness::InhabitedPredicate,
467    rustc_middle::ty::Instance,
468    rustc_middle::ty::BoundVariableKind,
469    rustc_middle::ty::InstanceKind,
470    rustc_middle::ty::layout::FnAbiError,
471    rustc_middle::ty::layout::LayoutError,
472    rustc_middle::ty::LitToConstInput,
473    rustc_middle::ty::ParamEnv,
474    rustc_middle::ty::TypingEnv,
475    rustc_middle::ty::Predicate,
476    rustc_middle::ty::SymbolName,
477    rustc_middle::ty::TraitRef,
478    rustc_middle::ty::Ty,
479    rustc_middle::ty::UnevaluatedConst,
480    rustc_middle::ty::ValTree,
481    rustc_middle::ty::VtblEntry,
482}
rustc_middle/query/erase.rs

rustc_middle/query/
erase.rs
