rustc_middle/query/
erase.rs

1//! To improve compile times and code size for the compiler itself, query
2//! values are "erased" in some contexts (e.g. inside in-memory cache types),
3//! to reduce the number of generic instantiations created during codegen.
4//!
5//! See <https://github.com/rust-lang/rust/pull/151715> for some bootstrap-time
6//! and performance benchmarks.
7
8use std::ffi::OsStr;
9use std::intrinsics::transmute_unchecked;
10use std::mem::MaybeUninit;
11
12use rustc_ast::tokenstream::TokenStream;
13use rustc_span::ErrorGuaranteed;
14use rustc_span::source_map::Spanned;
15
16use crate::mir::interpret::EvalToValTreeResult;
17use crate::mir::mono::{MonoItem, NormalizationErrorInMono};
18use crate::query::CyclePlaceholder;
19use crate::traits::solve;
20use crate::ty::adjustment::CoerceUnsizedInfo;
21use crate::ty::{self, Ty, TyCtxt};
22use crate::{mir, traits};
23
24/// Internal implementation detail of [`Erased`].
25#[derive(#[automatically_derived]
impl<Storage: ::core::marker::Copy + Copy> ::core::marker::Copy for
    ErasedData<Storage> {
}Copy, #[automatically_derived]
impl<Storage: ::core::clone::Clone + Copy> ::core::clone::Clone for
    ErasedData<Storage> {
    #[inline]
    fn clone(&self) -> ErasedData<Storage> {
        ErasedData { data: ::core::clone::Clone::clone(&self.data) }
    }
}Clone)]
26pub struct ErasedData<Storage: Copy> {
27    /// We use `MaybeUninit` here to make sure it's legal to store a transmuted
28    /// value that isn't actually of type `Storage`.
29    data: MaybeUninit<Storage>,
30}
31
32/// Trait for types that can be erased into [`Erased<Self>`].
33///
34/// Erasing and unerasing values is performed by [`erase_val`] and [`restore_val`].
35///
36/// FIXME: This whole trait could potentially be replaced by `T: Copy` and the
37/// storage type `[u8; size_of::<T>()]` when support for that is more mature.
38pub trait Erasable: Copy {
39    /// Storage type to used for erased values of this type.
40    /// Should be `[u8; N]`, where N is equal to `size_of::<Self>`.
41    ///
42    /// [`ErasedData`] wraps this storage type in `MaybeUninit` to ensure that
43    /// transmutes to/from erased storage are well-defined.
44    type Storage: Copy;
45}
46
47/// A value of `T` that has been "erased" into some opaque storage type.
48///
49/// This is helpful for reducing the number of concrete instantiations needed
50/// during codegen when building the compiler.
51///
52/// Using an opaque type alias allows the type checker to enforce that
53/// `Erased<T>` and `Erased<U>` are still distinct types, while allowing
54/// monomorphization to see that they might actually use the same storage type.
55pub type Erased<T: Erasable> = ErasedData<impl Copy>;
56
57/// Erases a value of type `T` into `Erased<T>`.
58///
59/// `Erased<T>` and `Erased<U>` are type-checked as distinct types, but codegen
60/// can see whether they actually have the same storage type.
61///
62/// FIXME: This might have soundness issues with erasable types that don't
63/// implement the same auto-traits as `[u8; _]`; see
64/// <https://github.com/rust-lang/rust/pull/151715#discussion_r2740113250>
65#[inline(always)]
66#[define_opaque(Erased)]
67pub fn erase_val<T: Erasable>(value: T) -> Erased<T> {
68    // Ensure the sizes match
69    const {
70        if size_of::<T>() != size_of::<T::Storage>() {
71            {
    ::core::panicking::panic_fmt(format_args!("size of T must match erased type <T as Erasable>::Storage"));
}panic!("size of T must match erased type <T as Erasable>::Storage")
72        }
73    };
74
75    ErasedData::<<T as Erasable>::Storage> {
76        // `transmute_unchecked` is needed here because it does not have `transmute`'s size check
77        // (and thus allows to transmute between `T` and `MaybeUninit<T::Storage>`) (we do the size
78        // check ourselves in the `const` block above).
79        //
80        // `transmute_copy` is also commonly used for this (and it would work here since
81        // `Erasable: Copy`), but `transmute_unchecked` better explains the intent.
82        //
83        // SAFETY: It is safe to transmute to MaybeUninit for types with the same sizes.
84        data: unsafe { transmute_unchecked::<T, MaybeUninit<T::Storage>>(value) },
85    }
86}
87
88/// Restores an erased value to its real type.
89///
90/// This relies on the fact that `Erased<T>` and `Erased<U>` are type-checked
91/// as distinct types, even if they use the same storage type.
92#[inline(always)]
93#[define_opaque(Erased)]
94pub fn restore_val<T: Erasable>(erased_value: Erased<T>) -> T {
95    let ErasedData { data }: ErasedData<<T as Erasable>::Storage> = erased_value;
96    // See comment in `erase_val` for why we use `transmute_unchecked`.
97    //
98    // SAFETY: Due to the use of impl Trait in `Erased` the only way to safely create an instance
99    // of `Erased` is to call `erase_val`, so we know that `erased_value.data` is a valid instance
100    // of `T` of the right size.
101    unsafe { transmute_unchecked::<MaybeUninit<T::Storage>, T>(data) }
102}
103
104// FIXME(#151565): Using `T: ?Sized` here should let us remove the separate
105// impls for fat reference types.
106impl<T> Erasable for &'_ T {
107    type Storage = [u8; size_of::<&'static ()>()];
108}
109
110impl<T> Erasable for &'_ [T] {
111    type Storage = [u8; size_of::<&'static [()]>()];
112}
113
114impl Erasable for &'_ OsStr {
115    type Storage = [u8; size_of::<&'static OsStr>()];
116}
117
118impl<T> Erasable for &'_ ty::List<T> {
119    type Storage = [u8; size_of::<&'static ty::List<()>>()];
120}
121
122impl<T> Erasable for &'_ ty::ListWithCachedTypeInfo<T> {
123    type Storage = [u8; size_of::<&'static ty::ListWithCachedTypeInfo<()>>()];
124}
125
126impl<I: rustc_index::Idx, T> Erasable for &'_ rustc_index::IndexSlice<I, T> {
127    type Storage = [u8; size_of::<&'static rustc_index::IndexSlice<u32, ()>>()];
128}
129
130impl<T> Erasable for Result<&'_ T, traits::query::NoSolution> {
131    type Storage = [u8; size_of::<Result<&'static (), traits::query::NoSolution>>()];
132}
133
134impl<T> Erasable for Result<&'_ [T], traits::query::NoSolution> {
135    type Storage = [u8; size_of::<Result<&'static [()], traits::query::NoSolution>>()];
136}
137
138impl<T> Erasable for Result<&'_ T, rustc_errors::ErrorGuaranteed> {
139    type Storage = [u8; size_of::<Result<&'static (), rustc_errors::ErrorGuaranteed>>()];
140}
141
142impl<T> Erasable for Result<&'_ [T], rustc_errors::ErrorGuaranteed> {
143    type Storage = [u8; size_of::<Result<&'static [()], rustc_errors::ErrorGuaranteed>>()];
144}
145
146impl<T> Erasable for Result<&'_ T, traits::CodegenObligationError> {
147    type Storage = [u8; size_of::<Result<&'static (), traits::CodegenObligationError>>()];
148}
149
150impl<T> Erasable for Result<&'_ T, &'_ ty::layout::FnAbiError<'_>> {
151    type Storage = [u8; size_of::<Result<&'static (), &'static ty::layout::FnAbiError<'static>>>()];
152}
153
154impl<T> Erasable for Result<(&'_ T, crate::thir::ExprId), rustc_errors::ErrorGuaranteed> {
155    type Storage = [u8; size_of::<
156        Result<(&'static (), crate::thir::ExprId), rustc_errors::ErrorGuaranteed>,
157    >()];
158}
159
160impl Erasable for Result<Option<ty::Instance<'_>>, rustc_errors::ErrorGuaranteed> {
161    type Storage =
162        [u8; size_of::<Result<Option<ty::Instance<'static>>, rustc_errors::ErrorGuaranteed>>()];
163}
164
165impl Erasable for Result<CoerceUnsizedInfo, rustc_errors::ErrorGuaranteed> {
166    type Storage = [u8; size_of::<Result<CoerceUnsizedInfo, rustc_errors::ErrorGuaranteed>>()];
167}
168
169impl Erasable
170    for Result<Option<ty::EarlyBinder<'_, ty::Const<'_>>>, rustc_errors::ErrorGuaranteed>
171{
172    type Storage = [u8; size_of::<
173        Result<Option<ty::EarlyBinder<'static, ty::Const<'static>>>, rustc_errors::ErrorGuaranteed>,
174    >()];
175}
176
177impl Erasable for Result<ty::GenericArg<'_>, traits::query::NoSolution> {
178    type Storage = [u8; size_of::<Result<ty::GenericArg<'static>, traits::query::NoSolution>>()];
179}
180
181impl Erasable for Result<bool, &ty::layout::LayoutError<'_>> {
182    type Storage = [u8; size_of::<Result<bool, &'static ty::layout::LayoutError<'static>>>()];
183}
184
185impl Erasable for Result<rustc_abi::TyAndLayout<'_, Ty<'_>>, &ty::layout::LayoutError<'_>> {
186    type Storage = [u8; size_of::<
187        Result<
188            rustc_abi::TyAndLayout<'static, Ty<'static>>,
189            &'static ty::layout::LayoutError<'static>,
190        >,
191    >()];
192}
193
194impl Erasable for Result<mir::ConstAlloc<'_>, mir::interpret::ErrorHandled> {
195    type Storage =
196        [u8; size_of::<Result<mir::ConstAlloc<'static>, mir::interpret::ErrorHandled>>()];
197}
198
199impl Erasable for Result<mir::ConstValue, mir::interpret::ErrorHandled> {
200    type Storage = [u8; size_of::<Result<mir::ConstValue, mir::interpret::ErrorHandled>>()];
201}
202
203impl Erasable for Option<(mir::ConstValue, Ty<'_>)> {
204    type Storage = [u8; size_of::<Option<(mir::ConstValue, Ty<'_>)>>()];
205}
206
207impl Erasable for EvalToValTreeResult<'_> {
208    type Storage = [u8; size_of::<EvalToValTreeResult<'static>>()];
209}
210
211impl Erasable for Result<&'_ ty::List<Ty<'_>>, ty::util::AlwaysRequiresDrop> {
212    type Storage =
213        [u8; size_of::<Result<&'static ty::List<Ty<'static>>, ty::util::AlwaysRequiresDrop>>()];
214}
215
216impl Erasable for Result<ty::EarlyBinder<'_, Ty<'_>>, CyclePlaceholder> {
217    type Storage = [u8; size_of::<Result<ty::EarlyBinder<'static, Ty<'_>>, CyclePlaceholder>>()];
218}
219
220impl Erasable
221    for Result<(&'_ [Spanned<MonoItem<'_>>], &'_ [Spanned<MonoItem<'_>>]), NormalizationErrorInMono>
222{
223    type Storage = [u8; size_of::<
224        Result<
225            (&'static [Spanned<MonoItem<'static>>], &'static [Spanned<MonoItem<'static>>]),
226            NormalizationErrorInMono,
227        >,
228    >()];
229}
230
231impl Erasable for Result<&'_ TokenStream, ()> {
232    type Storage = [u8; size_of::<Result<&'static TokenStream, ()>>()];
233}
234
235impl<T> Erasable for Option<&'_ T> {
236    type Storage = [u8; size_of::<Option<&'static ()>>()];
237}
238
239impl<T> Erasable for Option<&'_ [T]> {
240    type Storage = [u8; size_of::<Option<&'static [()]>>()];
241}
242
243impl Erasable for Option<&'_ OsStr> {
244    type Storage = [u8; size_of::<Option<&'static OsStr>>()];
245}
246
247impl Erasable for Option<mir::DestructuredConstant<'_>> {
248    type Storage = [u8; size_of::<Option<mir::DestructuredConstant<'static>>>()];
249}
250
251impl Erasable for ty::ImplTraitHeader<'_> {
252    type Storage = [u8; size_of::<ty::ImplTraitHeader<'static>>()];
253}
254
255impl Erasable for Option<ty::EarlyBinder<'_, Ty<'_>>> {
256    type Storage = [u8; size_of::<Option<ty::EarlyBinder<'static, Ty<'static>>>>()];
257}
258
259impl Erasable for rustc_hir::MaybeOwner<'_> {
260    type Storage = [u8; size_of::<rustc_hir::MaybeOwner<'static>>()];
261}
262
263impl<T: Erasable> Erasable for ty::EarlyBinder<'_, T> {
264    type Storage = T::Storage;
265}
266
267impl Erasable for ty::Binder<'_, ty::FnSig<'_>> {
268    type Storage = [u8; size_of::<ty::Binder<'static, ty::FnSig<'static>>>()];
269}
270
271impl Erasable for ty::Binder<'_, ty::CoroutineWitnessTypes<TyCtxt<'_>>> {
272    type Storage =
273        [u8; size_of::<ty::Binder<'static, ty::CoroutineWitnessTypes<TyCtxt<'static>>>>()];
274}
275
276impl Erasable for ty::Binder<'_, &'_ ty::List<Ty<'_>>> {
277    type Storage = [u8; size_of::<ty::Binder<'static, &'static ty::List<Ty<'static>>>>()];
278}
279
280impl<T0, T1> Erasable for (&'_ T0, &'_ T1) {
281    type Storage = [u8; size_of::<(&'static (), &'static ())>()];
282}
283
284impl<T0> Erasable for (solve::QueryResult<'_>, &'_ T0) {
285    type Storage = [u8; size_of::<(solve::QueryResult<'static>, &'static ())>()];
286}
287
288impl<T0, T1> Erasable for (&'_ T0, &'_ [T1]) {
289    type Storage = [u8; size_of::<(&'static (), &'static [()])>()];
290}
291
292impl<T0, T1> Erasable for (&'_ [T0], &'_ [T1]) {
293    type Storage = [u8; size_of::<(&'static [()], &'static [()])>()];
294}
295
296impl<T0> Erasable for (&'_ T0, Result<(), ErrorGuaranteed>) {
297    type Storage = [u8; size_of::<(&'static (), Result<(), ErrorGuaranteed>)>()];
298}
299
300macro_rules! impl_erasable_for_simple_types {
301    ($($ty:ty),+ $(,)?) => {
302        $(
303            impl Erasable for $ty {
304                type Storage = [u8; size_of::<$ty>()];
305            }
306        )*
307    }
308}
309
310// For concrete types with no lifetimes, the erased storage for `Foo` is
311// `[u8; size_of::<Foo>()]`.
312impl Erasable for usize {
    type Storage = [u8; size_of::<usize>()];
}impl_erasable_for_simple_types! {
313    // FIXME(#151565): Add `tidy-alphabetical-{start,end}` and sort this.
314    (),
315    bool,
316    Option<(rustc_span::def_id::DefId, rustc_session::config::EntryFnType)>,
317    Option<rustc_ast::expand::allocator::AllocatorKind>,
318    Option<rustc_hir::ConstStability>,
319    Option<rustc_hir::DefaultBodyStability>,
320    Option<rustc_hir::Stability>,
321    Option<rustc_data_structures::svh::Svh>,
322    Option<rustc_hir::def::DefKind>,
323    Option<rustc_hir::CoroutineKind>,
324    Option<rustc_hir::HirId>,
325    Option<rustc_middle::middle::stability::DeprecationEntry>,
326    Option<rustc_middle::ty::AsyncDestructor>,
327    Option<rustc_middle::ty::Destructor>,
328    Option<rustc_middle::ty::ImplTraitInTraitData>,
329    Option<rustc_middle::ty::ScalarInt>,
330    Option<rustc_span::def_id::CrateNum>,
331    Option<rustc_span::def_id::DefId>,
332    Option<rustc_span::def_id::LocalDefId>,
333    Option<rustc_span::Span>,
334    Option<rustc_abi::FieldIdx>,
335    Option<rustc_target::spec::PanicStrategy>,
336    Option<usize>,
337    Option<rustc_middle::ty::IntrinsicDef>,
338    Option<rustc_abi::Align>,
339    Result<(), rustc_errors::ErrorGuaranteed>,
340    Result<(), rustc_middle::traits::query::NoSolution>,
341    Result<rustc_middle::traits::EvaluationResult, rustc_middle::traits::OverflowError>,
342    rustc_abi::ReprOptions,
343    rustc_ast::expand::allocator::AllocatorKind,
344    rustc_hir::DefaultBodyStability,
345    rustc_hir::attrs::Deprecation,
346    rustc_hir::attrs::EiiDecl,
347    rustc_hir::attrs::EiiImpl,
348    rustc_data_structures::svh::Svh,
349    rustc_errors::ErrorGuaranteed,
350    rustc_hir::Constness,
351    rustc_hir::ConstStability,
352    rustc_hir::def_id::DefId,
353    rustc_hir::def_id::DefIndex,
354    rustc_hir::def_id::LocalDefId,
355    rustc_hir::def_id::LocalModDefId,
356    rustc_hir::def::DefKind,
357    rustc_hir::Defaultness,
358    rustc_hir::definitions::DefKey,
359    rustc_hir::CoroutineKind,
360    rustc_hir::HirId,
361    rustc_hir::IsAsync,
362    rustc_hir::ItemLocalId,
363    rustc_hir::LangItem,
364    rustc_hir::OpaqueTyOrigin<rustc_hir::def_id::DefId>,
365    rustc_hir::OwnerId,
366    rustc_hir::Stability,
367    rustc_hir::Upvar,
368    rustc_index::bit_set::FiniteBitSet<u32>,
369    rustc_middle::middle::deduced_param_attrs::DeducedParamAttrs,
370    rustc_middle::middle::dependency_format::Linkage,
371    rustc_middle::middle::exported_symbols::SymbolExportInfo,
372    rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
373    rustc_middle::middle::resolve_bound_vars::ResolvedArg,
374    rustc_middle::middle::stability::DeprecationEntry,
375    rustc_middle::mir::ConstQualifs,
376    rustc_middle::mir::ConstValue,
377    rustc_middle::mir::interpret::AllocId,
378    rustc_middle::mir::interpret::CtfeProvenance,
379    rustc_middle::mir::interpret::ErrorHandled,
380    rustc_middle::thir::ExprId,
381    rustc_middle::traits::CodegenObligationError,
382    rustc_middle::traits::EvaluationResult,
383    rustc_middle::traits::OverflowError,
384    rustc_middle::traits::query::NoSolution,
385    rustc_middle::traits::WellFormedLoc,
386    rustc_middle::ty::adjustment::CoerceUnsizedInfo,
387    rustc_middle::ty::AssocItem,
388    rustc_middle::ty::AssocContainer,
389    rustc_middle::ty::Asyncness,
390    rustc_middle::ty::AsyncDestructor,
391    rustc_middle::ty::AnonConstKind,
392    rustc_middle::ty::Destructor,
393    rustc_middle::ty::fast_reject::SimplifiedType,
394    rustc_middle::ty::ImplPolarity,
395    rustc_middle::ty::Representability,
396    rustc_middle::ty::UnusedGenericParams,
397    rustc_middle::ty::util::AlwaysRequiresDrop,
398    rustc_middle::ty::Visibility<rustc_span::def_id::DefId>,
399    rustc_middle::middle::codegen_fn_attrs::SanitizerFnAttrs,
400    rustc_session::config::CrateType,
401    rustc_session::config::EntryFnType,
402    rustc_session::config::OptLevel,
403    rustc_session::config::SymbolManglingVersion,
404    rustc_session::cstore::CrateDepKind,
405    rustc_session::cstore::ExternCrate,
406    rustc_session::cstore::LinkagePreference,
407    rustc_session::Limits,
408    rustc_session::lint::LintExpectationId,
409    rustc_span::def_id::CrateNum,
410    rustc_span::def_id::DefPathHash,
411    rustc_span::ExpnHash,
412    rustc_span::ExpnId,
413    rustc_span::Span,
414    rustc_span::Symbol,
415    rustc_span::Ident,
416    rustc_target::spec::PanicStrategy,
417    rustc_type_ir::Variance,
418    u32,
419    usize,
420}
421
422macro_rules! impl_erasable_for_single_lifetime_types {
423    ($($($fake_path:ident)::+),+ $(,)?) => {
424        $(
425            impl<'tcx> Erasable for $($fake_path)::+<'tcx> {
426                type Storage = [u8; size_of::<$($fake_path)::+<'static>>()];
427            }
428        )*
429    }
430}
431
432// For types containing a single lifetime and no other generics, e.g.
433// `Foo<'tcx>`, the erased storage is `[u8; size_of::<Foo<'static>>()]`.
434//
435// FIXME(#151565): Some of the hand-written impls above that only use one
436// lifetime can probably be migrated here.
437impl<'tcx> Erasable for rustc_middle::ty::VtblEntry<'tcx> {
    type Storage = [u8; size_of::<rustc_middle::ty::VtblEntry<'static>>()];
}impl_erasable_for_single_lifetime_types! {
438    // FIXME(#151565): Add `tidy-alphabetical-{start,end}` and sort this.
439    rustc_middle::middle::exported_symbols::ExportedSymbol,
440    rustc_middle::mir::Const,
441    rustc_middle::mir::DestructuredConstant,
442    rustc_middle::mir::ConstAlloc,
443    rustc_middle::mir::interpret::GlobalId,
444    rustc_middle::mir::interpret::LitToConstInput,
445    rustc_middle::mir::interpret::EvalStaticInitializerRawResult,
446    rustc_middle::mir::mono::MonoItemPartitions,
447    rustc_middle::traits::query::MethodAutoderefStepsResult,
448    rustc_middle::traits::query::type_op::AscribeUserType,
449    rustc_middle::traits::query::type_op::Eq,
450    rustc_middle::traits::query::type_op::ProvePredicate,
451    rustc_middle::traits::query::type_op::Subtype,
452    rustc_middle::ty::AdtDef,
453    rustc_middle::ty::AliasTy,
454    rustc_middle::ty::ClauseKind,
455    rustc_middle::ty::ClosureTypeInfo,
456    rustc_middle::ty::Const,
457    rustc_middle::ty::DestructuredAdtConst,
458    rustc_middle::ty::ExistentialTraitRef,
459    rustc_middle::ty::FnSig,
460    rustc_middle::ty::GenericArg,
461    rustc_middle::ty::GenericPredicates,
462    rustc_middle::ty::ConstConditions,
463    rustc_middle::ty::inhabitedness::InhabitedPredicate,
464    rustc_middle::ty::Instance,
465    rustc_middle::ty::BoundVariableKind,
466    rustc_middle::ty::InstanceKind,
467    rustc_middle::ty::layout::FnAbiError,
468    rustc_middle::ty::layout::LayoutError,
469    rustc_middle::ty::ParamEnv,
470    rustc_middle::ty::TypingEnv,
471    rustc_middle::ty::Predicate,
472    rustc_middle::ty::SymbolName,
473    rustc_middle::ty::TraitRef,
474    rustc_middle::ty::Ty,
475    rustc_middle::ty::UnevaluatedConst,
476    rustc_middle::ty::ValTree,
477    rustc_middle::ty::VtblEntry,
478}
rustc_middle/query/erase.rs

rustc_middle/query/
erase.rs
