rustc_middle/query/
erase.rs

1//! To improve compile times and code size for the compiler itself, query
2//! values are "erased" in some contexts (e.g. inside in-memory cache types),
3//! to reduce the number of generic instantiations created during codegen.
4//!
5//! See <https://github.com/rust-lang/rust/pull/151715> for some bootstrap-time
6//! and performance benchmarks.
7
8use std::ffi::OsStr;
9use std::intrinsics::transmute_unchecked;
10use std::mem::MaybeUninit;
11
12use rustc_ast::tokenstream::TokenStream;
13use rustc_span::{ErrorGuaranteed, Spanned};
14
15use crate::mir::interpret::EvalToValTreeResult;
16use crate::mir::mono::{MonoItem, NormalizationErrorInMono};
17use crate::query::plumbing::CyclePlaceholder;
18use crate::traits::solve;
19use crate::ty::adjustment::CoerceUnsizedInfo;
20use crate::ty::{self, Ty, TyCtxt};
21use crate::{mir, traits};
22
23/// Internal implementation detail of [`Erased`].
24#[derive(#[automatically_derived]
impl<Storage: ::core::marker::Copy + Copy> ::core::marker::Copy for
    ErasedData<Storage> {
}Copy, #[automatically_derived]
impl<Storage: ::core::clone::Clone + Copy> ::core::clone::Clone for
    ErasedData<Storage> {
    #[inline]
    fn clone(&self) -> ErasedData<Storage> {
        ErasedData { data: ::core::clone::Clone::clone(&self.data) }
    }
}Clone)]
25pub struct ErasedData<Storage: Copy> {
26    /// We use `MaybeUninit` here to make sure it's legal to store a transmuted
27    /// value that isn't actually of type `Storage`.
28    data: MaybeUninit<Storage>,
29}
30
31/// Trait for types that can be erased into [`Erased<Self>`].
32///
33/// Erasing and unerasing values is performed by [`erase_val`] and [`restore_val`].
34///
35/// FIXME: This whole trait could potentially be replaced by `T: Copy` and the
36/// storage type `[u8; size_of::<T>()]` when support for that is more mature.
37pub trait Erasable: Copy {
38    /// Storage type to used for erased values of this type.
39    /// Should be `[u8; N]`, where N is equal to `size_of::<Self>`.
40    ///
41    /// [`ErasedData`] wraps this storage type in `MaybeUninit` to ensure that
42    /// transmutes to/from erased storage are well-defined.
43    type Storage: Copy;
44}
45
46/// A value of `T` that has been "erased" into some opaque storage type.
47///
48/// This is helpful for reducing the number of concrete instantiations needed
49/// during codegen when building the compiler.
50///
51/// Using an opaque type alias allows the type checker to enforce that
52/// `Erased<T>` and `Erased<U>` are still distinct types, while allowing
53/// monomorphization to see that they might actually use the same storage type.
54pub type Erased<T: Erasable> = ErasedData<impl Copy>;
55
56/// Erases a value of type `T` into `Erased<T>`.
57///
58/// `Erased<T>` and `Erased<U>` are type-checked as distinct types, but codegen
59/// can see whether they actually have the same storage type.
60///
61/// FIXME: This might have soundness issues with erasable types that don't
62/// implement the same auto-traits as `[u8; _]`; see
63/// <https://github.com/rust-lang/rust/pull/151715#discussion_r2740113250>
64#[inline(always)]
65#[define_opaque(Erased)]
66pub fn erase_val<T: Erasable>(value: T) -> Erased<T> {
67    // Ensure the sizes match
68    const {
69        if size_of::<T>() != size_of::<T::Storage>() {
70            {
    ::core::panicking::panic_fmt(format_args!("size of T must match erased type <T as Erasable>::Storage"));
}panic!("size of T must match erased type <T as Erasable>::Storage")
71        }
72    };
73
74    ErasedData::<<T as Erasable>::Storage> {
75        // `transmute_unchecked` is needed here because it does not have `transmute`'s size check
76        // (and thus allows to transmute between `T` and `MaybeUninit<T::Storage>`) (we do the size
77        // check ourselves in the `const` block above).
78        //
79        // `transmute_copy` is also commonly used for this (and it would work here since
80        // `Erasable: Copy`), but `transmute_unchecked` better explains the intent.
81        //
82        // SAFETY: It is safe to transmute to MaybeUninit for types with the same sizes.
83        data: unsafe { transmute_unchecked::<T, MaybeUninit<T::Storage>>(value) },
84    }
85}
86
87/// Restores an erased value to its real type.
88///
89/// This relies on the fact that `Erased<T>` and `Erased<U>` are type-checked
90/// as distinct types, even if they use the same storage type.
91#[inline(always)]
92#[define_opaque(Erased)]
93pub fn restore_val<T: Erasable>(erased_value: Erased<T>) -> T {
94    let ErasedData { data }: ErasedData<<T as Erasable>::Storage> = erased_value;
95    // See comment in `erase_val` for why we use `transmute_unchecked`.
96    //
97    // SAFETY: Due to the use of impl Trait in `Erased` the only way to safely create an instance
98    // of `Erased` is to call `erase_val`, so we know that `erased_value.data` is a valid instance
99    // of `T` of the right size.
100    unsafe { transmute_unchecked::<MaybeUninit<T::Storage>, T>(data) }
101}
102
103// FIXME(#151565): Using `T: ?Sized` here should let us remove the separate
104// impls for fat reference types.
105impl<T> Erasable for &'_ T {
106    type Storage = [u8; size_of::<&'static ()>()];
107}
108
109impl<T> Erasable for &'_ [T] {
110    type Storage = [u8; size_of::<&'static [()]>()];
111}
112
113impl Erasable for &'_ OsStr {
114    type Storage = [u8; size_of::<&'static OsStr>()];
115}
116
117impl<T> Erasable for &'_ ty::List<T> {
118    type Storage = [u8; size_of::<&'static ty::List<()>>()];
119}
120
121impl<T> Erasable for &'_ ty::ListWithCachedTypeInfo<T> {
122    type Storage = [u8; size_of::<&'static ty::ListWithCachedTypeInfo<()>>()];
123}
124
125impl<I: rustc_index::Idx, T> Erasable for &'_ rustc_index::IndexSlice<I, T> {
126    type Storage = [u8; size_of::<&'static rustc_index::IndexSlice<u32, ()>>()];
127}
128
129impl<T> Erasable for Result<&'_ T, traits::query::NoSolution> {
130    type Storage = [u8; size_of::<Result<&'static (), traits::query::NoSolution>>()];
131}
132
133impl<T> Erasable for Result<&'_ [T], traits::query::NoSolution> {
134    type Storage = [u8; size_of::<Result<&'static [()], traits::query::NoSolution>>()];
135}
136
137impl<T> Erasable for Result<&'_ T, rustc_errors::ErrorGuaranteed> {
138    type Storage = [u8; size_of::<Result<&'static (), rustc_errors::ErrorGuaranteed>>()];
139}
140
141impl<T> Erasable for Result<&'_ [T], rustc_errors::ErrorGuaranteed> {
142    type Storage = [u8; size_of::<Result<&'static [()], rustc_errors::ErrorGuaranteed>>()];
143}
144
145impl<T> Erasable for Result<&'_ T, traits::CodegenObligationError> {
146    type Storage = [u8; size_of::<Result<&'static (), traits::CodegenObligationError>>()];
147}
148
149impl<T> Erasable for Result<&'_ T, &'_ ty::layout::FnAbiError<'_>> {
150    type Storage = [u8; size_of::<Result<&'static (), &'static ty::layout::FnAbiError<'static>>>()];
151}
152
153impl<T> Erasable for Result<(&'_ T, crate::thir::ExprId), rustc_errors::ErrorGuaranteed> {
154    type Storage = [u8; size_of::<
155        Result<(&'static (), crate::thir::ExprId), rustc_errors::ErrorGuaranteed>,
156    >()];
157}
158
159impl Erasable for Result<Option<ty::Instance<'_>>, rustc_errors::ErrorGuaranteed> {
160    type Storage =
161        [u8; size_of::<Result<Option<ty::Instance<'static>>, rustc_errors::ErrorGuaranteed>>()];
162}
163
164impl Erasable for Result<CoerceUnsizedInfo, rustc_errors::ErrorGuaranteed> {
165    type Storage = [u8; size_of::<Result<CoerceUnsizedInfo, rustc_errors::ErrorGuaranteed>>()];
166}
167
168impl Erasable
169    for Result<Option<ty::EarlyBinder<'_, ty::Const<'_>>>, rustc_errors::ErrorGuaranteed>
170{
171    type Storage = [u8; size_of::<
172        Result<Option<ty::EarlyBinder<'static, ty::Const<'static>>>, rustc_errors::ErrorGuaranteed>,
173    >()];
174}
175
176impl Erasable for Result<ty::GenericArg<'_>, traits::query::NoSolution> {
177    type Storage = [u8; size_of::<Result<ty::GenericArg<'static>, traits::query::NoSolution>>()];
178}
179
180impl Erasable for Result<bool, &ty::layout::LayoutError<'_>> {
181    type Storage = [u8; size_of::<Result<bool, &'static ty::layout::LayoutError<'static>>>()];
182}
183
184impl Erasable for Result<rustc_abi::TyAndLayout<'_, Ty<'_>>, &ty::layout::LayoutError<'_>> {
185    type Storage = [u8; size_of::<
186        Result<
187            rustc_abi::TyAndLayout<'static, Ty<'static>>,
188            &'static ty::layout::LayoutError<'static>,
189        >,
190    >()];
191}
192
193impl Erasable for Result<mir::ConstAlloc<'_>, mir::interpret::ErrorHandled> {
194    type Storage =
195        [u8; size_of::<Result<mir::ConstAlloc<'static>, mir::interpret::ErrorHandled>>()];
196}
197
198impl Erasable for Result<mir::ConstValue, mir::interpret::ErrorHandled> {
199    type Storage = [u8; size_of::<Result<mir::ConstValue, mir::interpret::ErrorHandled>>()];
200}
201
202impl Erasable for Option<(mir::ConstValue, Ty<'_>)> {
203    type Storage = [u8; size_of::<Option<(mir::ConstValue, Ty<'_>)>>()];
204}
205
206impl Erasable for EvalToValTreeResult<'_> {
207    type Storage = [u8; size_of::<EvalToValTreeResult<'static>>()];
208}
209
210impl Erasable for Result<&'_ ty::List<Ty<'_>>, ty::util::AlwaysRequiresDrop> {
211    type Storage =
212        [u8; size_of::<Result<&'static ty::List<Ty<'static>>, ty::util::AlwaysRequiresDrop>>()];
213}
214
215impl Erasable for Result<ty::EarlyBinder<'_, Ty<'_>>, CyclePlaceholder> {
216    type Storage = [u8; size_of::<Result<ty::EarlyBinder<'static, Ty<'_>>, CyclePlaceholder>>()];
217}
218
219impl Erasable
220    for Result<(&'_ [Spanned<MonoItem<'_>>], &'_ [Spanned<MonoItem<'_>>]), NormalizationErrorInMono>
221{
222    type Storage = [u8; size_of::<
223        Result<
224            (&'static [Spanned<MonoItem<'static>>], &'static [Spanned<MonoItem<'static>>]),
225            NormalizationErrorInMono,
226        >,
227    >()];
228}
229
230impl Erasable for Result<&'_ TokenStream, ()> {
231    type Storage = [u8; size_of::<Result<&'static TokenStream, ()>>()];
232}
233
234impl<T> Erasable for Option<&'_ T> {
235    type Storage = [u8; size_of::<Option<&'static ()>>()];
236}
237
238impl<T> Erasable for Option<&'_ [T]> {
239    type Storage = [u8; size_of::<Option<&'static [()]>>()];
240}
241
242impl Erasable for Option<&'_ OsStr> {
243    type Storage = [u8; size_of::<Option<&'static OsStr>>()];
244}
245
246impl Erasable for Option<mir::DestructuredConstant<'_>> {
247    type Storage = [u8; size_of::<Option<mir::DestructuredConstant<'static>>>()];
248}
249
250impl Erasable for ty::ImplTraitHeader<'_> {
251    type Storage = [u8; size_of::<ty::ImplTraitHeader<'static>>()];
252}
253
254impl Erasable for Option<ty::EarlyBinder<'_, Ty<'_>>> {
255    type Storage = [u8; size_of::<Option<ty::EarlyBinder<'static, Ty<'static>>>>()];
256}
257
258impl Erasable for Option<ty::Value<'_>> {
259    type Storage = [u8; size_of::<Option<ty::Value<'static>>>()];
260}
261
262impl Erasable for rustc_hir::MaybeOwner<'_> {
263    type Storage = [u8; size_of::<rustc_hir::MaybeOwner<'static>>()];
264}
265
266impl<T: Erasable> Erasable for ty::EarlyBinder<'_, T> {
267    type Storage = T::Storage;
268}
269
270impl Erasable for ty::Binder<'_, ty::FnSig<'_>> {
271    type Storage = [u8; size_of::<ty::Binder<'static, ty::FnSig<'static>>>()];
272}
273
274impl Erasable for ty::Binder<'_, ty::CoroutineWitnessTypes<TyCtxt<'_>>> {
275    type Storage =
276        [u8; size_of::<ty::Binder<'static, ty::CoroutineWitnessTypes<TyCtxt<'static>>>>()];
277}
278
279impl Erasable for ty::Binder<'_, &'_ ty::List<Ty<'_>>> {
280    type Storage = [u8; size_of::<ty::Binder<'static, &'static ty::List<Ty<'static>>>>()];
281}
282
283impl<T0, T1> Erasable for (&'_ T0, &'_ T1) {
284    type Storage = [u8; size_of::<(&'static (), &'static ())>()];
285}
286
287impl<T0> Erasable for (solve::QueryResult<'_>, &'_ T0) {
288    type Storage = [u8; size_of::<(solve::QueryResult<'static>, &'static ())>()];
289}
290
291impl<T0, T1> Erasable for (&'_ T0, &'_ [T1]) {
292    type Storage = [u8; size_of::<(&'static (), &'static [()])>()];
293}
294
295impl<T0, T1> Erasable for (&'_ [T0], &'_ [T1]) {
296    type Storage = [u8; size_of::<(&'static [()], &'static [()])>()];
297}
298
299impl<T0> Erasable for (&'_ T0, Result<(), ErrorGuaranteed>) {
300    type Storage = [u8; size_of::<(&'static (), Result<(), ErrorGuaranteed>)>()];
301}
302
303macro_rules! impl_erasable_for_simple_types {
304    ($($ty:ty),+ $(,)?) => {
305        $(
306            impl Erasable for $ty {
307                type Storage = [u8; size_of::<$ty>()];
308            }
309        )*
310    }
311}
312
313// For concrete types with no lifetimes, the erased storage for `Foo` is
314// `[u8; size_of::<Foo>()]`.
315impl Erasable for usize {
    type Storage = [u8; size_of::<usize>()];
}impl_erasable_for_simple_types! {
316    // FIXME(#151565): Add `tidy-alphabetical-{start,end}` and sort this.
317    (),
318    bool,
319    Option<(rustc_span::def_id::DefId, rustc_session::config::EntryFnType)>,
320    Option<rustc_ast::expand::allocator::AllocatorKind>,
321    Option<rustc_hir::ConstStability>,
322    Option<rustc_hir::DefaultBodyStability>,
323    Option<rustc_hir::Stability>,
324    Option<rustc_data_structures::svh::Svh>,
325    Option<rustc_hir::def::DefKind>,
326    Option<rustc_hir::CoroutineKind>,
327    Option<rustc_hir::HirId>,
328    Option<rustc_middle::middle::stability::DeprecationEntry>,
329    Option<rustc_middle::ty::AsyncDestructor>,
330    Option<rustc_middle::ty::Destructor>,
331    Option<rustc_middle::ty::ImplTraitInTraitData>,
332    Option<rustc_middle::ty::ScalarInt>,
333    Option<rustc_span::def_id::CrateNum>,
334    Option<rustc_span::def_id::DefId>,
335    Option<rustc_span::def_id::LocalDefId>,
336    Option<rustc_span::Span>,
337    Option<rustc_abi::FieldIdx>,
338    Option<rustc_target::spec::PanicStrategy>,
339    Option<usize>,
340    Option<rustc_middle::ty::IntrinsicDef>,
341    Option<rustc_abi::Align>,
342    Result<(), rustc_errors::ErrorGuaranteed>,
343    Result<(), rustc_middle::traits::query::NoSolution>,
344    Result<rustc_middle::traits::EvaluationResult, rustc_middle::traits::OverflowError>,
345    rustc_abi::ReprOptions,
346    rustc_ast::expand::allocator::AllocatorKind,
347    rustc_hir::DefaultBodyStability,
348    rustc_hir::attrs::Deprecation,
349    rustc_hir::attrs::EiiDecl,
350    rustc_hir::attrs::EiiImpl,
351    rustc_data_structures::svh::Svh,
352    rustc_errors::ErrorGuaranteed,
353    rustc_hir::Constness,
354    rustc_hir::ConstStability,
355    rustc_hir::def_id::DefId,
356    rustc_hir::def_id::DefIndex,
357    rustc_hir::def_id::LocalDefId,
358    rustc_hir::def_id::LocalModDefId,
359    rustc_hir::def::DefKind,
360    rustc_hir::Defaultness,
361    rustc_hir::definitions::DefKey,
362    rustc_hir::CoroutineKind,
363    rustc_hir::HirId,
364    rustc_hir::IsAsync,
365    rustc_hir::ItemLocalId,
366    rustc_hir::LangItem,
367    rustc_hir::OpaqueTyOrigin<rustc_hir::def_id::DefId>,
368    rustc_hir::OwnerId,
369    rustc_hir::Stability,
370    rustc_hir::Upvar,
371    rustc_index::bit_set::FiniteBitSet<u32>,
372    rustc_middle::middle::deduced_param_attrs::DeducedParamAttrs,
373    rustc_middle::middle::dependency_format::Linkage,
374    rustc_middle::middle::exported_symbols::SymbolExportInfo,
375    rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
376    rustc_middle::middle::resolve_bound_vars::ResolvedArg,
377    rustc_middle::middle::stability::DeprecationEntry,
378    rustc_middle::mir::ConstQualifs,
379    rustc_middle::mir::ConstValue,
380    rustc_middle::mir::interpret::AllocId,
381    rustc_middle::mir::interpret::CtfeProvenance,
382    rustc_middle::mir::interpret::ErrorHandled,
383    rustc_middle::thir::ExprId,
384    rustc_middle::traits::CodegenObligationError,
385    rustc_middle::traits::EvaluationResult,
386    rustc_middle::traits::OverflowError,
387    rustc_middle::traits::query::NoSolution,
388    rustc_middle::traits::WellFormedLoc,
389    rustc_middle::ty::adjustment::CoerceUnsizedInfo,
390    rustc_middle::ty::AssocItem,
391    rustc_middle::ty::AssocContainer,
392    rustc_middle::ty::Asyncness,
393    rustc_middle::ty::AsyncDestructor,
394    rustc_middle::ty::AnonConstKind,
395    rustc_middle::ty::Destructor,
396    rustc_middle::ty::fast_reject::SimplifiedType,
397    rustc_middle::ty::ImplPolarity,
398    rustc_middle::ty::Representability,
399    rustc_middle::ty::UnusedGenericParams,
400    rustc_middle::ty::util::AlwaysRequiresDrop,
401    rustc_middle::ty::Visibility<rustc_span::def_id::DefId>,
402    rustc_middle::middle::codegen_fn_attrs::SanitizerFnAttrs,
403    rustc_session::config::CrateType,
404    rustc_session::config::EntryFnType,
405    rustc_session::config::OptLevel,
406    rustc_session::config::SymbolManglingVersion,
407    rustc_session::cstore::CrateDepKind,
408    rustc_session::cstore::ExternCrate,
409    rustc_session::cstore::LinkagePreference,
410    rustc_session::Limits,
411    rustc_session::lint::LintExpectationId,
412    rustc_span::def_id::CrateNum,
413    rustc_span::def_id::DefPathHash,
414    rustc_span::ExpnHash,
415    rustc_span::ExpnId,
416    rustc_span::Span,
417    rustc_span::Symbol,
418    rustc_span::Ident,
419    rustc_target::spec::PanicStrategy,
420    rustc_type_ir::Variance,
421    u32,
422    usize,
423}
424
425macro_rules! impl_erasable_for_single_lifetime_types {
426    ($($($fake_path:ident)::+),+ $(,)?) => {
427        $(
428            impl<'tcx> Erasable for $($fake_path)::+<'tcx> {
429                type Storage = [u8; size_of::<$($fake_path)::+<'static>>()];
430            }
431        )*
432    }
433}
434
435// For types containing a single lifetime and no other generics, e.g.
436// `Foo<'tcx>`, the erased storage is `[u8; size_of::<Foo<'static>>()]`.
437//
438// FIXME(#151565): Some of the hand-written impls above that only use one
439// lifetime can probably be migrated here.
440impl<'tcx> Erasable for rustc_middle::ty::VtblEntry<'tcx> {
    type Storage = [u8; size_of::<rustc_middle::ty::VtblEntry<'static>>()];
}impl_erasable_for_single_lifetime_types! {
441    // FIXME(#151565): Add `tidy-alphabetical-{start,end}` and sort this.
442    rustc_middle::middle::exported_symbols::ExportedSymbol,
443    rustc_middle::mir::Const,
444    rustc_middle::mir::DestructuredConstant,
445    rustc_middle::mir::ConstAlloc,
446    rustc_middle::mir::interpret::GlobalId,
447    rustc_middle::mir::interpret::EvalStaticInitializerRawResult,
448    rustc_middle::mir::mono::MonoItemPartitions,
449    rustc_middle::traits::query::MethodAutoderefStepsResult,
450    rustc_middle::traits::query::type_op::AscribeUserType,
451    rustc_middle::traits::query::type_op::Eq,
452    rustc_middle::traits::query::type_op::ProvePredicate,
453    rustc_middle::traits::query::type_op::Subtype,
454    rustc_middle::ty::AdtDef,
455    rustc_middle::ty::AliasTy,
456    rustc_middle::ty::ClauseKind,
457    rustc_middle::ty::ClosureTypeInfo,
458    rustc_middle::ty::Const,
459    rustc_middle::ty::DestructuredAdtConst,
460    rustc_middle::ty::ExistentialTraitRef,
461    rustc_middle::ty::FnSig,
462    rustc_middle::ty::GenericArg,
463    rustc_middle::ty::GenericPredicates,
464    rustc_middle::ty::ConstConditions,
465    rustc_middle::ty::inhabitedness::InhabitedPredicate,
466    rustc_middle::ty::Instance,
467    rustc_middle::ty::BoundVariableKind,
468    rustc_middle::ty::InstanceKind,
469    rustc_middle::ty::layout::FnAbiError,
470    rustc_middle::ty::layout::LayoutError,
471    rustc_middle::ty::LitToConstInput,
472    rustc_middle::ty::ParamEnv,
473    rustc_middle::ty::TypingEnv,
474    rustc_middle::ty::Predicate,
475    rustc_middle::ty::SymbolName,
476    rustc_middle::ty::TraitRef,
477    rustc_middle::ty::Ty,
478    rustc_middle::ty::UnevaluatedConst,
479    rustc_middle::ty::ValTree,
480    rustc_middle::ty::VtblEntry,
481}
rustc_middle/query/erase.rs

rustc_middle/query/
erase.rs
