Skip to main content

rustc_metadata/rmeta/
table.rs

1use rustc_hir::def::CtorOf;
2use rustc_index::Idx;
3
4use crate::rmeta::decoder::MetaBlob;
5use crate::rmeta::remapped_def_id::RemappedRawDefId;
6use crate::rmeta::*;
7
8const SPARSE_BLOCK_LEN: usize = 64;
9const SPARSE_INDEX_BYTES: usize = 12;
10
11pub(super) trait IsDefault: Default {
12    fn is_default(&self) -> bool;
13}
14
15impl<T> IsDefault for Option<T> {
16    fn is_default(&self) -> bool {
17        self.is_none()
18    }
19}
20
21impl IsDefault for AttrFlags {
22    fn is_default(&self) -> bool {
23        self.is_empty()
24    }
25}
26
27impl IsDefault for bool {
28    fn is_default(&self) -> bool {
29        !self
30    }
31}
32
33impl IsDefault for u32 {
34    fn is_default(&self) -> bool {
35        *self == 0
36    }
37}
38
39impl IsDefault for u64 {
40    fn is_default(&self) -> bool {
41        *self == 0
42    }
43}
44
45impl<T> IsDefault for LazyArray<T> {
46    fn is_default(&self) -> bool {
47        self.num_elems == 0
48    }
49}
50
51/// Helper trait, for encoding to, and decoding from, a fixed number of bytes.
52/// Used mainly for Lazy positions and lengths.
53///
54/// Invariant: `Self::default()` should encode as `[0; BYTE_LEN]`,
55/// but this has no impact on safety.
56/// In debug builds, this invariant is checked in `[TableBuilder::set]`
57pub(super) trait FixedSizeEncoding: IsDefault {
58    /// This should be `[u8; BYTE_LEN]`;
59    /// Cannot use an associated `const BYTE_LEN: usize` instead due to const eval limitations.
60    type ByteArray;
61
62    fn from_bytes(b: &Self::ByteArray) -> Self;
63    fn write_to_bytes(self, b: &mut Self::ByteArray);
64}
65
66impl FixedSizeEncoding for u64 {
67    type ByteArray = [u8; 8];
68
69    #[inline]
70    fn from_bytes(b: &[u8; 8]) -> Self {
71        Self::from_le_bytes(*b)
72    }
73
74    #[inline]
75    fn write_to_bytes(self, b: &mut [u8; 8]) {
76        *b = self.to_le_bytes();
77    }
78}
79
80macro_rules! fixed_size_enum {
81    ($ty:ty { $(($($pat:tt)*))* } $( unreachable { $(($($upat:tt)*))+ } )?) => {
82        impl FixedSizeEncoding for Option<$ty> {
83            type ByteArray = [u8;1];
84
85            #[inline]
86            fn from_bytes(b: &[u8;1]) -> Self {
87                use $ty::*;
88                if b[0] == 0 {
89                    return None;
90                }
91                match b[0] - 1 {
92                    $(${index()} => Some($($pat)*),)*
93                    _ => panic!("Unexpected {} code: {:?}", stringify!($ty), b[0]),
94                }
95            }
96
97            #[inline]
98            fn write_to_bytes(self, b: &mut [u8;1]) {
99                use $ty::*;
100                b[0] = match self {
101                    None => unreachable!(),
102                    $(Some($($pat)*) => 1 + ${index()},)*
103                    $(Some($($($upat)*)|+) => unreachable!(),)?
104                }
105            }
106        }
107    }
108}
109
110macro_rules! defaulted_enum {
111    ($ty:ty { $(($($pat:tt)*))* } $( unreachable { $(($($upat:tt)*))+ } )?) => {
112        impl FixedSizeEncoding for $ty {
113            type ByteArray = [u8; 1];
114
115            #[inline]
116            fn from_bytes(b: &[u8; 1]) -> Self {
117                use $ty::*;
118                let val = match b[0] {
119                    $(${index()} => $($pat)*,)*
120                    _ => panic!("Unexpected {} code: {:?}", stringify!($ty), b[0]),
121                };
122                // Make sure the first entry is always the default value,
123                // and none of the other values are the default value
124                debug_assert_ne!((b[0] != 0), IsDefault::is_default(&val));
125                val
126            }
127
128            #[inline]
129            fn write_to_bytes(self, b: &mut [u8; 1]) {
130                debug_assert!(!IsDefault::is_default(&self));
131                use $ty::*;
132                b[0] = match self {
133                    $($($pat)* => ${index()},)*
134                    $($($($upat)*)|+ => unreachable!(),)?
135                };
136                debug_assert_ne!(b[0], 0);
137            }
138        }
139        impl IsDefault for $ty {
140            fn is_default(&self) -> bool {
141                <$ty as Default>::default() == *self
142            }
143        }
144    }
145}
146
147// Ensure that we get a compilation error if MacroKinds gets extended without updating metadata.
148const _: () = if !MacroKinds::DERIVE_ATTR_BANG.is_all() {
    ::core::panicking::panic("assertion failed: MacroKinds::DERIVE_ATTR_BANG.is_all()")
}assert!(MacroKinds::DERIVE_ATTR_BANG.is_all());
149
150impl FixedSizeEncoding for Option<DefKind> {
    type ByteArray = [u8; 1];
    #[inline]
    fn from_bytes(b: &[u8; 1]) -> Self {
        use DefKind::*;
        if b[0] == 0 { return None; }
        match b[0] - 1 {
            0 => Some(Mod),
            1 => Some(Struct),
            2 => Some(Union),
            3 => Some(Enum),
            4 => Some(Variant),
            5 => Some(Trait),
            6 => Some(TyAlias),
            7 => Some(ForeignTy),
            8 => Some(TraitAlias),
            9 => Some(AssocTy),
            10 => Some(TyParam),
            11 => Some(Fn),
            12 => Some(Const),
            13 => Some(ConstParam),
            14 => Some(AssocFn),
            15 => Some(AssocConst),
            16 => Some(ExternCrate),
            17 => Some(Use),
            18 => Some(ForeignMod),
            19 => Some(AnonConst),
            20 => Some(OpaqueTy),
            21 => Some(Field),
            22 => Some(LifetimeParam),
            23 => Some(GlobalAsm),
            24 => Some(Impl { of_trait: false }),
            25 => Some(Impl { of_trait: true }),
            26 => Some(Closure),
            27 =>
                Some(Static {
                        safety: hir::Safety::Unsafe,
                        mutability: ast::Mutability::Not,
                        nested: false,
                    }),
            28 =>
                Some(Static {
                        safety: hir::Safety::Safe,
                        mutability: ast::Mutability::Not,
                        nested: false,
                    }),
            29 =>
                Some(Static {
                        safety: hir::Safety::Unsafe,
                        mutability: ast::Mutability::Mut,
                        nested: false,
                    }),
            30 =>
                Some(Static {
                        safety: hir::Safety::Safe,
                        mutability: ast::Mutability::Mut,
                        nested: false,
                    }),
            31 =>
                Some(Static {
                        safety: hir::Safety::Unsafe,
                        mutability: ast::Mutability::Not,
                        nested: true,
                    }),
            32 =>
                Some(Static {
                        safety: hir::Safety::Safe,
                        mutability: ast::Mutability::Not,
                        nested: true,
                    }),
            33 =>
                Some(Static {
                        safety: hir::Safety::Unsafe,
                        mutability: ast::Mutability::Mut,
                        nested: true,
                    }),
            34 =>
                Some(Static {
                        safety: hir::Safety::Safe,
                        mutability: ast::Mutability::Mut,
                        nested: true,
                    }),
            35 => Some(Ctor(CtorOf::Struct, CtorKind::Fn)),
            36 => Some(Ctor(CtorOf::Struct, CtorKind::Const)),
            37 => Some(Ctor(CtorOf::Variant, CtorKind::Fn)),
            38 => Some(Ctor(CtorOf::Variant, CtorKind::Const)),
            39 => Some(Macro(MacroKinds::BANG)),
            40 => Some(Macro(MacroKinds::ATTR)),
            41 => Some(Macro(MacroKinds::DERIVE)),
            42 => Some(Macro(MacroKinds::ATTR_BANG)),
            43 => Some(Macro(MacroKinds::DERIVE_ATTR)),
            44 => Some(Macro(MacroKinds::DERIVE_BANG)),
            45 => Some(Macro(MacroKinds::DERIVE_ATTR_BANG)),
            46 => Some(SyntheticCoroutineBody),
            47 => Some(TestBinderConstraints),
            _ => {
                ::core::panicking::panic_fmt(format_args!("Unexpected {0} code: {1:?}",
                        "DefKind", b[0]));
            }
        }
    }
    #[inline]
    fn write_to_bytes(self, b: &mut [u8; 1]) {
        use DefKind::*;
        b[0] =
            match self {
                None => ::core::panicking::unreachable_default_msg(),
                Some(Mod) => 1 + 0,
                Some(Struct) => 1 + 1,
                Some(Union) => 1 + 2,
                Some(Enum) => 1 + 3,
                Some(Variant) => 1 + 4,
                Some(Trait) => 1 + 5,
                Some(TyAlias) => 1 + 6,
                Some(ForeignTy) => 1 + 7,
                Some(TraitAlias) => 1 + 8,
                Some(AssocTy) => 1 + 9,
                Some(TyParam) => 1 + 10,
                Some(Fn) => 1 + 11,
                Some(Const) => 1 + 12,
                Some(ConstParam) => 1 + 13,
                Some(AssocFn) => 1 + 14,
                Some(AssocConst) => 1 + 15,
                Some(ExternCrate) => 1 + 16,
                Some(Use) => 1 + 17,
                Some(ForeignMod) => 1 + 18,
                Some(AnonConst) => 1 + 19,
                Some(OpaqueTy) => 1 + 20,
                Some(Field) => 1 + 21,
                Some(LifetimeParam) => 1 + 22,
                Some(GlobalAsm) => 1 + 23,
                Some(Impl { of_trait: false }) => 1 + 24,
                Some(Impl { of_trait: true }) => 1 + 25,
                Some(Closure) => 1 + 26,
                Some(Static {
                    safety: hir::Safety::Unsafe,
                    mutability: ast::Mutability::Not,
                    nested: false }) => 1 + 27,
                Some(Static {
                    safety: hir::Safety::Safe,
                    mutability: ast::Mutability::Not,
                    nested: false }) => 1 + 28,
                Some(Static {
                    safety: hir::Safety::Unsafe,
                    mutability: ast::Mutability::Mut,
                    nested: false }) => 1 + 29,
                Some(Static {
                    safety: hir::Safety::Safe,
                    mutability: ast::Mutability::Mut,
                    nested: false }) => 1 + 30,
                Some(Static {
                    safety: hir::Safety::Unsafe,
                    mutability: ast::Mutability::Not,
                    nested: true }) => 1 + 31,
                Some(Static {
                    safety: hir::Safety::Safe,
                    mutability: ast::Mutability::Not,
                    nested: true }) => 1 + 32,
                Some(Static {
                    safety: hir::Safety::Unsafe,
                    mutability: ast::Mutability::Mut,
                    nested: true }) => 1 + 33,
                Some(Static {
                    safety: hir::Safety::Safe,
                    mutability: ast::Mutability::Mut,
                    nested: true }) => 1 + 34,
                Some(Ctor(CtorOf::Struct, CtorKind::Fn)) => 1 + 35,
                Some(Ctor(CtorOf::Struct, CtorKind::Const)) => 1 + 36,
                Some(Ctor(CtorOf::Variant, CtorKind::Fn)) => 1 + 37,
                Some(Ctor(CtorOf::Variant, CtorKind::Const)) => 1 + 38,
                Some(Macro(MacroKinds::BANG)) => 1 + 39,
                Some(Macro(MacroKinds::ATTR)) => 1 + 40,
                Some(Macro(MacroKinds::DERIVE)) => 1 + 41,
                Some(Macro(MacroKinds::ATTR_BANG)) => 1 + 42,
                Some(Macro(MacroKinds::DERIVE_ATTR)) => 1 + 43,
                Some(Macro(MacroKinds::DERIVE_BANG)) => 1 + 44,
                Some(Macro(MacroKinds::DERIVE_ATTR_BANG)) => 1 + 45,
                Some(SyntheticCoroutineBody) => 1 + 46,
                Some(TestBinderConstraints) => 1 + 47,
                Some(Macro(_)) =>
                    ::core::panicking::unreachable_default_msg(),
            }
    }
}fixed_size_enum! {
151    DefKind {
152        ( Mod                                      )
153        ( Struct                                   )
154        ( Union                                    )
155        ( Enum                                     )
156        ( Variant                                  )
157        ( Trait                                    )
158        ( TyAlias                                  )
159        ( ForeignTy                                )
160        ( TraitAlias                               )
161        ( AssocTy                                  )
162        ( TyParam                                  )
163        ( Fn                                       )
164        ( Const                                    )
165        ( ConstParam                               )
166        ( AssocFn                                  )
167        ( AssocConst                               )
168        ( ExternCrate                              )
169        ( Use                                      )
170        ( ForeignMod                               )
171        ( AnonConst                                )
172        ( OpaqueTy                                 )
173        ( Field                                    )
174        ( LifetimeParam                            )
175        ( GlobalAsm                                )
176        ( Impl { of_trait: false }                 )
177        ( Impl { of_trait: true }                  )
178        ( Closure                                  )
179        ( Static { safety: hir::Safety::Unsafe, mutability: ast::Mutability::Not, nested: false } )
180        ( Static { safety: hir::Safety::Safe, mutability: ast::Mutability::Not, nested: false } )
181        ( Static { safety: hir::Safety::Unsafe, mutability: ast::Mutability::Mut, nested: false } )
182        ( Static { safety: hir::Safety::Safe, mutability: ast::Mutability::Mut, nested: false } )
183        ( Static { safety: hir::Safety::Unsafe, mutability: ast::Mutability::Not, nested: true } )
184        ( Static { safety: hir::Safety::Safe, mutability: ast::Mutability::Not, nested: true } )
185        ( Static { safety: hir::Safety::Unsafe, mutability: ast::Mutability::Mut, nested: true } )
186        ( Static { safety: hir::Safety::Safe, mutability: ast::Mutability::Mut, nested: true } )
187        ( Ctor(CtorOf::Struct, CtorKind::Fn)       )
188        ( Ctor(CtorOf::Struct, CtorKind::Const)    )
189        ( Ctor(CtorOf::Variant, CtorKind::Fn)      )
190        ( Ctor(CtorOf::Variant, CtorKind::Const)   )
191        ( Macro(MacroKinds::BANG)                  )
192        ( Macro(MacroKinds::ATTR)                  )
193        ( Macro(MacroKinds::DERIVE)                )
194        ( Macro(MacroKinds::ATTR_BANG)             )
195        ( Macro(MacroKinds::DERIVE_ATTR)           )
196        ( Macro(MacroKinds::DERIVE_BANG)           )
197        ( Macro(MacroKinds::DERIVE_ATTR_BANG)      )
198        ( SyntheticCoroutineBody                   )
199        ( TestBinderConstraints                    )
200    } unreachable {
201        ( Macro(_)                                 )
202    }
203}
204
205impl FixedSizeEncoding for hir::Defaultness {
    type ByteArray = [u8; 1];
    #[inline]
    fn from_bytes(b: &[u8; 1]) -> Self {
        use hir::Defaultness::*;
        let val =
            match b[0] {
                0 => Final,
                1 => Default { has_value: false },
                2 => Default { has_value: true },
                _ => {
                    ::core::panicking::panic_fmt(format_args!("Unexpected {0} code: {1:?}",
                            "hir::Defaultness", b[0]));
                }
            };
        if true {
            {
                match (&(b[0] != 0), &IsDefault::is_default(&val)) {
                    (left_val, right_val) => {
                        if *left_val == *right_val {
                            let kind = ::core::panicking::AssertKind::Ne;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
        };
        val
    }
    #[inline]
    fn write_to_bytes(self, b: &mut [u8; 1]) {
        if true {
            if !!IsDefault::is_default(&self) {
                ::core::panicking::panic("assertion failed: !IsDefault::is_default(&self)")
            };
        };
        use hir::Defaultness::*;
        b[0] =
            match self {
                Final => 0,
                Default { has_value: false } => 1,
                Default { has_value: true } => 2,
            };
        if true {
            {
                match (&b[0], &0) {
                    (left_val, right_val) => {
                        if *left_val == *right_val {
                            let kind = ::core::panicking::AssertKind::Ne;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
        };
    }
}
impl IsDefault for hir::Defaultness {
    fn is_default(&self) -> bool {
        <hir::Defaultness as Default>::default() == *self
    }
}defaulted_enum! {
206    hir::Defaultness {
207        ( Final                        )
208        ( Default { has_value: false } )
209        ( Default { has_value: true }  )
210    }
211}
212
213impl FixedSizeEncoding for ty::Asyncness {
    type ByteArray = [u8; 1];
    #[inline]
    fn from_bytes(b: &[u8; 1]) -> Self {
        use ty::Asyncness::*;
        let val =
            match b[0] {
                0 => No,
                1 => Yes,
                _ => {
                    ::core::panicking::panic_fmt(format_args!("Unexpected {0} code: {1:?}",
                            "ty::Asyncness", b[0]));
                }
            };
        if true {
            {
                match (&(b[0] != 0), &IsDefault::is_default(&val)) {
                    (left_val, right_val) => {
                        if *left_val == *right_val {
                            let kind = ::core::panicking::AssertKind::Ne;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
        };
        val
    }
    #[inline]
    fn write_to_bytes(self, b: &mut [u8; 1]) {
        if true {
            if !!IsDefault::is_default(&self) {
                ::core::panicking::panic("assertion failed: !IsDefault::is_default(&self)")
            };
        };
        use ty::Asyncness::*;
        b[0] = match self { No => 0, Yes => 1, };
        if true {
            {
                match (&b[0], &0) {
                    (left_val, right_val) => {
                        if *left_val == *right_val {
                            let kind = ::core::panicking::AssertKind::Ne;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
        };
    }
}
impl IsDefault for ty::Asyncness {
    fn is_default(&self) -> bool {
        <ty::Asyncness as Default>::default() == *self
    }
}defaulted_enum! {
214    ty::Asyncness {
215        ( No  )
216        ( Yes )
217    }
218}
219
220impl FixedSizeEncoding for hir::Constness {
    type ByteArray = [u8; 1];
    #[inline]
    fn from_bytes(b: &[u8; 1]) -> Self {
        use hir::Constness::*;
        let val =
            match b[0] {
                0 => Const { always: false },
                1 => NotConst,
                2 => Const { always: true },
                _ => {
                    ::core::panicking::panic_fmt(format_args!("Unexpected {0} code: {1:?}",
                            "hir::Constness", b[0]));
                }
            };
        if true {
            {
                match (&(b[0] != 0), &IsDefault::is_default(&val)) {
                    (left_val, right_val) => {
                        if *left_val == *right_val {
                            let kind = ::core::panicking::AssertKind::Ne;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
        };
        val
    }
    #[inline]
    fn write_to_bytes(self, b: &mut [u8; 1]) {
        if true {
            if !!IsDefault::is_default(&self) {
                ::core::panicking::panic("assertion failed: !IsDefault::is_default(&self)")
            };
        };
        use hir::Constness::*;
        b[0] =
            match self {
                Const { always: false } => 0,
                NotConst => 1,
                Const { always: true } => 2,
            };
        if true {
            {
                match (&b[0], &0) {
                    (left_val, right_val) => {
                        if *left_val == *right_val {
                            let kind = ::core::panicking::AssertKind::Ne;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
        };
    }
}
impl IsDefault for hir::Constness {
    fn is_default(&self) -> bool {
        <hir::Constness as Default>::default() == *self
    }
}defaulted_enum! {
221    hir::Constness {
222        ( Const { always: false } )
223        ( NotConst )
224        ( Const { always: true } )
225    }
226}
227
228impl FixedSizeEncoding for hir::Safety {
    type ByteArray = [u8; 1];
    #[inline]
    fn from_bytes(b: &[u8; 1]) -> Self {
        use hir::Safety::*;
        let val =
            match b[0] {
                0 => Unsafe,
                1 => Safe,
                _ => {
                    ::core::panicking::panic_fmt(format_args!("Unexpected {0} code: {1:?}",
                            "hir::Safety", b[0]));
                }
            };
        if true {
            {
                match (&(b[0] != 0), &IsDefault::is_default(&val)) {
                    (left_val, right_val) => {
                        if *left_val == *right_val {
                            let kind = ::core::panicking::AssertKind::Ne;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
        };
        val
    }
    #[inline]
    fn write_to_bytes(self, b: &mut [u8; 1]) {
        if true {
            if !!IsDefault::is_default(&self) {
                ::core::panicking::panic("assertion failed: !IsDefault::is_default(&self)")
            };
        };
        use hir::Safety::*;
        b[0] = match self { Unsafe => 0, Safe => 1, };
        if true {
            {
                match (&b[0], &0) {
                    (left_val, right_val) => {
                        if *left_val == *right_val {
                            let kind = ::core::panicking::AssertKind::Ne;
                            ::core::panicking::assert_failed(kind, &*left_val,
                                &*right_val, ::core::option::Option::None);
                        }
                    }
                }
            };
        };
    }
}
impl IsDefault for hir::Safety {
    fn is_default(&self) -> bool {
        <hir::Safety as Default>::default() == *self
    }
}defaulted_enum! {
229    hir::Safety {
230        ( Unsafe )
231        ( Safe   )
232    }
233}
234
235impl FixedSizeEncoding for Option<hir::CoroutineKind> {
    type ByteArray = [u8; 1];
    #[inline]
    fn from_bytes(b: &[u8; 1]) -> Self {
        use hir::CoroutineKind::*;
        if b[0] == 0 { return None; }
        match b[0] - 1 {
            0 => Some(Coroutine(hir::Movability::Movable)),
            1 => Some(Coroutine(hir::Movability::Static)),
            2 =>
                Some(Desugared(hir::CoroutineDesugaring::Gen,
                        hir::CoroutineSource::Block)),
            3 =>
                Some(Desugared(hir::CoroutineDesugaring::Gen,
                        hir::CoroutineSource::Fn)),
            4 =>
                Some(Desugared(hir::CoroutineDesugaring::Gen,
                        hir::CoroutineSource::Closure)),
            5 =>
                Some(Desugared(hir::CoroutineDesugaring::Async,
                        hir::CoroutineSource::Block)),
            6 =>
                Some(Desugared(hir::CoroutineDesugaring::Async,
                        hir::CoroutineSource::Fn)),
            7 =>
                Some(Desugared(hir::CoroutineDesugaring::Async,
                        hir::CoroutineSource::Closure)),
            8 =>
                Some(Desugared(hir::CoroutineDesugaring::AsyncGen,
                        hir::CoroutineSource::Block)),
            9 =>
                Some(Desugared(hir::CoroutineDesugaring::AsyncGen,
                        hir::CoroutineSource::Fn)),
            10 =>
                Some(Desugared(hir::CoroutineDesugaring::AsyncGen,
                        hir::CoroutineSource::Closure)),
            _ => {
                ::core::panicking::panic_fmt(format_args!("Unexpected {0} code: {1:?}",
                        "hir::CoroutineKind", b[0]));
            }
        }
    }
    #[inline]
    fn write_to_bytes(self, b: &mut [u8; 1]) {
        use hir::CoroutineKind::*;
        b[0] =
            match self {
                None => ::core::panicking::unreachable_default_msg(),
                Some(Coroutine(hir::Movability::Movable)) => 1 + 0,
                Some(Coroutine(hir::Movability::Static)) => 1 + 1,
                Some(Desugared(hir::CoroutineDesugaring::Gen,
                    hir::CoroutineSource::Block)) => 1 + 2,
                Some(Desugared(hir::CoroutineDesugaring::Gen,
                    hir::CoroutineSource::Fn)) => 1 + 3,
                Some(Desugared(hir::CoroutineDesugaring::Gen,
                    hir::CoroutineSource::Closure)) => 1 + 4,
                Some(Desugared(hir::CoroutineDesugaring::Async,
                    hir::CoroutineSource::Block)) => 1 + 5,
                Some(Desugared(hir::CoroutineDesugaring::Async,
                    hir::CoroutineSource::Fn)) => 1 + 6,
                Some(Desugared(hir::CoroutineDesugaring::Async,
                    hir::CoroutineSource::Closure)) => 1 + 7,
                Some(Desugared(hir::CoroutineDesugaring::AsyncGen,
                    hir::CoroutineSource::Block)) => 1 + 8,
                Some(Desugared(hir::CoroutineDesugaring::AsyncGen,
                    hir::CoroutineSource::Fn)) => 1 + 9,
                Some(Desugared(hir::CoroutineDesugaring::AsyncGen,
                    hir::CoroutineSource::Closure)) => 1 + 10,
            }
    }
}fixed_size_enum! {
236    hir::CoroutineKind {
237        ( Coroutine(hir::Movability::Movable)                                          )
238        ( Coroutine(hir::Movability::Static)                                           )
239        ( Desugared(hir::CoroutineDesugaring::Gen, hir::CoroutineSource::Block)        )
240        ( Desugared(hir::CoroutineDesugaring::Gen, hir::CoroutineSource::Fn)           )
241        ( Desugared(hir::CoroutineDesugaring::Gen, hir::CoroutineSource::Closure)      )
242        ( Desugared(hir::CoroutineDesugaring::Async, hir::CoroutineSource::Block)      )
243        ( Desugared(hir::CoroutineDesugaring::Async, hir::CoroutineSource::Fn)         )
244        ( Desugared(hir::CoroutineDesugaring::Async, hir::CoroutineSource::Closure)    )
245        ( Desugared(hir::CoroutineDesugaring::AsyncGen, hir::CoroutineSource::Block)   )
246        ( Desugared(hir::CoroutineDesugaring::AsyncGen, hir::CoroutineSource::Fn)      )
247        ( Desugared(hir::CoroutineDesugaring::AsyncGen, hir::CoroutineSource::Closure) )
248    }
249}
250
251impl FixedSizeEncoding for Option<MacroKind> {
    type ByteArray = [u8; 1];
    #[inline]
    fn from_bytes(b: &[u8; 1]) -> Self {
        use MacroKind::*;
        if b[0] == 0 { return None; }
        match b[0] - 1 {
            0 => Some(Attr),
            1 => Some(Bang),
            2 => Some(Derive),
            _ => {
                ::core::panicking::panic_fmt(format_args!("Unexpected {0} code: {1:?}",
                        "MacroKind", b[0]));
            }
        }
    }
    #[inline]
    fn write_to_bytes(self, b: &mut [u8; 1]) {
        use MacroKind::*;
        b[0] =
            match self {
                None => ::core::panicking::unreachable_default_msg(),
                Some(Attr) => 1 + 0,
                Some(Bang) => 1 + 1,
                Some(Derive) => 1 + 2,
            }
    }
}fixed_size_enum! {
252    MacroKind {
253        ( Attr   )
254        ( Bang   )
255        ( Derive )
256    }
257}
258
259// We directly encode RawDefId because using a `LazyValue` would incur a 50% overhead in the worst case.
260impl FixedSizeEncoding for Option<RemappedRawDefId> {
261    type ByteArray = [u8; 8];
262
263    #[inline]
264    fn from_bytes(encoded: &[u8; 8]) -> Self {
265        let (index, krate) = decode_interleaved(encoded);
266        let krate = u32::from_le_bytes(krate);
267        if krate == 0 {
268            return None;
269        }
270        let index = u32::from_le_bytes(index);
271
272        Some(RemappedRawDefId::new_unchecked(krate - 1, index))
273    }
274
275    #[inline]
276    fn write_to_bytes(self, dest: &mut [u8; 8]) {
277        match self {
278            None => ::core::panicking::unreachable_default_msg()unreachable!(),
279            Some(id) => {
280                let (krate, index) = id.decompose();
281                if true {
    if !(krate < u32::MAX) {
        ::core::panicking::panic("assertion failed: krate < u32::MAX")
    };
};debug_assert!(krate < u32::MAX);
282                // CrateNum is less than `CrateNum::MAX_AS_U32`.
283                let krate = (krate + 1).to_le_bytes();
284                let index = index.to_le_bytes();
285
286                // CrateNum is usually much smaller than the index within the crate, so put it in
287                // the second slot.
288                encode_interleaved(index, krate, dest);
289            }
290        }
291    }
292}
293
294impl FixedSizeEncoding for AttrFlags {
295    type ByteArray = [u8; 1];
296
297    #[inline]
298    fn from_bytes(b: &[u8; 1]) -> Self {
299        AttrFlags::from_bits_truncate(b[0])
300    }
301
302    #[inline]
303    fn write_to_bytes(self, b: &mut [u8; 1]) {
304        if true {
    if !!self.is_default() {
        ::core::panicking::panic("assertion failed: !self.is_default()")
    };
};debug_assert!(!self.is_default());
305        b[0] = self.bits();
306    }
307}
308
309impl FixedSizeEncoding for bool {
310    type ByteArray = [u8; 1];
311
312    #[inline]
313    fn from_bytes(b: &[u8; 1]) -> Self {
314        b[0] != 0
315    }
316
317    #[inline]
318    fn write_to_bytes(self, b: &mut [u8; 1]) {
319        if true {
    if !!self.is_default() {
        ::core::panicking::panic("assertion failed: !self.is_default()")
    };
};debug_assert!(!self.is_default());
320        b[0] = self as u8
321    }
322}
323
324// NOTE(eddyb) there could be an impl for `usize`, which would enable a more
325// generic `LazyValue<T>` impl, but in the general case we might not need / want
326// to fit every `usize` in `u32`.
327impl<T> FixedSizeEncoding for Option<LazyValue<T>> {
328    type ByteArray = [u8; 8];
329
330    #[inline]
331    fn from_bytes(b: &[u8; 8]) -> Self {
332        let position = NonZero::new(u64::from_bytes(b) as usize)?;
333        Some(LazyValue::from_position(position))
334    }
335
336    #[inline]
337    fn write_to_bytes(self, b: &mut [u8; 8]) {
338        match self {
339            None => ::core::panicking::unreachable_default_msg()unreachable!(),
340            Some(lazy) => {
341                let position = lazy.position.get();
342                let position: u64 = position.try_into().unwrap();
343                position.write_to_bytes(b)
344            }
345        }
346    }
347}
348
349impl<T> LazyArray<T> {
350    #[inline]
351    fn write_to_bytes_impl(self, dest: &mut [u8; 16]) {
352        let position = (self.position.get() as u64).to_le_bytes();
353        let len = (self.num_elems as u64).to_le_bytes();
354
355        encode_interleaved(position, len, dest)
356    }
357
358    fn from_bytes_impl(position: &[u8; 8], meta: &[u8; 8]) -> Option<LazyArray<T>> {
359        let position = NonZero::new(u64::from_bytes(position) as usize)?;
360        let len = u64::from_bytes(meta) as usize;
361        Some(LazyArray::from_position_and_num_elems(position, len))
362    }
363}
364
365// Interleaving the bytes of the two integers exposes trailing bytes in the first integer
366// to the varint scheme that we use for tables.
367#[inline]
368fn decode_interleaved<const N: usize, const M: usize>(encoded: &[u8; N]) -> ([u8; M], [u8; M]) {
369    {
    match (&(M * 2), &N) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(M * 2, N);
370    let mut first = [0u8; M];
371    let mut second = [0u8; M];
372    for i in 0..M {
373        first[i] = encoded[2 * i];
374        second[i] = encoded[2 * i + 1];
375    }
376    (first, second)
377}
378
379// Element width is selected at runtime on a per-table basis by omitting trailing
380// zero bytes in table elements. This works very naturally when table elements are
381// simple numbers but sometimes we have a pair of integers. If naively encoded, the second element
382// would shield the trailing zeroes in the first. Interleaving the bytes exposes trailing zeroes in
383// both to the optimization.
384//
385// Prefer passing a and b such that `b` is usually smaller.
386#[inline]
387fn encode_interleaved<const N: usize, const M: usize>(a: [u8; M], b: [u8; M], dest: &mut [u8; N]) {
388    {
    match (&(M * 2), &N) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(M * 2, N);
389    for i in 0..M {
390        dest[2 * i] = a[i];
391        dest[2 * i + 1] = b[i];
392    }
393}
394
395impl<T> FixedSizeEncoding for LazyArray<T> {
396    type ByteArray = [u8; 16];
397
398    #[inline]
399    fn from_bytes(b: &[u8; 16]) -> Self {
400        let (position, meta) = decode_interleaved(b);
401
402        if meta == [0; 8] {
403            return Default::default();
404        }
405        LazyArray::from_bytes_impl(&position, &meta).unwrap()
406    }
407
408    #[inline]
409    fn write_to_bytes(self, b: &mut [u8; 16]) {
410        if !!self.is_default() {
    ::core::panicking::panic("assertion failed: !self.is_default()")
};assert!(!self.is_default());
411        self.write_to_bytes_impl(b)
412    }
413}
414
415impl<T> FixedSizeEncoding for Option<LazyArray<T>> {
416    type ByteArray = [u8; 16];
417
418    #[inline]
419    fn from_bytes(b: &[u8; 16]) -> Self {
420        let (position, meta) = decode_interleaved(b);
421
422        LazyArray::from_bytes_impl(&position, &meta)
423    }
424
425    #[inline]
426    fn write_to_bytes(self, b: &mut [u8; 16]) {
427        match self {
428            None => ::core::panicking::unreachable_default_msg()unreachable!(),
429            Some(lazy) => lazy.write_to_bytes_impl(b),
430        }
431    }
432}
433
434/// Helper for constructing a table's serialization (also see `Table`).
435pub(super) struct TableBuilder<IdxEncode: Idx, IdxDecode: Idx, T: FixedSizeEncoding> {
436    width: usize,
437    blocks: IndexVec<IdxEncode, T::ByteArray>,
438    _marker: PhantomData<(IdxDecode, T)>,
439}
440
441impl<Ie: Idx, Id: Idx, T: FixedSizeEncoding> Default for TableBuilder<Ie, Id, T> {
442    fn default() -> Self {
443        TableBuilder { width: 0, blocks: Default::default(), _marker: PhantomData }
444    }
445}
446
447impl<Ie: Idx, Id: Idx, const N: usize, T> TableBuilder<Ie, Id, Option<T>>
448where
449    Option<T>: FixedSizeEncoding<ByteArray = [u8; N]>,
450{
451    pub(crate) fn set_some(&mut self, i: Ie, value: T) {
452        self.set(i, Some(value))
453    }
454}
455
456impl<Ie: Idx, Id: Idx, const N: usize, T: FixedSizeEncoding<ByteArray = [u8; N]>>
457    TableBuilder<Ie, Id, T>
458{
459    /// Sets the table value if it is not default.
460    /// ATTENTION: For optimization default values are simply ignored by this function, because
461    /// right now metadata tables never need to reset non-default values to default. If such need
462    /// arises in the future then a new method (e.g. `clear` or `reset`) will need to be introduced
463    /// for doing that explicitly.
464    pub(crate) fn set(&mut self, i: Ie, value: T) {
465        #[cfg(debug_assertions)]
466        {
467            if true {
    if !T::from_bytes(&[0; N]).is_default() {
        {
            ::core::panicking::panic_fmt(format_args!("expected all-zeroes to decode to the default value, as per the invariant of FixedSizeEncoding"));
        }
    };
};debug_assert!(
468                T::from_bytes(&[0; N]).is_default(),
469                "expected all-zeroes to decode to the default value, as per the invariant of FixedSizeEncoding"
470            );
471        }
472        if !value.is_default() {
473            let block = self.blocks.ensure_contains_elem(i, || [0; N]);
474            value.write_to_bytes(block);
475            if self.width != N {
476                let width = N - trailing_zeros(block);
477                self.width = self.width.max(width);
478            }
479        }
480    }
481
482    pub(crate) fn encode(&self, buf: &mut FileEncoder<'_>) -> LazyTable<Ie, Id, T> {
483        let pos = buf.position();
484
485        let width = self.width;
486        let len = self.blocks.len();
487        let index_bytes = len.div_ceil(SPARSE_BLOCK_LEN) * SPARSE_INDEX_BYTES;
488        let non_defaults = if index_bytes < len * width {
489            self.blocks.iter().filter(|block| **block != [0; N]).count()
490        } else {
491            len
492        };
493        // Use dense encoding when at least 75% of the entries are non-default.
494        // This avoids sparse lookup overhead for tables with few default entries.
495        let sparse = len - non_defaults > len / 4
496            && u32::try_from(non_defaults).is_ok()
497            && index_bytes + non_defaults * width < len * width;
498
499        // Sparse tables store one block index per 64 table entries, followed by the
500        // non-default values in table order:
501        //
502        // [ block index 0 ][ block index 1 ] ... [ non-default values, each `width` bytes ]
503        //
504        // Each block index occupies 12 bytes, with both integers stored little-endian:
505        //
506        // [ bitmap: u64 (8 bytes) | base: u32 (4 bytes) ]
507        //
508        // Bitmap bits mark non-default entries, starting with the least significant
509        // bit. `base` counts the non-default entries in all preceding blocks, so
510        // lookups do not need to scan their bitmaps.
511        //
512        // The `LazyTable` header combines `width` and `sparse` in one integer:
513        //
514        // [ width bits ... | sparse ]
515        //
516        // Shifting `width` left by one bit makes room for the `sparse` flag.
517        // The least significant bit is 1 for sparse tables and 0 for dense tables.
518        if sparse {
519            let mut base = 0u32;
520            for chunk in self.blocks.raw.chunks(SPARSE_BLOCK_LEN) {
521                let mut bitmap = 0u64;
522                for (i, block) in chunk.iter().enumerate() {
523                    if *block != [0; N] {
524                        bitmap |= 1 << i;
525                    }
526                }
527                buf.write_array(bitmap.to_le_bytes());
528                buf.write_array(base.to_le_bytes());
529                base += bitmap.count_ones();
530            }
531        }
532
533        for block in &self.blocks {
534            if !sparse || *block != [0; N] {
535                buf.write_with(|dest| {
536                    *dest = *block;
537                    width
538                });
539            }
540        }
541
542        if true {
    if !(width <= usize::MAX >> 1) {
        ::core::panicking::panic("assertion failed: width <= usize::MAX >> 1")
    };
};debug_assert!(width <= usize::MAX >> 1);
543        LazyTable::from_position_and_encoded_size(
544            NonZero::new(pos).unwrap(),
545            (width << 1) | usize::from(sparse),
546            len,
547        )
548    }
549}
550
551fn trailing_zeros(x: &[u8]) -> usize {
552    x.iter().rev().take_while(|b| **b == 0).count()
553}
554
555impl<
556    Ie: Idx,
557    Id: Idx,
558    const N: usize,
559    T: FixedSizeEncoding<ByteArray = [u8; N]> + ParameterizedOverTcx,
560> LazyTable<Ie, Id, T>
561where
562    for<'tcx> T::Value<'tcx>: FixedSizeEncoding<ByteArray = [u8; N]>,
563{
564    /// Given the metadata, extract out the value at a particular index (if any).
565    pub(super) fn get<'a, 'tcx, M: MetaBlob<'a>>(&self, metadata: M, i: Id) -> T::Value<'tcx> {
566        // Access past the end of the table returns a Default
567        if i.index() >= self.len {
568            return Default::default();
569        }
570
571        let width = self.width_and_sparse >> 1;
572        let mut position = self.position.get();
573        let mut index = i.index();
574        let blob = metadata.blob();
575        // An unset bitmap bit represents a default value. For a set bit, the
576        // index among the packed values is:
577        //
578        //     base + number of set bits before this bit in the current block
579        //
580        // Skip all block indices to reach the start of the packed values.
581        if self.width_and_sparse & 1 != 0 {
582            let block = index / SPARSE_BLOCK_LEN;
583            let bit = 1u64 << (index % SPARSE_BLOCK_LEN);
584            let start = position + block * SPARSE_INDEX_BYTES;
585            let entry = &blob[start..start + SPARSE_INDEX_BYTES];
586            let bitmap = u64::from_le_bytes(entry[..8].try_into().unwrap());
587            if bitmap & bit == 0 {
588                return Default::default();
589            }
590            let base = u32::from_le_bytes(entry[8..].try_into().unwrap());
591            index = base as usize + (bitmap & (bit - 1)).count_ones() as usize;
592            position += self.len.div_ceil(SPARSE_BLOCK_LEN) * SPARSE_INDEX_BYTES;
593        }
594        let start = position + width * index;
595        let end = start + width;
596        let bytes = &blob[start..end];
597
598        if let Ok(fixed) = bytes.try_into() {
599            FixedSizeEncoding::from_bytes(fixed)
600        } else {
601            let mut fixed = [0u8; N];
602            fixed[..width].copy_from_slice(bytes);
603            FixedSizeEncoding::from_bytes(&fixed)
604        }
605    }
606
607    /// Size of the table in entries, including possible gaps.
608    pub(super) fn size(&self) -> usize {
609        self.len
610    }
611}