1use std::ops::Deref;
2
3use rustc_data_structures::sync::{AtomicU64, WorkerLocal};
4use rustc_hir::def_id::{DefId, LocalDefId};
5use rustc_hir::hir_id::OwnerId;
6use rustc_macros::HashStable;
7use rustc_query_system::HandleCycleError;
8use rustc_query_system::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
9pub(crate) use rustc_query_system::query::QueryJobId;
10use rustc_query_system::query::*;
11use rustc_span::{ErrorGuaranteed, Span};
12pub use sealed::IntoQueryParam;
13
14use crate::dep_graph;
15use crate::dep_graph::DepKind;
16use crate::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache};
17use crate::query::{
18 DynamicQueries, ExternProviders, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
19};
20use crate::ty::TyCtxt;
21
22pub struct DynamicQuery<'tcx, C: QueryCache> {
23 pub name: &'static str,
24 pub eval_always: bool,
25 pub dep_kind: DepKind,
26 pub handle_cycle_error: HandleCycleError,
27 pub query_state: usize,
29 pub query_cache: usize,
31 pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
32 pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
33 pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
34 pub can_load_from_disk: bool,
35 pub try_load_from_disk: fn(
36 tcx: TyCtxt<'tcx>,
37 key: &C::Key,
38 prev_index: SerializedDepNodeIndex,
39 index: DepNodeIndex,
40 ) -> Option<C::Value>,
41 pub loadable_from_disk:
42 fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool,
43 pub hash_result: HashResult<C::Value>,
44 pub value_from_cycle_error:
45 fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
46 pub format_value: fn(&C::Value) -> String,
47}
48
49pub struct QuerySystemFns {
50 pub engine: QueryEngine,
51 pub local_providers: Providers,
52 pub extern_providers: ExternProviders,
53 pub encode_query_results: for<'tcx> fn(
54 tcx: TyCtxt<'tcx>,
55 encoder: &mut CacheEncoder<'_, 'tcx>,
56 query_result_index: &mut EncodedDepNodeIndex,
57 ),
58 pub try_mark_green: for<'tcx> fn(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool,
59}
60
61pub struct QuerySystem<'tcx> {
62 pub states: QueryStates<'tcx>,
63 pub arenas: WorkerLocal<QueryArenas<'tcx>>,
64 pub caches: QueryCaches<'tcx>,
65 pub dynamic_queries: DynamicQueries<'tcx>,
66
67 pub on_disk_cache: Option<OnDiskCache>,
72
73 pub fns: QuerySystemFns,
74
75 pub jobs: AtomicU64,
76}
77
78#[derive(Copy, Clone)]
79pub struct TyCtxtAt<'tcx> {
80 pub tcx: TyCtxt<'tcx>,
81 pub span: Span,
82}
83
84impl<'tcx> Deref for TyCtxtAt<'tcx> {
85 type Target = TyCtxt<'tcx>;
86 #[inline(always)]
87 fn deref(&self) -> &Self::Target {
88 &self.tcx
89 }
90}
91
92#[derive(Copy, Clone)]
93#[must_use]
94pub struct TyCtxtEnsureOk<'tcx> {
95 pub tcx: TyCtxt<'tcx>,
96}
97
98#[derive(Copy, Clone)]
99#[must_use]
100pub struct TyCtxtEnsureDone<'tcx> {
101 pub tcx: TyCtxt<'tcx>,
102}
103
104impl<'tcx> TyCtxt<'tcx> {
105 #[inline(always)]
132 pub fn ensure_ok(self) -> TyCtxtEnsureOk<'tcx> {
133 TyCtxtEnsureOk { tcx: self }
134 }
135
136 #[inline(always)]
153 pub fn ensure_done(self) -> TyCtxtEnsureDone<'tcx> {
154 TyCtxtEnsureDone { tcx: self }
155 }
156
157 #[inline(always)]
160 pub fn at(self, span: Span) -> TyCtxtAt<'tcx> {
161 TyCtxtAt { tcx: self, span }
162 }
163
164 pub fn try_mark_green(self, dep_node: &dep_graph::DepNode) -> bool {
165 (self.query_system.fns.try_mark_green)(self, dep_node)
166 }
167}
168
169macro_rules! query_ensure_select {
172 ([]$($args:tt)*) => {
173 crate::query::inner::query_ensure($($args)*)
174 };
175 ([(return_result_from_ensure_ok) $($rest:tt)*]$($args:tt)*) => {
176 crate::query::inner::query_ensure_error_guaranteed($($args)*)
177 };
178 ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
179 query_ensure_select!([$($modifiers)*]$($args)*)
180 };
181}
182
183macro_rules! query_helper_param_ty {
184 (DefId) => { impl IntoQueryParam<DefId> };
185 (LocalDefId) => { impl IntoQueryParam<LocalDefId> };
186 ($K:ty) => { $K };
187}
188
189macro_rules! query_if_arena {
190 ([] $arena:tt $no_arena:tt) => {
191 $no_arena
192 };
193 ([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
194 $arena
195 };
196 ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
197 query_if_arena!([$($modifiers)*]$($args)*)
198 };
199}
200
201macro_rules! local_key_if_separate_extern {
204 ([] $($K:tt)*) => {
205 $($K)*
206 };
207 ([(separate_provide_extern) $($rest:tt)*] $($K:tt)*) => {
208 <$($K)* as AsLocalKey>::LocalKey
209 };
210 ([$other:tt $($modifiers:tt)*] $($K:tt)*) => {
211 local_key_if_separate_extern!([$($modifiers)*] $($K)*)
212 };
213}
214
215macro_rules! separate_provide_extern_decl {
216 ([][$name:ident]) => {
217 ()
218 };
219 ([(separate_provide_extern) $($rest:tt)*][$name:ident]) => {
220 for<'tcx> fn(
221 TyCtxt<'tcx>,
222 queries::$name::Key<'tcx>,
223 ) -> queries::$name::ProvidedValue<'tcx>
224 };
225 ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
226 separate_provide_extern_decl!([$($modifiers)*][$($args)*])
227 };
228}
229
230macro_rules! ensure_ok_result {
231 ( [] ) => {
232 ()
233 };
234 ( [(return_result_from_ensure_ok) $($rest:tt)*] ) => {
235 Result<(), ErrorGuaranteed>
236 };
237 ( [$other:tt $($modifiers:tt)*] ) => {
238 ensure_ok_result!( [$($modifiers)*] )
239 };
240}
241
242macro_rules! separate_provide_extern_default {
243 ([][$name:ident]) => {
244 ()
245 };
246 ([(separate_provide_extern) $($rest:tt)*][$name:ident]) => {
247 |_, key| $crate::query::plumbing::default_extern_query(stringify!($name), &key)
248 };
249 ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
250 separate_provide_extern_default!([$($modifiers)*][$($args)*])
251 };
252}
253
254macro_rules! define_callbacks {
255 (
256 $(
257 $(#[$attr:meta])*
258 [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,
259 )*
260 ) => {
261
262 #[allow(unused_lifetimes)]
263 pub mod queries {
264 $(pub mod $name {
265 use super::super::*;
266
267 pub type Key<'tcx> = $($K)*;
268 pub type Value<'tcx> = $V;
269
270 pub type LocalKey<'tcx> = local_key_if_separate_extern!([$($modifiers)*] $($K)*);
271
272 pub type ProvidedValue<'tcx> = query_if_arena!(
276 [$($modifiers)*]
277 (<$V as $crate::query::arena_cached::ArenaCached<'tcx>>::Provided)
278 ($V)
279 );
280
281 #[inline(always)]
286 pub fn provided_to_erased<'tcx>(
287 _tcx: TyCtxt<'tcx>,
288 value: ProvidedValue<'tcx>,
289 ) -> Erase<Value<'tcx>> {
290 erase(query_if_arena!([$($modifiers)*]
291 {
292 use $crate::query::arena_cached::ArenaCached;
293
294 if mem::needs_drop::<<$V as ArenaCached<'tcx>>::Allocated>() {
295 <$V as ArenaCached>::alloc_in_arena(
296 |v| _tcx.query_system.arenas.$name.alloc(v),
297 value,
298 )
299 } else {
300 <$V as ArenaCached>::alloc_in_arena(
301 |v| _tcx.arena.dropless.alloc(v),
302 value,
303 )
304 }
305 }
306 (value)
307 ))
308 }
309
310 pub type Storage<'tcx> = <$($K)* as keys::Key>::Cache<Erase<$V>>;
311
312 #[cfg(target_pointer_width = "64")]
315 const _: () = {
316 if size_of::<Key<'static>>() > 88 {
317 panic!("{}", concat!(
318 "the query `",
319 stringify!($name),
320 "` has a key type `",
321 stringify!($($K)*),
322 "` that is too large"
323 ));
324 }
325 };
326
327 #[cfg(target_pointer_width = "64")]
330 #[cfg(not(feature = "rustc_randomized_layouts"))]
331 const _: () = {
332 if size_of::<Value<'static>>() > 64 {
333 panic!("{}", concat!(
334 "the query `",
335 stringify!($name),
336 "` has a value type `",
337 stringify!($V),
338 "` that is too large"
339 ));
340 }
341 };
342 })*
343 }
344
345 pub struct QueryArenas<'tcx> {
346 $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
347 (TypedArena<<$V as $crate::query::arena_cached::ArenaCached<'tcx>>::Allocated>)
348 ()
349 ),)*
350 }
351
352 impl Default for QueryArenas<'_> {
353 fn default() -> Self {
354 Self {
355 $($name: query_if_arena!([$($modifiers)*]
356 (Default::default())
357 ()
358 ),)*
359 }
360 }
361 }
362
363 #[derive(Default)]
364 pub struct QueryCaches<'tcx> {
365 $($(#[$attr])* pub $name: queries::$name::Storage<'tcx>,)*
366 }
367
368 impl<'tcx> TyCtxtEnsureOk<'tcx> {
369 $($(#[$attr])*
370 #[inline(always)]
371 pub fn $name(
372 self,
373 key: query_helper_param_ty!($($K)*),
374 ) -> ensure_ok_result!([$($modifiers)*]) {
375 query_ensure_select!(
376 [$($modifiers)*]
377 self.tcx,
378 self.tcx.query_system.fns.engine.$name,
379 &self.tcx.query_system.caches.$name,
380 key.into_query_param(),
381 false,
382 )
383 })*
384 }
385
386 impl<'tcx> TyCtxtEnsureDone<'tcx> {
387 $($(#[$attr])*
388 #[inline(always)]
389 pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
390 crate::query::inner::query_ensure(
391 self.tcx,
392 self.tcx.query_system.fns.engine.$name,
393 &self.tcx.query_system.caches.$name,
394 key.into_query_param(),
395 true,
396 );
397 })*
398 }
399
400 impl<'tcx> TyCtxt<'tcx> {
401 $($(#[$attr])*
402 #[inline(always)]
403 #[must_use]
404 pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
405 {
406 self.at(DUMMY_SP).$name(key)
407 })*
408 }
409
410 impl<'tcx> TyCtxtAt<'tcx> {
411 $($(#[$attr])*
412 #[inline(always)]
413 pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
414 {
415 restore::<$V>(crate::query::inner::query_get_at(
416 self.tcx,
417 self.tcx.query_system.fns.engine.$name,
418 &self.tcx.query_system.caches.$name,
419 self.span,
420 key.into_query_param(),
421 ))
422 })*
423 }
424
425 pub struct DynamicQueries<'tcx> {
426 $(
427 pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>>,
428 )*
429 }
430
431 #[derive(Default)]
432 pub struct QueryStates<'tcx> {
433 $(
434 pub $name: QueryState<$($K)*, QueryStackDeferred<'tcx>>,
435 )*
436 }
437
438 pub struct Providers {
439 $(pub $name: for<'tcx> fn(
440 TyCtxt<'tcx>,
441 queries::$name::LocalKey<'tcx>,
442 ) -> queries::$name::ProvidedValue<'tcx>,)*
443 }
444
445 pub struct ExternProviders {
446 $(pub $name: separate_provide_extern_decl!([$($modifiers)*][$name]),)*
447 }
448
449 impl Default for Providers {
450 fn default() -> Self {
451 Providers {
452 $($name: |_, key| $crate::query::plumbing::default_query(stringify!($name), &key)),*
453 }
454 }
455 }
456
457 impl Default for ExternProviders {
458 fn default() -> Self {
459 ExternProviders {
460 $($name: separate_provide_extern_default!([$($modifiers)*][$name]),)*
461 }
462 }
463 }
464
465 impl Copy for Providers {}
466 impl Clone for Providers {
467 fn clone(&self) -> Self { *self }
468 }
469
470 impl Copy for ExternProviders {}
471 impl Clone for ExternProviders {
472 fn clone(&self) -> Self { *self }
473 }
474
475 pub struct QueryEngine {
476 $(pub $name: for<'tcx> fn(
477 TyCtxt<'tcx>,
478 Span,
479 queries::$name::Key<'tcx>,
480 QueryMode,
481 ) -> Option<Erase<$V>>,)*
482 }
483 };
484}
485
486macro_rules! hash_result {
487 ([]) => {{
488 Some(dep_graph::hash_result)
489 }};
490 ([(no_hash) $($rest:tt)*]) => {{
491 None
492 }};
493 ([$other:tt $($modifiers:tt)*]) => {
494 hash_result!([$($modifiers)*])
495 };
496}
497
498macro_rules! define_feedable {
499 ($($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
500 $(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
501 $(#[$attr])*
502 #[inline(always)]
503 pub fn $name(self, value: queries::$name::ProvidedValue<'tcx>) {
504 let key = self.key().into_query_param();
505
506 let tcx = self.tcx;
507 let erased = queries::$name::provided_to_erased(tcx, value);
508 let cache = &tcx.query_system.caches.$name;
509
510 let dep_kind: dep_graph::DepKind = dep_graph::dep_kinds::$name;
511 let hasher: Option<fn(&mut StableHashingContext<'_>, &_) -> _> = hash_result!([$($modifiers)*]);
512
513 $crate::query::inner::query_feed(
514 tcx,
515 dep_kind,
516 hasher,
517 cache,
518 key,
519 erased,
520 );
521 }
522 })*
523 }
524}
525
526mod sealed {
539 use rustc_hir::def_id::{LocalModDefId, ModDefId};
540
541 use super::{DefId, LocalDefId, OwnerId};
542
543 pub trait IntoQueryParam<P> {
548 fn into_query_param(self) -> P;
549 }
550
551 impl<P> IntoQueryParam<P> for P {
552 #[inline(always)]
553 fn into_query_param(self) -> P {
554 self
555 }
556 }
557
558 impl<'a, P: Copy> IntoQueryParam<P> for &'a P {
559 #[inline(always)]
560 fn into_query_param(self) -> P {
561 *self
562 }
563 }
564
565 impl IntoQueryParam<LocalDefId> for OwnerId {
566 #[inline(always)]
567 fn into_query_param(self) -> LocalDefId {
568 self.def_id
569 }
570 }
571
572 impl IntoQueryParam<DefId> for LocalDefId {
573 #[inline(always)]
574 fn into_query_param(self) -> DefId {
575 self.to_def_id()
576 }
577 }
578
579 impl IntoQueryParam<DefId> for OwnerId {
580 #[inline(always)]
581 fn into_query_param(self) -> DefId {
582 self.to_def_id()
583 }
584 }
585
586 impl IntoQueryParam<DefId> for ModDefId {
587 #[inline(always)]
588 fn into_query_param(self) -> DefId {
589 self.to_def_id()
590 }
591 }
592
593 impl IntoQueryParam<DefId> for LocalModDefId {
594 #[inline(always)]
595 fn into_query_param(self) -> DefId {
596 self.to_def_id()
597 }
598 }
599
600 impl IntoQueryParam<LocalDefId> for LocalModDefId {
601 #[inline(always)]
602 fn into_query_param(self) -> LocalDefId {
603 self.into()
604 }
605 }
606}
607
608#[derive(Copy, Clone, Debug, HashStable)]
609pub struct CyclePlaceholder(pub ErrorGuaranteed);
610
611#[cold]
612pub(crate) fn default_query(name: &str, key: &dyn std::fmt::Debug) -> ! {
613 bug!(
614 "`tcx.{name}({key:?})` is not supported for this key;\n\
615 hint: Queries can be either made to the local crate, or the external crate. \
616 This error means you tried to use it for one that's not supported.\n\
617 If that's not the case, {name} was likely never assigned to a provider function.\n",
618 )
619}
620
621#[cold]
622pub(crate) fn default_extern_query(name: &str, key: &dyn std::fmt::Debug) -> ! {
623 bug!(
624 "`tcx.{name}({key:?})` unsupported by its crate; \
625 perhaps the `{name}` query was never assigned a provider function",
626 )
627}