1use std::ops::Deref;
2
3use rustc_data_structures::sync::{AtomicU64, WorkerLocal};
4use rustc_hir::def_id::{DefId, LocalDefId};
5use rustc_hir::hir_id::OwnerId;
6use rustc_macros::HashStable;
7use rustc_query_system::HandleCycleError;
8use rustc_query_system::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
9pub(crate) use rustc_query_system::query::QueryJobId;
10use rustc_query_system::query::*;
11use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
12
13use crate::dep_graph;
14use crate::dep_graph::DepKind;
15use crate::query::on_disk_cache::{CacheEncoder, EncodedDepNodeIndex, OnDiskCache};
16use crate::query::{
17 DynamicQueries, ExternProviders, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
18};
19use crate::ty::TyCtxt;
20
21pub struct DynamicQuery<'tcx, C: QueryCache> {
22 pub name: &'static str,
23 pub eval_always: bool,
24 pub dep_kind: DepKind,
25 pub handle_cycle_error: HandleCycleError,
26 pub query_state: usize,
28 pub query_cache: usize,
30 pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
31 pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
32 pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
33 pub can_load_from_disk: bool,
34 pub try_load_from_disk: fn(
35 tcx: TyCtxt<'tcx>,
36 key: &C::Key,
37 prev_index: SerializedDepNodeIndex,
38 index: DepNodeIndex,
39 ) -> Option<C::Value>,
40 pub loadable_from_disk:
41 fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool,
42 pub hash_result: HashResult<C::Value>,
43 pub value_from_cycle_error:
44 fn(tcx: TyCtxt<'tcx>, cycle_error: &CycleError, guar: ErrorGuaranteed) -> C::Value,
45 pub format_value: fn(&C::Value) -> String,
46}
47
48pub struct QuerySystemFns {
49 pub engine: QueryEngine,
50 pub local_providers: Providers,
51 pub extern_providers: ExternProviders,
52 pub encode_query_results: for<'tcx> fn(
53 tcx: TyCtxt<'tcx>,
54 encoder: &mut CacheEncoder<'_, 'tcx>,
55 query_result_index: &mut EncodedDepNodeIndex,
56 ),
57 pub try_mark_green: for<'tcx> fn(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool,
58}
59
60pub struct QuerySystem<'tcx> {
61 pub states: QueryStates<'tcx>,
62 pub arenas: WorkerLocal<QueryArenas<'tcx>>,
63 pub caches: QueryCaches<'tcx>,
64 pub dynamic_queries: DynamicQueries<'tcx>,
65
66 pub on_disk_cache: Option<OnDiskCache>,
71
72 pub fns: QuerySystemFns,
73
74 pub jobs: AtomicU64,
75}
76
77#[derive(Copy, Clone)]
78pub struct TyCtxtAt<'tcx> {
79 pub tcx: TyCtxt<'tcx>,
80 pub span: Span,
81}
82
83impl<'tcx> Deref for TyCtxtAt<'tcx> {
84 type Target = TyCtxt<'tcx>;
85 #[inline(always)]
86 fn deref(&self) -> &Self::Target {
87 &self.tcx
88 }
89}
90
91#[derive(Copy, Clone)]
92#[must_use]
93pub struct TyCtxtEnsureOk<'tcx> {
94 pub tcx: TyCtxt<'tcx>,
95}
96
97#[derive(Copy, Clone)]
98#[must_use]
99pub struct TyCtxtEnsureDone<'tcx> {
100 pub tcx: TyCtxt<'tcx>,
101}
102
103impl<'tcx> TyCtxt<'tcx> {
104 #[inline(always)]
131 pub fn ensure_ok(self) -> TyCtxtEnsureOk<'tcx> {
132 TyCtxtEnsureOk { tcx: self }
133 }
134
135 #[inline(always)]
152 pub fn ensure_done(self) -> TyCtxtEnsureDone<'tcx> {
153 TyCtxtEnsureDone { tcx: self }
154 }
155
156 #[inline(always)]
159 pub fn at(self, span: Span) -> TyCtxtAt<'tcx> {
160 TyCtxtAt { tcx: self, span }
161 }
162
163 pub fn try_mark_green(self, dep_node: &dep_graph::DepNode) -> bool {
164 (self.query_system.fns.try_mark_green)(self, dep_node)
165 }
166}
167
168#[inline(always)]
169pub fn query_get_at<'tcx, Cache>(
170 tcx: TyCtxt<'tcx>,
171 execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
172 query_cache: &Cache,
173 span: Span,
174 key: Cache::Key,
175) -> Cache::Value
176where
177 Cache: QueryCache,
178{
179 let key = key.into_query_param();
180 match try_get_cached(tcx, query_cache, &key) {
181 Some(value) => value,
182 None => execute_query(tcx, span, key, QueryMode::Get).unwrap(),
183 }
184}
185
186#[inline]
187pub fn query_ensure<'tcx, Cache>(
188 tcx: TyCtxt<'tcx>,
189 execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
190 query_cache: &Cache,
191 key: Cache::Key,
192 check_cache: bool,
193) where
194 Cache: QueryCache,
195{
196 let key = key.into_query_param();
197 if try_get_cached(tcx, query_cache, &key).is_none() {
198 execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { check_cache });
199 }
200}
201
202#[inline]
203pub fn query_ensure_error_guaranteed<'tcx, Cache, T>(
204 tcx: TyCtxt<'tcx>,
205 execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
206 query_cache: &Cache,
207 key: Cache::Key,
208 check_cache: bool,
209) -> Result<(), ErrorGuaranteed>
210where
211 Cache: QueryCache<Value = super::erase::Erase<Result<T, ErrorGuaranteed>>>,
212 Result<T, ErrorGuaranteed>: EraseType,
213{
214 let key = key.into_query_param();
215 if let Some(res) = try_get_cached(tcx, query_cache, &key) {
216 super::erase::restore(res).map(drop)
217 } else {
218 execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { check_cache })
219 .map(super::erase::restore)
220 .map(|res| res.map(drop))
221 .unwrap_or(Ok(()))
228 }
229}
230
231macro_rules! query_ensure {
232 ([]$($args:tt)*) => {
233 query_ensure($($args)*)
234 };
235 ([(return_result_from_ensure_ok) $($rest:tt)*]$($args:tt)*) => {
236 query_ensure_error_guaranteed($($args)*).map(|_| ())
237 };
238 ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
239 query_ensure!([$($modifiers)*]$($args)*)
240 };
241}
242
243macro_rules! query_helper_param_ty {
244 (DefId) => { impl IntoQueryParam<DefId> };
245 (LocalDefId) => { impl IntoQueryParam<LocalDefId> };
246 ($K:ty) => { $K };
247}
248
249macro_rules! query_if_arena {
250 ([] $arena:tt $no_arena:tt) => {
251 $no_arena
252 };
253 ([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => {
254 $arena
255 };
256 ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
257 query_if_arena!([$($modifiers)*]$($args)*)
258 };
259}
260
261macro_rules! local_key_if_separate_extern {
264 ([] $($K:tt)*) => {
265 $($K)*
266 };
267 ([(separate_provide_extern) $($rest:tt)*] $($K:tt)*) => {
268 <$($K)* as AsLocalKey>::LocalKey
269 };
270 ([$other:tt $($modifiers:tt)*] $($K:tt)*) => {
271 local_key_if_separate_extern!([$($modifiers)*] $($K)*)
272 };
273}
274
275macro_rules! separate_provide_extern_decl {
276 ([][$name:ident]) => {
277 ()
278 };
279 ([(separate_provide_extern) $($rest:tt)*][$name:ident]) => {
280 for<'tcx> fn(
281 TyCtxt<'tcx>,
282 queries::$name::Key<'tcx>,
283 ) -> queries::$name::ProvidedValue<'tcx>
284 };
285 ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
286 separate_provide_extern_decl!([$($modifiers)*][$($args)*])
287 };
288}
289
290macro_rules! ensure_ok_result {
291 ( [] ) => {
292 ()
293 };
294 ( [(return_result_from_ensure_ok) $($rest:tt)*] ) => {
295 Result<(), ErrorGuaranteed>
296 };
297 ( [$other:tt $($modifiers:tt)*] ) => {
298 ensure_ok_result!( [$($modifiers)*] )
299 };
300}
301
302macro_rules! separate_provide_extern_default {
303 ([][$name:ident]) => {
304 ()
305 };
306 ([(separate_provide_extern) $($rest:tt)*][$name:ident]) => {
307 |_, key| $crate::query::plumbing::default_extern_query(stringify!($name), &key)
308 };
309 ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
310 separate_provide_extern_default!([$($modifiers)*][$($args)*])
311 };
312}
313
314macro_rules! define_callbacks {
315 (
316 $($(#[$attr:meta])*
317 [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
318
319 #[allow(unused_lifetimes)]
320 pub mod queries {
321 $(pub mod $name {
322 use super::super::*;
323
324 pub type Key<'tcx> = $($K)*;
325 pub type Value<'tcx> = $V;
326
327 pub type LocalKey<'tcx> = local_key_if_separate_extern!([$($modifiers)*] $($K)*);
328
329 pub type ProvidedValue<'tcx> = query_if_arena!(
333 [$($modifiers)*]
334 (<$V as $crate::query::arena_cached::ArenaCached<'tcx>>::Provided)
335 ($V)
336 );
337
338 #[inline(always)]
343 pub fn provided_to_erased<'tcx>(
344 _tcx: TyCtxt<'tcx>,
345 value: ProvidedValue<'tcx>,
346 ) -> Erase<Value<'tcx>> {
347 erase(query_if_arena!([$($modifiers)*]
348 {
349 use $crate::query::arena_cached::ArenaCached;
350
351 if mem::needs_drop::<<$V as ArenaCached<'tcx>>::Allocated>() {
352 <$V as ArenaCached>::alloc_in_arena(
353 |v| _tcx.query_system.arenas.$name.alloc(v),
354 value,
355 )
356 } else {
357 <$V as ArenaCached>::alloc_in_arena(
358 |v| _tcx.arena.dropless.alloc(v),
359 value,
360 )
361 }
362 }
363 (value)
364 ))
365 }
366
367 pub type Storage<'tcx> = <$($K)* as keys::Key>::Cache<Erase<$V>>;
368
369 #[cfg(target_pointer_width = "64")]
372 const _: () = {
373 if size_of::<Key<'static>>() > 88 {
374 panic!("{}", concat!(
375 "the query `",
376 stringify!($name),
377 "` has a key type `",
378 stringify!($($K)*),
379 "` that is too large"
380 ));
381 }
382 };
383
384 #[cfg(target_pointer_width = "64")]
387 #[cfg(not(feature = "rustc_randomized_layouts"))]
388 const _: () = {
389 if size_of::<Value<'static>>() > 64 {
390 panic!("{}", concat!(
391 "the query `",
392 stringify!($name),
393 "` has a value type `",
394 stringify!($V),
395 "` that is too large"
396 ));
397 }
398 };
399 })*
400 }
401
402 pub struct QueryArenas<'tcx> {
403 $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
404 (TypedArena<<$V as $crate::query::arena_cached::ArenaCached<'tcx>>::Allocated>)
405 ()
406 ),)*
407 }
408
409 impl Default for QueryArenas<'_> {
410 fn default() -> Self {
411 Self {
412 $($name: query_if_arena!([$($modifiers)*]
413 (Default::default())
414 ()
415 ),)*
416 }
417 }
418 }
419
420 #[derive(Default)]
421 pub struct QueryCaches<'tcx> {
422 $($(#[$attr])* pub $name: queries::$name::Storage<'tcx>,)*
423 }
424
425 impl<'tcx> TyCtxtEnsureOk<'tcx> {
426 $($(#[$attr])*
427 #[inline(always)]
428 pub fn $name(
429 self,
430 key: query_helper_param_ty!($($K)*),
431 ) -> ensure_ok_result!([$($modifiers)*]) {
432 query_ensure!(
433 [$($modifiers)*]
434 self.tcx,
435 self.tcx.query_system.fns.engine.$name,
436 &self.tcx.query_system.caches.$name,
437 key.into_query_param(),
438 false,
439 )
440 })*
441 }
442
443 impl<'tcx> TyCtxtEnsureDone<'tcx> {
444 $($(#[$attr])*
445 #[inline(always)]
446 pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
447 query_ensure(
448 self.tcx,
449 self.tcx.query_system.fns.engine.$name,
450 &self.tcx.query_system.caches.$name,
451 key.into_query_param(),
452 true,
453 );
454 })*
455 }
456
457 impl<'tcx> TyCtxt<'tcx> {
458 $($(#[$attr])*
459 #[inline(always)]
460 #[must_use]
461 pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
462 {
463 self.at(DUMMY_SP).$name(key)
464 })*
465 }
466
467 impl<'tcx> TyCtxtAt<'tcx> {
468 $($(#[$attr])*
469 #[inline(always)]
470 pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
471 {
472 restore::<$V>(query_get_at(
473 self.tcx,
474 self.tcx.query_system.fns.engine.$name,
475 &self.tcx.query_system.caches.$name,
476 self.span,
477 key.into_query_param(),
478 ))
479 })*
480 }
481
482 pub struct DynamicQueries<'tcx> {
483 $(
484 pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>>,
485 )*
486 }
487
488 #[derive(Default)]
489 pub struct QueryStates<'tcx> {
490 $(
491 pub $name: QueryState<$($K)*>,
492 )*
493 }
494
495 pub struct Providers {
496 $(pub $name: for<'tcx> fn(
497 TyCtxt<'tcx>,
498 queries::$name::LocalKey<'tcx>,
499 ) -> queries::$name::ProvidedValue<'tcx>,)*
500 }
501
502 pub struct ExternProviders {
503 $(pub $name: separate_provide_extern_decl!([$($modifiers)*][$name]),)*
504 }
505
506 impl Default for Providers {
507 fn default() -> Self {
508 Providers {
509 $($name: |_, key| $crate::query::plumbing::default_query(stringify!($name), &key)),*
510 }
511 }
512 }
513
514 impl Default for ExternProviders {
515 fn default() -> Self {
516 ExternProviders {
517 $($name: separate_provide_extern_default!([$($modifiers)*][$name]),)*
518 }
519 }
520 }
521
522 impl Copy for Providers {}
523 impl Clone for Providers {
524 fn clone(&self) -> Self { *self }
525 }
526
527 impl Copy for ExternProviders {}
528 impl Clone for ExternProviders {
529 fn clone(&self) -> Self { *self }
530 }
531
532 pub struct QueryEngine {
533 $(pub $name: for<'tcx> fn(
534 TyCtxt<'tcx>,
535 Span,
536 queries::$name::Key<'tcx>,
537 QueryMode,
538 ) -> Option<Erase<$V>>,)*
539 }
540 };
541}
542
543macro_rules! hash_result {
544 ([]) => {{
545 Some(dep_graph::hash_result)
546 }};
547 ([(no_hash) $($rest:tt)*]) => {{
548 None
549 }};
550 ([$other:tt $($modifiers:tt)*]) => {
551 hash_result!([$($modifiers)*])
552 };
553}
554
555macro_rules! define_feedable {
556 ($($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
557 $(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
558 $(#[$attr])*
559 #[inline(always)]
560 pub fn $name(self, value: queries::$name::ProvidedValue<'tcx>) {
561 let key = self.key().into_query_param();
562
563 let tcx = self.tcx;
564 let erased = queries::$name::provided_to_erased(tcx, value);
565 let value = restore::<$V>(erased);
566 let cache = &tcx.query_system.caches.$name;
567
568 let hasher: Option<fn(&mut StableHashingContext<'_>, &_) -> _> = hash_result!([$($modifiers)*]);
569 match try_get_cached(tcx, cache, &key) {
570 Some(old) => {
571 let old = restore::<$V>(old);
572 if let Some(hasher) = hasher {
573 let (value_hash, old_hash): (Fingerprint, Fingerprint) = tcx.with_stable_hashing_context(|mut hcx|
574 (hasher(&mut hcx, &value), hasher(&mut hcx, &old))
575 );
576 if old_hash != value_hash {
577 tcx.dcx().delayed_bug(format!(
581 "Trying to feed an already recorded value for query {} key={key:?}:\n\
582 old value: {old:?}\nnew value: {value:?}",
583 stringify!($name),
584 ));
585 }
586 } else {
587 bug!(
591 "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
592 stringify!($name),
593 )
594 }
595 }
596 None => {
597 let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::dep_kinds::$name, &key);
598 let dep_node_index = tcx.dep_graph.with_feed_task(
599 dep_node,
600 tcx,
601 &value,
602 hash_result!([$($modifiers)*]),
603 );
604 cache.complete(key, erased, dep_node_index);
605 }
606 }
607 }
608 })*
609 }
610}
611
612mod sealed {
625 use rustc_hir::def_id::{LocalModDefId, ModDefId};
626
627 use super::{DefId, LocalDefId, OwnerId};
628
629 pub trait IntoQueryParam<P> {
634 fn into_query_param(self) -> P;
635 }
636
637 impl<P> IntoQueryParam<P> for P {
638 #[inline(always)]
639 fn into_query_param(self) -> P {
640 self
641 }
642 }
643
644 impl<'a, P: Copy> IntoQueryParam<P> for &'a P {
645 #[inline(always)]
646 fn into_query_param(self) -> P {
647 *self
648 }
649 }
650
651 impl IntoQueryParam<LocalDefId> for OwnerId {
652 #[inline(always)]
653 fn into_query_param(self) -> LocalDefId {
654 self.def_id
655 }
656 }
657
658 impl IntoQueryParam<DefId> for LocalDefId {
659 #[inline(always)]
660 fn into_query_param(self) -> DefId {
661 self.to_def_id()
662 }
663 }
664
665 impl IntoQueryParam<DefId> for OwnerId {
666 #[inline(always)]
667 fn into_query_param(self) -> DefId {
668 self.to_def_id()
669 }
670 }
671
672 impl IntoQueryParam<DefId> for ModDefId {
673 #[inline(always)]
674 fn into_query_param(self) -> DefId {
675 self.to_def_id()
676 }
677 }
678
679 impl IntoQueryParam<DefId> for LocalModDefId {
680 #[inline(always)]
681 fn into_query_param(self) -> DefId {
682 self.to_def_id()
683 }
684 }
685
686 impl IntoQueryParam<LocalDefId> for LocalModDefId {
687 #[inline(always)]
688 fn into_query_param(self) -> LocalDefId {
689 self.into()
690 }
691 }
692}
693
694pub use sealed::IntoQueryParam;
695
696use super::erase::EraseType;
697
698#[derive(Copy, Clone, Debug, HashStable)]
699pub struct CyclePlaceholder(pub ErrorGuaranteed);
700
701#[cold]
702pub(crate) fn default_query(name: &str, key: &dyn std::fmt::Debug) -> ! {
703 bug!(
704 "`tcx.{name}({key:?})` is not supported for this key;\n\
705 hint: Queries can be either made to the local crate, or the external crate. \
706 This error means you tried to use it for one that's not supported.\n\
707 If that's not the case, {name} was likely never assigned to a provider function.\n",
708 )
709}
710
711#[cold]
712pub(crate) fn default_extern_query(name: &str, key: &dyn std::fmt::Debug) -> ! {
713 bug!(
714 "`tcx.{name}({key:?})` unsupported by its crate; \
715 perhaps the `{name}` query was never assigned a provider function",
716 )
717}