rustc_query_impl/
plumbing.rs

1//! The implementation of the query system itself. This defines the macros that
2//! generate the actual methods on tcx which find and execute the provider,
3//! manage the caches, and so forth.
4
5use std::num::NonZero;
6
7use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
8use rustc_data_structures::unord::UnordMap;
9use rustc_hashes::Hash64;
10use rustc_index::Idx;
11use rustc_middle::bug;
12use rustc_middle::dep_graph::{
13    self, DepContext, DepKind, DepKindStruct, DepNode, DepNodeIndex, SerializedDepNodeIndex,
14    dep_kinds,
15};
16use rustc_middle::query::Key;
17use rustc_middle::query::on_disk_cache::{
18    AbsoluteBytePos, CacheDecoder, CacheEncoder, EncodedDepNodeIndex,
19};
20use rustc_middle::ty::codec::TyEncoder;
21use rustc_middle::ty::print::with_reduced_queries;
22use rustc_middle::ty::tls::{self, ImplicitCtxt};
23use rustc_middle::ty::{self, TyCtxt};
24use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext};
25use rustc_query_system::ich::StableHashingContext;
26use rustc_query_system::query::{
27    QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffect, QueryStackFrame,
28    force_query,
29};
30use rustc_query_system::{QueryOverflow, QueryOverflowNote};
31use rustc_serialize::{Decodable, Encodable};
32use rustc_session::Limit;
33use rustc_span::def_id::LOCAL_CRATE;
34
35use crate::QueryConfigRestored;
36
37#[derive(Copy, Clone)]
38pub struct QueryCtxt<'tcx> {
39    pub tcx: TyCtxt<'tcx>,
40}
41
42impl<'tcx> QueryCtxt<'tcx> {
43    #[inline]
44    pub fn new(tcx: TyCtxt<'tcx>) -> Self {
45        QueryCtxt { tcx }
46    }
47}
48
49impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
50    type Target = TyCtxt<'tcx>;
51
52    #[inline]
53    fn deref(&self) -> &Self::Target {
54        &self.tcx
55    }
56}
57
58impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
59    type Deps = rustc_middle::dep_graph::DepsType;
60    type DepContext = TyCtxt<'tcx>;
61
62    #[inline]
63    fn dep_context(&self) -> &Self::DepContext {
64        &self.tcx
65    }
66}
67
68impl QueryContext for QueryCtxt<'_> {
69    #[inline]
70    fn next_job_id(self) -> QueryJobId {
71        QueryJobId(
72            NonZero::new(self.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed))
73                .unwrap(),
74        )
75    }
76
77    #[inline]
78    fn current_query_job(self) -> Option<QueryJobId> {
79        tls::with_related_context(self.tcx, |icx| icx.query)
80    }
81
82    /// Returns a query map representing active query jobs.
83    /// It returns an incomplete map as an error if it fails
84    /// to take locks.
85    fn collect_active_jobs(self) -> Result<QueryMap, QueryMap> {
86        let mut jobs = QueryMap::default();
87        let mut complete = true;
88
89        for collect in super::TRY_COLLECT_ACTIVE_JOBS.iter() {
90            if collect(self.tcx, &mut jobs).is_none() {
91                complete = false;
92            }
93        }
94
95        if complete { Ok(jobs) } else { Err(jobs) }
96    }
97
98    // Interactions with on_disk_cache
99    fn load_side_effect(
100        self,
101        prev_dep_node_index: SerializedDepNodeIndex,
102    ) -> Option<QuerySideEffect> {
103        self.query_system
104            .on_disk_cache
105            .as_ref()
106            .and_then(|c| c.load_side_effect(self.tcx, prev_dep_node_index))
107    }
108
109    #[inline(never)]
110    #[cold]
111    fn store_side_effect(self, dep_node_index: DepNodeIndex, side_effect: QuerySideEffect) {
112        if let Some(c) = self.query_system.on_disk_cache.as_ref() {
113            c.store_side_effect(dep_node_index, side_effect)
114        }
115    }
116
117    /// Executes a job by changing the `ImplicitCtxt` to point to the
118    /// new query job while it executes.
119    #[inline(always)]
120    fn start_query<R>(
121        self,
122        token: QueryJobId,
123        depth_limit: bool,
124        compute: impl FnOnce() -> R,
125    ) -> R {
126        // The `TyCtxt` stored in TLS has the same global interner lifetime
127        // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
128        // when accessing the `ImplicitCtxt`.
129        tls::with_related_context(self.tcx, move |current_icx| {
130            if depth_limit && !self.recursion_limit().value_within_limit(current_icx.query_depth) {
131                self.depth_limit_error(token);
132            }
133
134            // Update the `ImplicitCtxt` to point to our new query job.
135            let new_icx = ImplicitCtxt {
136                tcx: self.tcx,
137                query: Some(token),
138                query_depth: current_icx.query_depth + depth_limit as usize,
139                task_deps: current_icx.task_deps,
140            };
141
142            // Use the `ImplicitCtxt` while we execute the query.
143            tls::enter_context(&new_icx, compute)
144        })
145    }
146
147    fn depth_limit_error(self, job: QueryJobId) {
148        // FIXME: `collect_active_jobs` expects no locks to be held, which doesn't hold for this call.
149        let query_map = match self.collect_active_jobs() {
150            Ok(query_map) => query_map,
151            Err(query_map) => query_map,
152        };
153        let (info, depth) = job.find_dep_kind_root(query_map);
154
155        let suggested_limit = match self.recursion_limit() {
156            Limit(0) => Limit(2),
157            limit => limit * 2,
158        };
159
160        self.sess.dcx().emit_fatal(QueryOverflow {
161            span: info.job.span,
162            note: QueryOverflowNote { desc: info.query.description, depth },
163            suggested_limit,
164            crate_name: self.crate_name(LOCAL_CRATE),
165        });
166    }
167}
168
169pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
170    tcx.dep_graph.try_mark_green(QueryCtxt::new(tcx), dep_node).is_some()
171}
172
173pub(super) fn encode_all_query_results<'tcx>(
174    tcx: TyCtxt<'tcx>,
175    encoder: &mut CacheEncoder<'_, 'tcx>,
176    query_result_index: &mut EncodedDepNodeIndex,
177) {
178    for encode in super::ENCODE_QUERY_RESULTS.iter().copied().flatten() {
179        encode(tcx, encoder, query_result_index);
180    }
181}
182
183pub fn query_key_hash_verify_all<'tcx>(tcx: TyCtxt<'tcx>) {
184    if tcx.sess().opts.unstable_opts.incremental_verify_ich || cfg!(debug_assertions) {
185        tcx.sess.time("query_key_hash_verify_all", || {
186            for verify in super::QUERY_KEY_HASH_VERIFY.iter() {
187                verify(tcx);
188            }
189        })
190    }
191}
192
193macro_rules! handle_cycle_error {
194    ([]) => {{
195        rustc_query_system::HandleCycleError::Error
196    }};
197    ([(fatal_cycle) $($rest:tt)*]) => {{
198        rustc_query_system::HandleCycleError::Fatal
199    }};
200    ([(cycle_stash) $($rest:tt)*]) => {{
201        rustc_query_system::HandleCycleError::Stash
202    }};
203    ([(cycle_delay_bug) $($rest:tt)*]) => {{
204        rustc_query_system::HandleCycleError::DelayBug
205    }};
206    ([$other:tt $($modifiers:tt)*]) => {
207        handle_cycle_error!([$($modifiers)*])
208    };
209}
210
211macro_rules! is_anon {
212    ([]) => {{
213        false
214    }};
215    ([(anon) $($rest:tt)*]) => {{
216        true
217    }};
218    ([$other:tt $($modifiers:tt)*]) => {
219        is_anon!([$($modifiers)*])
220    };
221}
222
223macro_rules! is_eval_always {
224    ([]) => {{
225        false
226    }};
227    ([(eval_always) $($rest:tt)*]) => {{
228        true
229    }};
230    ([$other:tt $($modifiers:tt)*]) => {
231        is_eval_always!([$($modifiers)*])
232    };
233}
234
235macro_rules! depth_limit {
236    ([]) => {{
237        false
238    }};
239    ([(depth_limit) $($rest:tt)*]) => {{
240        true
241    }};
242    ([$other:tt $($modifiers:tt)*]) => {
243        depth_limit!([$($modifiers)*])
244    };
245}
246
247macro_rules! feedable {
248    ([]) => {{
249        false
250    }};
251    ([(feedable) $($rest:tt)*]) => {{
252        true
253    }};
254    ([$other:tt $($modifiers:tt)*]) => {
255        feedable!([$($modifiers)*])
256    };
257}
258
259macro_rules! hash_result {
260    ([][$V:ty]) => {{
261        Some(|hcx, result| dep_graph::hash_result(hcx, &restore::<$V>(*result)))
262    }};
263    ([(no_hash) $($rest:tt)*][$V:ty]) => {{
264        None
265    }};
266    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
267        hash_result!([$($modifiers)*][$($args)*])
268    };
269}
270
271macro_rules! call_provider {
272    ([][$tcx:expr, $name:ident, $key:expr]) => {{
273        ($tcx.query_system.fns.local_providers.$name)($tcx, $key)
274    }};
275    ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
276        if let Some(key) = $key.as_local_key() {
277            ($tcx.query_system.fns.local_providers.$name)($tcx, key)
278        } else {
279            ($tcx.query_system.fns.extern_providers.$name)($tcx, $key)
280        }
281    }};
282    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
283        call_provider!([$($modifiers)*][$($args)*])
284    };
285}
286
287macro_rules! should_ever_cache_on_disk {
288    ([]$yes:tt $no:tt) => {{
289        $no
290    }};
291    ([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{
292        $yes
293    }};
294    ([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => {
295        should_ever_cache_on_disk!([$($modifiers)*]$yes $no)
296    };
297}
298
299pub(crate) fn create_query_frame<
300    'tcx,
301    K: Copy + Key + for<'a> HashStable<StableHashingContext<'a>>,
302>(
303    tcx: TyCtxt<'tcx>,
304    do_describe: fn(TyCtxt<'tcx>, K) -> String,
305    key: K,
306    kind: DepKind,
307    name: &'static str,
308) -> QueryStackFrame {
309    // If reduced queries are requested, we may be printing a query stack due
310    // to a panic. Avoid using `default_span` and `def_kind` in that case.
311    let reduce_queries = with_reduced_queries();
312
313    // Avoid calling queries while formatting the description
314    let description = ty::print::with_no_queries!(do_describe(tcx, key));
315    let description = if tcx.sess.verbose_internals() {
316        format!("{description} [{name:?}]")
317    } else {
318        description
319    };
320    let span = if kind == dep_graph::dep_kinds::def_span || reduce_queries {
321        // The `def_span` query is used to calculate `default_span`,
322        // so exit to avoid infinite recursion.
323        None
324    } else {
325        Some(key.default_span(tcx))
326    };
327    let def_id = key.key_as_def_id();
328    let def_kind = if kind == dep_graph::dep_kinds::def_kind || reduce_queries {
329        // Try to avoid infinite recursion.
330        None
331    } else {
332        def_id.and_then(|def_id| def_id.as_local()).map(|def_id| tcx.def_kind(def_id))
333    };
334    let hash = || {
335        tcx.with_stable_hashing_context(|mut hcx| {
336            let mut hasher = StableHasher::new();
337            kind.as_usize().hash_stable(&mut hcx, &mut hasher);
338            key.hash_stable(&mut hcx, &mut hasher);
339            hasher.finish::<Hash64>()
340        })
341    };
342    let def_id_for_ty_in_cycle = key.def_id_for_ty_in_cycle();
343
344    QueryStackFrame::new(description, span, def_id, def_kind, kind, def_id_for_ty_in_cycle, hash)
345}
346
347pub(crate) fn encode_query_results<'a, 'tcx, Q>(
348    query: Q::Config,
349    qcx: QueryCtxt<'tcx>,
350    encoder: &mut CacheEncoder<'a, 'tcx>,
351    query_result_index: &mut EncodedDepNodeIndex,
352) where
353    Q: super::QueryConfigRestored<'tcx>,
354    Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>,
355{
356    let _timer = qcx.profiler().generic_activity_with_arg("encode_query_results_for", query.name());
357
358    assert!(query.query_state(qcx).all_inactive());
359    let cache = query.query_cache(qcx);
360    cache.iter(&mut |key, value, dep_node| {
361        if query.cache_on_disk(qcx.tcx, key) {
362            let dep_node = SerializedDepNodeIndex::new(dep_node.index());
363
364            // Record position of the cache entry.
365            query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position())));
366
367            // Encode the type check tables with the `SerializedDepNodeIndex`
368            // as tag.
369            encoder.encode_tagged(dep_node, &Q::restore(*value));
370        }
371    });
372}
373
374pub(crate) fn query_key_hash_verify<'tcx>(
375    query: impl QueryConfig<QueryCtxt<'tcx>>,
376    qcx: QueryCtxt<'tcx>,
377) {
378    let _timer =
379        qcx.profiler().generic_activity_with_arg("query_key_hash_verify_for", query.name());
380
381    let mut map = UnordMap::default();
382
383    let cache = query.query_cache(qcx);
384    cache.iter(&mut |key, _, _| {
385        let node = DepNode::construct(qcx.tcx, query.dep_kind(), key);
386        if let Some(other_key) = map.insert(node, *key) {
387            bug!(
388                "query key:\n\
389                `{:?}`\n\
390                and key:\n\
391                `{:?}`\n\
392                mapped to the same dep node:\n\
393                {:?}",
394                key,
395                other_key,
396                node
397            );
398        }
399    });
400}
401
402fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode)
403where
404    Q: QueryConfig<QueryCtxt<'tcx>>,
405{
406    debug_assert!(tcx.dep_graph.is_green(&dep_node));
407
408    let key = Q::Key::recover(tcx, &dep_node).unwrap_or_else(|| {
409        panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)
410    });
411    if query.cache_on_disk(tcx, &key) {
412        let _ = query.execute_query(tcx, key);
413    }
414}
415
416pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeIndex) -> bool {
417    if let Some(cache) = tcx.query_system.on_disk_cache.as_ref() {
418        cache.loadable_from_disk(id)
419    } else {
420        false
421    }
422}
423
424pub(crate) fn try_load_from_disk<'tcx, V>(
425    tcx: TyCtxt<'tcx>,
426    prev_index: SerializedDepNodeIndex,
427    index: DepNodeIndex,
428) -> Option<V>
429where
430    V: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
431{
432    let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?;
433
434    let prof_timer = tcx.prof.incr_cache_loading();
435
436    // The call to `with_query_deserialization` enforces that no new `DepNodes`
437    // are created during deserialization. See the docs of that method for more
438    // details.
439    let value = tcx
440        .dep_graph
441        .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index));
442
443    prof_timer.finish_with_query_invocation_id(index.into());
444
445    value
446}
447
448fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool
449where
450    Q: QueryConfig<QueryCtxt<'tcx>>,
451{
452    // We must avoid ever having to call `force_from_dep_node()` for a
453    // `DepNode::codegen_unit`:
454    // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
455    // would always end up having to evaluate the first caller of the
456    // `codegen_unit` query that *is* reconstructible. This might very well be
457    // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
458    // to re-trigger calling the `codegen_unit` query with the right key. At
459    // that point we would already have re-done all the work we are trying to
460    // avoid doing in the first place.
461    // The solution is simple: Just explicitly call the `codegen_unit` query for
462    // each CGU, right after partitioning. This way `try_mark_green` will always
463    // hit the cache instead of having to go through `force_from_dep_node`.
464    // This assertion makes sure, we actually keep applying the solution above.
465    debug_assert!(
466        dep_node.kind != dep_kinds::codegen_unit,
467        "calling force_from_dep_node() on dep_kinds::codegen_unit"
468    );
469
470    if let Some(key) = Q::Key::recover(tcx, &dep_node) {
471        force_query(query, QueryCtxt::new(tcx), key, dep_node);
472        true
473    } else {
474        false
475    }
476}
477
478pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx>
479where
480    Q: QueryConfigRestored<'tcx>,
481{
482    let fingerprint_style = <Q::Config as QueryConfig<QueryCtxt<'tcx>>>::Key::fingerprint_style();
483
484    if is_anon || !fingerprint_style.reconstructible() {
485        return DepKindStruct {
486            is_anon,
487            is_eval_always,
488            fingerprint_style,
489            force_from_dep_node: None,
490            try_load_from_on_disk_cache: None,
491            name: Q::NAME,
492        };
493    }
494
495    DepKindStruct {
496        is_anon,
497        is_eval_always,
498        fingerprint_style,
499        force_from_dep_node: Some(|tcx, dep_node, _| {
500            force_from_dep_node(Q::config(tcx), tcx, dep_node)
501        }),
502        try_load_from_on_disk_cache: Some(|tcx, dep_node| {
503            try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node)
504        }),
505        name: Q::NAME,
506    }
507}
508
509macro_rules! item_if_cached {
510    ([] $tokens:tt) => {};
511    ([(cache) $($rest:tt)*] { $($tokens:tt)* }) => {
512        $($tokens)*
513    };
514    ([$other:tt $($modifiers:tt)*] $tokens:tt) => {
515        item_if_cached! { [$($modifiers)*] $tokens }
516    };
517}
518
519macro_rules! expand_if_cached {
520    ([], $tokens:expr) => {{
521        None
522    }};
523    ([(cache) $($rest:tt)*], $tokens:expr) => {{
524        Some($tokens)
525    }};
526    ([$other:tt $($modifiers:tt)*], $tokens:expr) => {
527        expand_if_cached!([$($modifiers)*], $tokens)
528    };
529}
530
531/// Don't show the backtrace for query system by default
532/// use `RUST_BACKTRACE=full` to show all the backtraces
533#[inline(never)]
534pub(crate) fn __rust_begin_short_backtrace<F, T>(f: F) -> T
535where
536    F: FnOnce() -> T,
537{
538    let result = f();
539    std::hint::black_box(());
540    result
541}
542
543// NOTE: `$V` isn't used here, but we still need to match on it so it can be passed to other macros
544// invoked by `rustc_query_append`.
545macro_rules! define_queries {
546    (
547     $($(#[$attr:meta])*
548        [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
549
550        pub(crate) mod query_impl { $(pub(crate) mod $name {
551            use super::super::*;
552            use std::marker::PhantomData;
553
554            pub(crate) mod get_query_incr {
555                use super::*;
556
557                // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames
558                // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming
559                #[inline(never)]
560                pub(crate) fn __rust_end_short_backtrace<'tcx>(
561                    tcx: TyCtxt<'tcx>,
562                    span: Span,
563                    key: queries::$name::Key<'tcx>,
564                    mode: QueryMode,
565                ) -> Option<Erase<queries::$name::Value<'tcx>>> {
566                    #[cfg(debug_assertions)]
567                    let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
568                    get_query_incr(
569                        QueryType::config(tcx),
570                        QueryCtxt::new(tcx),
571                        span,
572                        key,
573                        mode
574                    )
575                }
576            }
577
578            pub(crate) mod get_query_non_incr {
579                use super::*;
580
581                #[inline(never)]
582                pub(crate) fn __rust_end_short_backtrace<'tcx>(
583                    tcx: TyCtxt<'tcx>,
584                    span: Span,
585                    key: queries::$name::Key<'tcx>,
586                    __mode: QueryMode,
587                ) -> Option<Erase<queries::$name::Value<'tcx>>> {
588                    Some(get_query_non_incr(
589                        QueryType::config(tcx),
590                        QueryCtxt::new(tcx),
591                        span,
592                        key,
593                    ))
594                }
595            }
596
597            pub(crate) fn dynamic_query<'tcx>()
598                -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>>
599            {
600                DynamicQuery {
601                    name: stringify!($name),
602                    eval_always: is_eval_always!([$($modifiers)*]),
603                    dep_kind: dep_graph::dep_kinds::$name,
604                    handle_cycle_error: handle_cycle_error!([$($modifiers)*]),
605                    query_state: std::mem::offset_of!(QueryStates<'tcx>, $name),
606                    query_cache: std::mem::offset_of!(QueryCaches<'tcx>, $name),
607                    cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key),
608                    execute_query: |tcx, key| erase(tcx.$name(key)),
609                    compute: |tcx, key| {
610                        #[cfg(debug_assertions)]
611                        let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
612                        __rust_begin_short_backtrace(||
613                            queries::$name::provided_to_erased(
614                                tcx,
615                                {
616                                    let ret = call_provider!([$($modifiers)*][tcx, $name, key]);
617                                    rustc_middle::ty::print::with_reduced_queries!({
618                                        tracing::trace!(?ret);
619                                    });
620                                    ret
621                                }
622                            )
623                        )
624                    },
625                    can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false),
626                    try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] {
627                        |tcx, key, prev_index, index| {
628                            if ::rustc_middle::query::cached::$name(tcx, key) {
629                                let value = $crate::plumbing::try_load_from_disk::<
630                                    queries::$name::ProvidedValue<'tcx>
631                                >(
632                                    tcx,
633                                    prev_index,
634                                    index,
635                                );
636                                value.map(|value| queries::$name::provided_to_erased(tcx, value))
637                            } else {
638                                None
639                            }
640                        }
641                    } {
642                        |_tcx, _key, _prev_index, _index| None
643                    }),
644                    value_from_cycle_error: |tcx, cycle, guar| {
645                        let result: queries::$name::Value<'tcx> = Value::from_cycle_error(tcx, cycle, guar);
646                        erase(result)
647                    },
648                    loadable_from_disk: |_tcx, _key, _index| {
649                        should_ever_cache_on_disk!([$($modifiers)*] {
650                            ::rustc_middle::query::cached::$name(_tcx, _key) &&
651                                $crate::plumbing::loadable_from_disk(_tcx, _index)
652                        } {
653                            false
654                        })
655                    },
656                    hash_result: hash_result!([$($modifiers)*][queries::$name::Value<'tcx>]),
657                    format_value: |value| format!("{:?}", restore::<queries::$name::Value<'tcx>>(*value)),
658                }
659            }
660
661            #[derive(Copy, Clone, Default)]
662            pub(crate) struct QueryType<'tcx> {
663                data: PhantomData<&'tcx ()>
664            }
665
666            impl<'tcx> QueryConfigRestored<'tcx> for QueryType<'tcx> {
667                type RestoredValue = queries::$name::Value<'tcx>;
668                type Config = DynamicConfig<
669                    'tcx,
670                    queries::$name::Storage<'tcx>,
671                    { is_anon!([$($modifiers)*]) },
672                    { depth_limit!([$($modifiers)*]) },
673                    { feedable!([$($modifiers)*]) },
674                >;
675
676                const NAME: &'static &'static str = &stringify!($name);
677
678                #[inline(always)]
679                fn config(tcx: TyCtxt<'tcx>) -> Self::Config {
680                    DynamicConfig {
681                        dynamic: &tcx.query_system.dynamic_queries.$name,
682                    }
683                }
684
685                #[inline(always)]
686                fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue {
687                    restore::<queries::$name::Value<'tcx>>(value)
688                }
689            }
690
691            pub(crate) fn try_collect_active_jobs<'tcx>(tcx: TyCtxt<'tcx>, qmap: &mut QueryMap) -> Option<()> {
692                let make_query = |tcx, key| {
693                    let kind = rustc_middle::dep_graph::dep_kinds::$name;
694                    let name = stringify!($name);
695                    $crate::plumbing::create_query_frame(tcx, rustc_middle::query::descs::$name, key, kind, name)
696                };
697                let res = tcx.query_system.states.$name.try_collect_active_jobs(
698                    tcx,
699                    make_query,
700                    qmap,
701                );
702                // this can be called during unwinding, and the function has a `try_`-prefix, so
703                // don't `unwrap()` here, just manually check for `None` and do best-effort error
704                // reporting.
705                if res.is_none() {
706                    tracing::warn!(
707                        "Failed to collect active jobs for query with name `{}`!",
708                        stringify!($name)
709                    );
710                }
711                res
712            }
713
714            pub(crate) fn alloc_self_profile_query_strings<'tcx>(
715                tcx: TyCtxt<'tcx>,
716                string_cache: &mut QueryKeyStringCache
717            ) {
718                $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache(
719                    tcx,
720                    stringify!($name),
721                    &tcx.query_system.caches.$name,
722                    string_cache,
723                )
724            }
725
726            item_if_cached! { [$($modifiers)*] {
727                pub(crate) fn encode_query_results<'tcx>(
728                    tcx: TyCtxt<'tcx>,
729                    encoder: &mut CacheEncoder<'_, 'tcx>,
730                    query_result_index: &mut EncodedDepNodeIndex
731                ) {
732                    $crate::plumbing::encode_query_results::<query_impl::$name::QueryType<'tcx>>(
733                        query_impl::$name::QueryType::config(tcx),
734                        QueryCtxt::new(tcx),
735                        encoder,
736                        query_result_index,
737                    )
738                }
739            }}
740
741            pub(crate) fn query_key_hash_verify<'tcx>(tcx: TyCtxt<'tcx>) {
742                $crate::plumbing::query_key_hash_verify(
743                    query_impl::$name::QueryType::config(tcx),
744                    QueryCtxt::new(tcx),
745                )
746            }
747        })*}
748
749        pub(crate) fn engine(incremental: bool) -> QueryEngine {
750            if incremental {
751                QueryEngine {
752                    $($name: query_impl::$name::get_query_incr::__rust_end_short_backtrace,)*
753                }
754            } else {
755                QueryEngine {
756                    $($name: query_impl::$name::get_query_non_incr::__rust_end_short_backtrace,)*
757                }
758            }
759        }
760
761        pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> {
762            DynamicQueries {
763                $(
764                    $name: query_impl::$name::dynamic_query(),
765                )*
766            }
767        }
768
769        // These arrays are used for iteration and can't be indexed by `DepKind`.
770
771        const TRY_COLLECT_ACTIVE_JOBS: &[for<'tcx> fn(TyCtxt<'tcx>, &mut QueryMap) -> Option<()>] =
772            &[$(query_impl::$name::try_collect_active_jobs),*];
773
774        const ALLOC_SELF_PROFILE_QUERY_STRINGS: &[
775            for<'tcx> fn(TyCtxt<'tcx>, &mut QueryKeyStringCache)
776        ] = &[$(query_impl::$name::alloc_self_profile_query_strings),*];
777
778        const ENCODE_QUERY_RESULTS: &[
779            Option<for<'tcx> fn(
780                TyCtxt<'tcx>,
781                &mut CacheEncoder<'_, 'tcx>,
782                &mut EncodedDepNodeIndex)
783            >
784        ] = &[$(expand_if_cached!([$($modifiers)*], query_impl::$name::encode_query_results)),*];
785
786        const QUERY_KEY_HASH_VERIFY: &[
787            for<'tcx> fn(TyCtxt<'tcx>)
788        ] = &[$(query_impl::$name::query_key_hash_verify),*];
789
790        #[allow(nonstandard_style)]
791        mod query_callbacks {
792            use super::*;
793            use rustc_middle::bug;
794            use rustc_query_system::dep_graph::FingerprintStyle;
795
796            // We use this for most things when incr. comp. is turned off.
797            pub(crate) fn Null<'tcx>() -> DepKindStruct<'tcx> {
798                DepKindStruct {
799                    is_anon: false,
800                    is_eval_always: false,
801                    fingerprint_style: FingerprintStyle::Unit,
802                    force_from_dep_node: Some(|_, dep_node, _| bug!("force_from_dep_node: encountered {:?}", dep_node)),
803                    try_load_from_on_disk_cache: None,
804                    name: &"Null",
805                }
806            }
807
808            // We use this for the forever-red node.
809            pub(crate) fn Red<'tcx>() -> DepKindStruct<'tcx> {
810                DepKindStruct {
811                    is_anon: false,
812                    is_eval_always: false,
813                    fingerprint_style: FingerprintStyle::Unit,
814                    force_from_dep_node: Some(|_, dep_node, _| bug!("force_from_dep_node: encountered {:?}", dep_node)),
815                    try_load_from_on_disk_cache: None,
816                    name: &"Red",
817                }
818            }
819
820            pub(crate) fn SideEffect<'tcx>() -> DepKindStruct<'tcx> {
821                DepKindStruct {
822                    is_anon: false,
823                    is_eval_always: false,
824                    fingerprint_style: FingerprintStyle::Unit,
825                    force_from_dep_node: Some(|tcx, _, prev_index| {
826                        tcx.dep_graph.force_diagnostic_node(QueryCtxt::new(tcx), prev_index);
827                        true
828                    }),
829                    try_load_from_on_disk_cache: None,
830                    name: &"SideEffect",
831                }
832            }
833
834            pub(crate) fn TraitSelect<'tcx>() -> DepKindStruct<'tcx> {
835                DepKindStruct {
836                    is_anon: true,
837                    is_eval_always: false,
838                    fingerprint_style: FingerprintStyle::Unit,
839                    force_from_dep_node: None,
840                    try_load_from_on_disk_cache: None,
841                    name: &"TraitSelect",
842                }
843            }
844
845            pub(crate) fn CompileCodegenUnit<'tcx>() -> DepKindStruct<'tcx> {
846                DepKindStruct {
847                    is_anon: false,
848                    is_eval_always: false,
849                    fingerprint_style: FingerprintStyle::Opaque,
850                    force_from_dep_node: None,
851                    try_load_from_on_disk_cache: None,
852                    name: &"CompileCodegenUnit",
853                }
854            }
855
856            pub(crate) fn CompileMonoItem<'tcx>() -> DepKindStruct<'tcx> {
857                DepKindStruct {
858                    is_anon: false,
859                    is_eval_always: false,
860                    fingerprint_style: FingerprintStyle::Opaque,
861                    force_from_dep_node: None,
862                    try_load_from_on_disk_cache: None,
863                    name: &"CompileMonoItem",
864                }
865            }
866
867            $(pub(crate) fn $name<'tcx>()-> DepKindStruct<'tcx> {
868                $crate::plumbing::query_callback::<query_impl::$name::QueryType<'tcx>>(
869                    is_anon!([$($modifiers)*]),
870                    is_eval_always!([$($modifiers)*]),
871                )
872            })*
873        }
874
875        pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct<'tcx>] {
876            arena.alloc_from_iter(rustc_middle::make_dep_kind_array!(query_callbacks))
877        }
878
879        pub fn dep_kind_names() -> Vec<&'static str> {
880            rustc_middle::make_dep_kind_name_array!(query_callbacks)
881        }
882    }
883}