1use std::fmt::Debug;
11use std::iter;
12
13use rustc_index::{Idx, IndexVec};
14use rustc_middle::arena::ArenaAllocatable;
15use rustc_middle::mir::ConstraintCategory;
16use rustc_middle::ty::{self, BoundVar, GenericArg, GenericArgKind, Ty, TyCtxt, TypeFoldable};
17use rustc_middle::{bug, span_bug};
18use tracing::{debug, instrument};
19
20use crate::infer::canonical::instantiate::{CanonicalExt, instantiate_value};
21use crate::infer::canonical::{
22 Canonical, CanonicalQueryResponse, CanonicalVarValues, Certainty, OriginalQueryValues,
23 QueryOutlivesConstraint, QueryRegionConstraints, QueryResponse,
24};
25use crate::infer::region_constraints::{Constraint, RegionConstraintData};
26use crate::infer::{DefineOpaqueTypes, InferCtxt, InferOk, InferResult};
27use crate::traits::query::NoSolution;
28use crate::traits::{
29 Obligation, ObligationCause, PredicateObligation, PredicateObligations, ScrubbedTraitError,
30 TraitEngine,
31};
32
33impl<'tcx> InferCtxt<'tcx> {
34 #[instrument(skip(self, inference_vars, answer, fulfill_cx), level = "trace")]
54 pub fn make_canonicalized_query_response<T>(
55 &self,
56 inference_vars: CanonicalVarValues<'tcx>,
57 answer: T,
58 fulfill_cx: &mut dyn TraitEngine<'tcx, ScrubbedTraitError<'tcx>>,
59 ) -> Result<CanonicalQueryResponse<'tcx, T>, NoSolution>
60 where
61 T: Debug + TypeFoldable<TyCtxt<'tcx>>,
62 Canonical<'tcx, QueryResponse<'tcx, T>>: ArenaAllocatable<'tcx>,
63 {
64 let query_response = self.make_query_response(inference_vars, answer, fulfill_cx)?;
65 debug!("query_response = {:#?}", query_response);
66 let canonical_result = self.canonicalize_response(query_response);
67 debug!("canonical_result = {:#?}", canonical_result);
68
69 Ok(self.tcx.arena.alloc(canonical_result))
70 }
71
72 pub fn make_query_response_ignoring_pending_obligations<T>(
82 &self,
83 inference_vars: CanonicalVarValues<'tcx>,
84 answer: T,
85 ) -> Canonical<'tcx, QueryResponse<'tcx, T>>
86 where
87 T: Debug + TypeFoldable<TyCtxt<'tcx>>,
88 {
89 self.canonicalize_response(QueryResponse {
90 var_values: inference_vars,
91 region_constraints: QueryRegionConstraints::default(),
92 certainty: Certainty::Proven, opaque_types: vec![],
94 value: answer,
95 })
96 }
97
98 #[instrument(skip(self, fulfill_cx), level = "debug")]
101 fn make_query_response<T>(
102 &self,
103 inference_vars: CanonicalVarValues<'tcx>,
104 answer: T,
105 fulfill_cx: &mut dyn TraitEngine<'tcx, ScrubbedTraitError<'tcx>>,
106 ) -> Result<QueryResponse<'tcx, T>, NoSolution>
107 where
108 T: Debug + TypeFoldable<TyCtxt<'tcx>>,
109 {
110 let tcx = self.tcx;
111
112 let errors = fulfill_cx.select_all_or_error(self);
114
115 if errors.iter().any(|e| e.is_true_error()) {
117 return Err(NoSolution);
118 }
119
120 let region_obligations = self.take_registered_region_obligations();
121 debug!(?region_obligations);
122 let region_constraints = self.with_region_constraints(|region_constraints| {
123 make_query_region_constraints(
124 tcx,
125 region_obligations
126 .iter()
127 .map(|r_o| (r_o.sup_type, r_o.sub_region, r_o.origin.to_constraint_category())),
128 region_constraints,
129 )
130 });
131 debug!(?region_constraints);
132
133 let certainty = if errors.is_empty() { Certainty::Proven } else { Certainty::Ambiguous };
134
135 let opaque_types = self
136 .inner
137 .borrow_mut()
138 .opaque_type_storage
139 .take_opaque_types()
140 .map(|(k, v)| (k, v.ty))
141 .collect();
142
143 Ok(QueryResponse {
144 var_values: inference_vars,
145 region_constraints,
146 certainty,
147 value: answer,
148 opaque_types,
149 })
150 }
151
152 pub fn instantiate_query_response_and_region_obligations<R>(
163 &self,
164 cause: &ObligationCause<'tcx>,
165 param_env: ty::ParamEnv<'tcx>,
166 original_values: &OriginalQueryValues<'tcx>,
167 query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
168 ) -> InferResult<'tcx, R>
169 where
170 R: Debug + TypeFoldable<TyCtxt<'tcx>>,
171 {
172 let InferOk { value: result_args, mut obligations } =
173 self.query_response_instantiation(cause, param_env, original_values, query_response)?;
174
175 obligations.extend(self.query_outlives_constraints_into_obligations(
176 cause,
177 param_env,
178 &query_response.value.region_constraints.outlives,
179 &result_args,
180 ));
181
182 let user_result: R =
183 query_response.instantiate_projected(self.tcx, &result_args, |q_r| q_r.value.clone());
184
185 Ok(InferOk { value: user_result, obligations })
186 }
187
188 pub fn instantiate_nll_query_response_and_region_obligations<R>(
224 &self,
225 cause: &ObligationCause<'tcx>,
226 param_env: ty::ParamEnv<'tcx>,
227 original_values: &OriginalQueryValues<'tcx>,
228 query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
229 output_query_region_constraints: &mut QueryRegionConstraints<'tcx>,
230 ) -> InferResult<'tcx, R>
231 where
232 R: Debug + TypeFoldable<TyCtxt<'tcx>>,
233 {
234 let InferOk { value: result_args, mut obligations } = self
235 .query_response_instantiation_guess(
236 cause,
237 param_env,
238 original_values,
239 query_response,
240 )?;
241
242 let constraint_category = cause.to_constraint_category();
247
248 for (index, original_value) in original_values.var_values.iter().enumerate() {
249 let result_value = query_response.instantiate_projected(self.tcx, &result_args, |v| {
251 v.var_values[BoundVar::new(index)]
252 });
253 match (original_value.unpack(), result_value.unpack()) {
254 (GenericArgKind::Lifetime(re1), GenericArgKind::Lifetime(re2))
255 if re1.is_erased() && re2.is_erased() =>
256 {
257 }
259
260 (GenericArgKind::Lifetime(v_o), GenericArgKind::Lifetime(v_r)) => {
261 if v_o != v_r {
263 output_query_region_constraints
264 .outlives
265 .push((ty::OutlivesPredicate(v_o.into(), v_r), constraint_category));
266 output_query_region_constraints
267 .outlives
268 .push((ty::OutlivesPredicate(v_r.into(), v_o), constraint_category));
269 }
270 }
271
272 (GenericArgKind::Type(v1), GenericArgKind::Type(v2)) => {
273 obligations.extend(
274 self.at(&cause, param_env)
275 .eq(DefineOpaqueTypes::Yes, v1, v2)?
276 .into_obligations(),
277 );
278 }
279
280 (GenericArgKind::Const(v1), GenericArgKind::Const(v2)) => {
281 obligations.extend(
282 self.at(&cause, param_env)
283 .eq(DefineOpaqueTypes::Yes, v1, v2)?
284 .into_obligations(),
285 );
286 }
287
288 _ => {
289 bug!("kind mismatch, cannot unify {:?} and {:?}", original_value, result_value);
290 }
291 }
292 }
293
294 output_query_region_constraints.outlives.extend(
296 query_response.value.region_constraints.outlives.iter().filter_map(|&r_c| {
297 let r_c = instantiate_value(self.tcx, &result_args, r_c);
298
299 let ty::OutlivesPredicate(k1, r2) = r_c.0;
301 if k1 != r2.into() { Some(r_c) } else { None }
302 }),
303 );
304
305 let user_result: R =
306 query_response.instantiate_projected(self.tcx, &result_args, |q_r| q_r.value.clone());
307
308 Ok(InferOk { value: user_result, obligations })
309 }
310
311 fn query_response_instantiation<R>(
322 &self,
323 cause: &ObligationCause<'tcx>,
324 param_env: ty::ParamEnv<'tcx>,
325 original_values: &OriginalQueryValues<'tcx>,
326 query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
327 ) -> InferResult<'tcx, CanonicalVarValues<'tcx>>
328 where
329 R: Debug + TypeFoldable<TyCtxt<'tcx>>,
330 {
331 debug!(
332 "query_response_instantiation(original_values={:#?}, query_response={:#?})",
333 original_values, query_response,
334 );
335
336 let mut value = self.query_response_instantiation_guess(
337 cause,
338 param_env,
339 original_values,
340 query_response,
341 )?;
342
343 value.obligations.extend(
344 self.unify_query_response_instantiation_guess(
345 cause,
346 param_env,
347 original_values,
348 &value.value,
349 query_response,
350 )?
351 .into_obligations(),
352 );
353
354 Ok(value)
355 }
356
357 #[instrument(level = "debug", skip(self, param_env))]
367 fn query_response_instantiation_guess<R>(
368 &self,
369 cause: &ObligationCause<'tcx>,
370 param_env: ty::ParamEnv<'tcx>,
371 original_values: &OriginalQueryValues<'tcx>,
372 query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
373 ) -> InferResult<'tcx, CanonicalVarValues<'tcx>>
374 where
375 R: Debug + TypeFoldable<TyCtxt<'tcx>>,
376 {
377 let mut universe_map = original_values.universe_map.clone();
381 let num_universes_in_query = original_values.universe_map.len();
382 let num_universes_in_response = query_response.max_universe.as_usize() + 1;
383 for _ in num_universes_in_query..num_universes_in_response {
384 universe_map.push(self.create_next_universe());
385 }
386 assert!(!universe_map.is_empty()); assert_eq!(universe_map[ty::UniverseIndex::ROOT.as_usize()], ty::UniverseIndex::ROOT);
388
389 let result_values = &query_response.value.var_values;
394 assert_eq!(original_values.var_values.len(), result_values.len());
395
396 let mut opt_values: IndexVec<BoundVar, Option<GenericArg<'tcx>>> =
404 IndexVec::from_elem_n(None, query_response.variables.len());
405
406 for (original_value, result_value) in iter::zip(&original_values.var_values, result_values)
409 {
410 match result_value.unpack() {
411 GenericArgKind::Type(result_value) => {
412 if let ty::Bound(debruijn, b) = *result_value.kind() {
414 assert_eq!(debruijn, ty::INNERMOST);
418 opt_values[b.var] = Some(*original_value);
419 }
420 }
421 GenericArgKind::Lifetime(result_value) => {
422 if let ty::ReBound(debruijn, br) = result_value.kind() {
424 assert_eq!(debruijn, ty::INNERMOST);
428 opt_values[br.var] = Some(*original_value);
429 }
430 }
431 GenericArgKind::Const(result_value) => {
432 if let ty::ConstKind::Bound(debruijn, b) = result_value.kind() {
433 assert_eq!(debruijn, ty::INNERMOST);
437 opt_values[b] = Some(*original_value);
438 }
439 }
440 }
441 }
442
443 let result_args = CanonicalVarValues {
447 var_values: self.tcx.mk_args_from_iter(
448 query_response.variables.iter().enumerate().map(|(index, info)| {
449 if info.universe() != ty::UniverseIndex::ROOT {
450 self.instantiate_canonical_var(cause.span, info, |u| {
453 universe_map[u.as_usize()]
454 })
455 } else if info.is_existential() {
456 match opt_values[BoundVar::new(index)] {
457 Some(k) => k,
458 None => self.instantiate_canonical_var(cause.span, info, |u| {
459 universe_map[u.as_usize()]
460 }),
461 }
462 } else {
463 opt_values[BoundVar::new(index)].expect(
466 "expected placeholder to be unified with itself during response",
467 )
468 }
469 }),
470 ),
471 };
472
473 let mut obligations = PredicateObligations::new();
474
475 for &(a, b) in &query_response.value.opaque_types {
477 let a = instantiate_value(self.tcx, &result_args, a);
478 let b = instantiate_value(self.tcx, &result_args, b);
479 debug!(?a, ?b, "constrain opaque type");
480 obligations.extend(
485 self.at(cause, param_env)
486 .eq(
487 DefineOpaqueTypes::Yes,
488 Ty::new_opaque(self.tcx, a.def_id.to_def_id(), a.args),
489 b,
490 )?
491 .obligations,
492 );
493 }
494
495 Ok(InferOk { value: result_args, obligations })
496 }
497
498 fn unify_query_response_instantiation_guess<R>(
505 &self,
506 cause: &ObligationCause<'tcx>,
507 param_env: ty::ParamEnv<'tcx>,
508 original_values: &OriginalQueryValues<'tcx>,
509 result_args: &CanonicalVarValues<'tcx>,
510 query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>,
511 ) -> InferResult<'tcx, ()>
512 where
513 R: Debug + TypeFoldable<TyCtxt<'tcx>>,
514 {
515 let instantiated_query_response = |index: BoundVar| -> GenericArg<'tcx> {
520 query_response.instantiate_projected(self.tcx, result_args, |v| v.var_values[index])
521 };
522
523 self.unify_canonical_vars(cause, param_env, original_values, instantiated_query_response)
526 }
527
528 fn query_outlives_constraints_into_obligations(
531 &self,
532 cause: &ObligationCause<'tcx>,
533 param_env: ty::ParamEnv<'tcx>,
534 uninstantiated_region_constraints: &[QueryOutlivesConstraint<'tcx>],
535 result_args: &CanonicalVarValues<'tcx>,
536 ) -> impl Iterator<Item = PredicateObligation<'tcx>> {
537 uninstantiated_region_constraints.iter().map(move |&constraint| {
538 let predicate = instantiate_value(self.tcx, result_args, constraint);
539 self.query_outlives_constraint_to_obligation(predicate, cause.clone(), param_env)
540 })
541 }
542
543 pub fn query_outlives_constraint_to_obligation(
544 &self,
545 (predicate, _): QueryOutlivesConstraint<'tcx>,
546 cause: ObligationCause<'tcx>,
547 param_env: ty::ParamEnv<'tcx>,
548 ) -> Obligation<'tcx, ty::Predicate<'tcx>> {
549 let ty::OutlivesPredicate(k1, r2) = predicate;
550
551 let atom = match k1.unpack() {
552 GenericArgKind::Lifetime(r1) => ty::PredicateKind::Clause(
553 ty::ClauseKind::RegionOutlives(ty::OutlivesPredicate(r1, r2)),
554 ),
555 GenericArgKind::Type(t1) => ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(
556 ty::OutlivesPredicate(t1, r2),
557 )),
558 GenericArgKind::Const(..) => {
559 span_bug!(cause.span, "unexpected const outlives {:?}", predicate);
562 }
563 };
564 let predicate = ty::Binder::dummy(atom);
565
566 Obligation::new(self.tcx, cause, param_env, predicate)
567 }
568
569 fn unify_canonical_vars(
572 &self,
573 cause: &ObligationCause<'tcx>,
574 param_env: ty::ParamEnv<'tcx>,
575 variables1: &OriginalQueryValues<'tcx>,
576 variables2: impl Fn(BoundVar) -> GenericArg<'tcx>,
577 ) -> InferResult<'tcx, ()> {
578 let mut obligations = PredicateObligations::new();
579 for (index, value1) in variables1.var_values.iter().enumerate() {
580 let value2 = variables2(BoundVar::new(index));
581
582 match (value1.unpack(), value2.unpack()) {
583 (GenericArgKind::Type(v1), GenericArgKind::Type(v2)) => {
584 obligations.extend(
585 self.at(cause, param_env)
586 .eq(DefineOpaqueTypes::Yes, v1, v2)?
587 .into_obligations(),
588 );
589 }
590 (GenericArgKind::Lifetime(re1), GenericArgKind::Lifetime(re2))
591 if re1.is_erased() && re2.is_erased() =>
592 {
593 }
595 (GenericArgKind::Lifetime(v1), GenericArgKind::Lifetime(v2)) => {
596 obligations.extend(
597 self.at(cause, param_env)
598 .eq(DefineOpaqueTypes::Yes, v1, v2)?
599 .into_obligations(),
600 );
601 }
602 (GenericArgKind::Const(v1), GenericArgKind::Const(v2)) => {
603 let ok = self.at(cause, param_env).eq(DefineOpaqueTypes::Yes, v1, v2)?;
604 obligations.extend(ok.into_obligations());
605 }
606 _ => {
607 bug!("kind mismatch, cannot unify {:?} and {:?}", value1, value2,);
608 }
609 }
610 }
611 Ok(InferOk { value: (), obligations })
612 }
613}
614
615pub fn make_query_region_constraints<'tcx>(
618 tcx: TyCtxt<'tcx>,
619 outlives_obligations: impl Iterator<Item = (Ty<'tcx>, ty::Region<'tcx>, ConstraintCategory<'tcx>)>,
620 region_constraints: &RegionConstraintData<'tcx>,
621) -> QueryRegionConstraints<'tcx> {
622 let RegionConstraintData { constraints, verifys } = region_constraints;
623
624 assert!(verifys.is_empty());
625
626 debug!(?constraints);
627
628 let outlives: Vec<_> = constraints
629 .iter()
630 .map(|(k, origin)| {
631 let constraint = match *k {
632 Constraint::VarSubVar(v1, v2) => ty::OutlivesPredicate(
635 ty::Region::new_var(tcx, v2).into(),
636 ty::Region::new_var(tcx, v1),
637 ),
638 Constraint::VarSubReg(v1, r2) => {
639 ty::OutlivesPredicate(r2.into(), ty::Region::new_var(tcx, v1))
640 }
641 Constraint::RegSubVar(r1, v2) => {
642 ty::OutlivesPredicate(ty::Region::new_var(tcx, v2).into(), r1)
643 }
644 Constraint::RegSubReg(r1, r2) => ty::OutlivesPredicate(r2.into(), r1),
645 };
646 (constraint, origin.to_constraint_category())
647 })
648 .chain(outlives_obligations.map(|(ty, r, constraint_category)| {
649 (ty::OutlivesPredicate(ty.into(), r), constraint_category)
650 }))
651 .collect();
652
653 QueryRegionConstraints { outlives }
654}