Return type op fast path results without the full fulfillment machinery [-0.83% avg]

compiler/rustc_borrowck/src/type_check/relate_tys.rs

@@ -160,7 +160,15 @@ impl<'a, 'b, 'tcx> NllTypeRelating<'a, 'b, 'tcx> {
                 "expected at least one opaque type in `relate_opaques`, got {a} and {b}."
             ),
         };
-        self.register_goals(infcx.handle_opaque_type(a, b, self.span(), self.param_env())?);
+        // Equating the hidden type registers region constraints on the
+        // inference context eagerly, before its non-region goals reach
+        // `InstantiateOpaqueType` below. If the equation fails partway, roll
+        // those back rather than leaving them pending: nothing would scrape
+        // them until an unrelated later type op, and query type op fast
+        // paths rely on no region state being pending between type ops.
+        let goals = infcx
+            .commit_if_ok(|_| infcx.handle_opaque_type(a, b, self.span(), self.param_env()))?;
+        self.register_goals(goals);
         Ok(())
     }
 

compiler/rustc_infer/src/infer/outlives/mod.rs

@@ -119,6 +119,22 @@ impl<'tcx> InferCtxt<'tcx> {
     /// region constraints as normal, but then we take them and
     /// translate them into the form that the NLL solver
     /// understands. See the NLL module for mode details.
+    /// Returns `true` if any region obligations, region assumptions, or
+    /// region constraints are currently registered on this inference
+    /// context. Nothing may be pending here outside a type op:
+    /// `scrape_region_constraints` drains this state when each op
+    /// completes, and type op fast paths debug-assert emptiness before
+    /// skipping that machinery.
+    ///
+    /// N.B.: the checks here must stay in sync with the kinds of pending
+    /// region state drained by `scrape_region_constraints`.
+    pub fn has_pending_region_state(&self) -> bool {
+        let mut inner = self.inner.borrow_mut();
+        !inner.region_obligations.is_empty()
+            || !inner.region_assumptions.is_empty()
+            || !inner.unwrap_region_constraints().data().is_empty()
+    }
+
     pub fn take_and_reset_region_constraints(&self) -> RegionConstraintData<'tcx> {
         assert!(
             self.inner.borrow().region_obligations.is_empty(),

compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs

@@ -62,6 +62,10 @@ impl<F> fmt::Debug for CustomTypeOp<F> {
 
 /// Executes `op` and then scrapes out all the "old style" region
 /// constraints that result, creating query-region-constraints.
+///
+/// N.B.: the kinds of pending region state drained here must stay in sync
+/// with `InferCtxt::has_pending_region_state`, which type op fast paths
+/// debug-assert to be empty when skipping this function.
 pub fn scrape_region_constraints<'tcx, Op, R>(
     infcx: &InferCtxt<'tcx>,
     root_def_id: LocalDefId,

compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs

@@ -110,12 +110,6 @@ pub trait QueryTypeOp<'tcx>: fmt::Debug + Copy + TypeFoldable<TyCtxt<'tcx>> + 't
         ),
         NoSolution,
     > {
-        if !infcx.disable_trait_solver_fast_paths()
-            && let Some(result) = QueryTypeOp::try_fast_path(infcx.tcx, &query_key)
-        {
-            return Ok((result, None, PredicateObligations::new(), Certainty::Proven));
-        }
-
         let mut canonical_var_values = OriginalQueryValues::default();
         let old_param_env = query_key.param_env;
         let canonical_self = infcx.canonicalize_query(query_key, &mut canonical_var_values);
@@ -147,6 +141,41 @@ where
         root_def_id: LocalDefId,
         span: Span,
     ) -> Result<TypeOpOutput<'tcx, Self>, ErrorGuaranteed> {
+        // Fast-path results are final: they never register obligations or
+        // produce region constraints, so we can skip opening a snapshot,
+        // building a fulfillment context, and scraping region constraints.
+        // The overwhelming majority of type ops performed during MIR type
+        // check take this path.
+        //
+        // This is only sound while the inference context has no pending
+        // region state: `scrape_region_constraints` unconditionally drains
+        // region constraints registered on the inference context and
+        // attributes them to the current type op, and some of that state is
+        // registered outside any type op. In particular, when equating
+        // hidden types, `insert_hidden_type` eagerly registers region
+        // constraints on the inference context and its non-region goals may
+        // never reach `InstantiateOpaqueType` if the equation fails. If
+        // anything is pending, fall through to the full machinery, which
+        // scrapes (and asserts) exactly as before.
+        if !infcx.disable_trait_solver_fast_paths()
+            && let Some(output) = QueryTypeOp::try_fast_path(infcx.tcx, &self)
+        {
+            // Skipping the machinery relies on no region state being pending
+            // on the inference context: `scrape_region_constraints` drains
+            // whatever is registered during an op once it completes, and
+            // nothing may register region state outside an op. These
+            // assertions and borrowck's leftover-constraints check enforce
+            // that invariant, like the machinery's own assertions did here
+            // before the fast path was hoisted out of it.
+            debug_assert!(
+                !infcx.has_pending_region_state(),
+                "region state pending at a query type op fast path"
+            );
+            debug_assert!(!infcx.in_snapshot(), "query type op performed inside a snapshot");
+            let output = infcx.resolve_vars_if_possible(output);
+            return Ok(TypeOpOutput { output, constraints: None, error_info: None });
+        }
+
         // In the new trait solver, query type ops are performed locally. This
         // is because query type ops currently use the old canonicalizer, and
         // that doesn't preserve things like opaques which have been registered
@@ -160,14 +189,7 @@ where
                 root_def_id,
                 "query type op",
                 span,
-                |ocx| {
-                    if !infcx.disable_trait_solver_fast_paths()
-                        && let Some(result) = QueryTypeOp::try_fast_path(infcx.tcx, &self)
-                    {
-                        return Ok(result);
-                    }
-                    QueryTypeOp::perform_locally_with_next_solver(ocx, self, span)
-                },
+                |ocx| QueryTypeOp::perform_locally_with_next_solver(ocx, self, span),
             )?
             .0);
         }