From 5fa60a5d25739142cd45aed22df3b10739066854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 16 Feb 2023 19:29:11 +0100 Subject: [PATCH 1/4] Reduce calls to `current_query_job` --- compiler/rustc_query_system/src/query/plumbing.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 5499165930db0..0338a35230614 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -188,12 +188,12 @@ where #[cfg(not(parallel_compiler))] let mut state_lock = state.active.lock(); let lock = &mut *state_lock; + let current_job_id = qcx.current_query_job(); match lock.entry(key) { Entry::Vacant(entry) => { let id = qcx.next_job_id(); - let job = qcx.current_query_job(); - let job = QueryJob::new(id, span, job); + let job = QueryJob::new(id, span, current_job_id); let key = *entry.key(); entry.insert(QueryResult::Started(job)); @@ -212,7 +212,7 @@ where // so we just return the error. return TryGetJob::Cycle(id.find_cycle_in_stack( qcx.try_collect_active_jobs().unwrap(), - &qcx.current_query_job(), + ¤t_job_id, span, )); } @@ -230,7 +230,7 @@ where // With parallel queries we might just have to wait on some other // thread. - let result = latch.wait_on(qcx.current_query_job(), span); + let result = latch.wait_on(current_job_id, span); match result { Ok(()) => TryGetJob::JobCompleted(query_blocked_prof_timer), From a049550c457dc78e81ffa69d03e1619f4ff689a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 16 Feb 2023 19:34:31 +0100 Subject: [PATCH 2/4] Move `ensure_sufficient_stack` to `try_execute_query` callers --- compiler/rustc_query_impl/src/plumbing.rs | 4 +--- .../rustc_query_system/src/query/plumbing.rs | 23 +++++++------------ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index e2884f2026eb0..a8592bd70862c 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -124,9 +124,7 @@ impl QueryContext for QueryCtxt<'_> { }; // Use the `ImplicitCtxt` while we execute the query. - tls::enter_context(&new_icx, || { - rustc_data_structures::stack::ensure_sufficient_stack(compute) - }) + tls::enter_context(&new_icx, compute) }) } diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 0338a35230614..148d7f09f5111 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -15,6 +15,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::TimingGuard; #[cfg(parallel_compiler)] use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::sync::Lock; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError}; use rustc_session::Session; @@ -348,8 +349,6 @@ where fn try_execute_query( qcx: Qcx, - state: &QueryState, - cache: &Q::Cache, span: Span, key: Q::Key, dep_node: Option>, @@ -358,9 +357,11 @@ where Q: QueryConfig, Qcx: QueryContext, { + let state = Q::query_state(qcx); match JobOwner::<'_, Q::Key, Qcx::DepKind>::try_start(&qcx, state, span, key) { TryGetJob::NotYetStarted(job) => { let (result, dep_node_index) = execute_job::(qcx, key, dep_node, job.id); + let cache = Q::query_cache(qcx); if Q::FEEDABLE { // We should not compute queries that also got a value via feeding. // This can't happen, as query feeding adds the very dependencies to the fed query @@ -381,7 +382,7 @@ where } #[cfg(parallel_compiler)] TryGetJob::JobCompleted(query_blocked_prof_timer) => { - let Some((v, index)) = cache.lookup(&key) else { + let Some((v, index)) = Q::query_cache(qcx).lookup(&key) else { panic!("value must be in cache after waiting") }; @@ -739,14 +740,8 @@ where None }; - let (result, dep_node_index) = try_execute_query::( - qcx, - Q::query_state(qcx), - Q::query_cache(qcx), - span, - key, - dep_node, - ); + let (result, dep_node_index) = + ensure_sufficient_stack(|| try_execute_query::(qcx, span, key, dep_node)); if let Some(dep_node_index) = dep_node_index { qcx.dep_context().dep_graph().read_index(dep_node_index) } @@ -762,14 +757,12 @@ where { // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. - let cache = Q::query_cache(qcx); - if let Some((_, index)) = cache.lookup(&key) { + if let Some((_, index)) = Q::query_cache(qcx).lookup(&key) { qcx.dep_context().profiler().query_cache_hit(index.into()); return; } - let state = Q::query_state(qcx); debug_assert!(!Q::ANON); - try_execute_query::(qcx, state, cache, DUMMY_SP, key, Some(dep_node)); + ensure_sufficient_stack(|| try_execute_query::(qcx, DUMMY_SP, key, Some(dep_node))); } From ab5d3fbe7db281ca48a7b88f90c81bf5e795564c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Thu, 23 Feb 2023 01:14:48 +0100 Subject: [PATCH 3/4] Add inlining attributes for query system functions --- compiler/rustc_middle/src/ty/context.rs | 1 + compiler/rustc_query_system/src/dep_graph/graph.rs | 5 +++++ compiler/rustc_query_system/src/query/plumbing.rs | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 796daa7646f9c..2c0b3b690cfcc 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1011,6 +1011,7 @@ impl<'tcx> TyCtxt<'tcx> { /// Note that this is *untracked* and should only be used within the query /// system if the result is otherwise tracked through queries + #[inline] pub fn cstore_untracked(self) -> MappedReadGuard<'tcx, CrateStoreDyn> { ReadGuard::map(self.untracked.cstore.read(), |c| &**c) } diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 35fa932e0f1f0..59e0c35974559 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -279,6 +279,7 @@ impl DepGraph { /// `arg` parameter. /// /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/incremental-compilation.html + #[inline(always)] pub fn with_task, A: Debug, R>( &self, key: DepNode, @@ -298,6 +299,7 @@ impl DepGraph { } } + #[inline(always)] fn with_task_impl, A: Debug, R>( &self, key: DepNode, @@ -598,6 +600,7 @@ impl DepGraph { self.data.is_some() && self.dep_node_index_of_opt(dep_node).is_some() } + #[inline] pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Option { self.data.as_ref().unwrap().previous.fingerprint_of(dep_node) } @@ -1127,6 +1130,7 @@ impl CurrentDepGraph { /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it. /// Assumes that this is a node that has no equivalent in the previous dep-graph. + #[inline(always)] fn intern_new_node( &self, profiler: &SelfProfilerRef, @@ -1365,6 +1369,7 @@ impl DepNodeColorMap { } } + #[inline] fn insert(&self, index: SerializedDepNodeIndex, color: DepNodeColor) { self.values[index].store( match color { diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 148d7f09f5111..fa877ceaf2375 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -394,6 +394,7 @@ where } } +#[inline(always)] fn execute_job( qcx: Qcx, key: Q::Key, @@ -479,6 +480,7 @@ where (result, dep_node_index) } +#[inline(always)] fn try_load_from_disk_and_cache_in_memory( qcx: Qcx, key: &Q::Key, @@ -569,6 +571,7 @@ where Some((result, dep_node_index)) } +#[inline] #[instrument(skip(tcx, result, hash_result), level = "debug")] pub(crate) fn incremental_verify_ich( tcx: Tcx, @@ -723,6 +726,7 @@ pub enum QueryMode { Ensure, } +#[inline(always)] pub fn get_query(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option where D: DepKind, From dd73080cc037678410ae7e016298a6bbafaae3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sat, 25 Feb 2023 05:16:38 +0100 Subject: [PATCH 4/4] Don't inline try_execute_query --- compiler/rustc_query_system/src/query/plumbing.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index fa877ceaf2375..586bd38fbb80b 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -347,6 +347,7 @@ where } } +#[inline(never)] fn try_execute_query( qcx: Qcx, span: Span,