resume one waiter at once

compiler/rustc_query_system/src/query/job.rs

@@ -378,7 +378,7 @@ where
 fn remove_cycle(
     query_map: &QueryMap,
     jobs: &mut Vec<QueryJobId>,
-    wakelist: &mut Vec<Arc<QueryWaiter>>,
+    wakelist: &Mutex<Vec<Arc<QueryWaiter>>>,
 ) -> bool {
     let mut visited = FxHashSet::default();
     let mut stack = Vec::new();
@@ -466,7 +466,7 @@ fn remove_cycle(
         *waiter.cycle.lock() = Some(error);
 
         // Put the waiter on the list of things to resume
-        wakelist.push(waiter);
+        wakelist.lock().push(waiter);
 
         true
     } else {
@@ -478,36 +478,37 @@ fn remove_cycle(
 /// If a query cycle is found it will break the cycle by finding an edge which
 /// uses a query latch and then resuming that waiter.
 /// There may be multiple cycles involved in a deadlock, so this searches
-/// all active queries for cycles before finally resuming all the waiters at once.
+/// all active queries for cycles. But only one waiter will be resumed at once.
 pub fn break_query_cycles(query_map: QueryMap, registry: &rayon_core::Registry) {
-    let mut wakelist = Vec::new();
+    static WAKELIST: Mutex<Vec<Arc<QueryWaiter>>> = Mutex::new(Vec::new());
     let mut jobs: Vec<QueryJobId> = query_map.keys().cloned().collect();
 
     let mut found_cycle = false;
 
     while jobs.len() > 0 {
-        if remove_cycle(&query_map, &mut jobs, &mut wakelist) {
+        if remove_cycle(&query_map, &mut jobs, &WAKELIST) {
             found_cycle = true;
         }
     }
-
+    let mut wake = WAKELIST.lock();
     // Check that a cycle was found. It is possible for a deadlock to occur without
     // a query cycle if a query which can be waited on uses Rayon to do multithreading
     // internally. Such a query (X) may be executing on 2 threads (A and B) and A may
     // wait using Rayon on B. Rayon may then switch to executing another query (Y)
     // which in turn will wait on X causing a deadlock. We have a false dependency from
     // X to Y due to Rayon waiting and a true dependency from Y to X. The algorithm here
     // only considers the true dependency and won't detect a cycle.
-    if !found_cycle {
+    if !found_cycle && wake.is_empty() {
         panic!(
             "deadlock detected as we're unable to find a query cycle to break\n\
             current query map:\n{:#?}",
             query_map
         );
     }
 
-    // FIXME: Ensure this won't cause a deadlock before we return
-    for waiter in wakelist.into_iter() {
+    // Only one waiter is resumed at a time to avoid waking up multiple
+    // waiters at the same time and causing deadlock due to thread grabbing.
+    if let Some(waiter) = wake.pop() {
         waiter.notify(registry);
     }
 }