Merge 'tokio-1.51.2' into 'tokio-1.52.x' (#8114)

Author: Darksonn

tokio/CHANGELOG.md

@@ -50,6 +50,14 @@
 [#8035]: https://github.com/tokio-rs/tokio/pull/8035
 [#8040]: https://github.com/tokio-rs/tokio/pull/8040
 
+# 1.51.2 (May 4th, 2026)
+
+This release reverts the LIFO slot stealing change introduced in 1.51.0
+([#7431]), due to [its performance impact][#8065]. ([#8100])
+
+[#8065]: https://github.com/tokio-rs/tokio/pull/8065
+[#8100]: https://github.com/tokio-rs/tokio/pull/8100
+
 # 1.51.1 (April 8th, 2026)
 
 ### Fixed

tokio/src/runtime/builder.rs

@@ -1298,21 +1298,22 @@ impl Builder {
         /// scheduled task being polled first.
         ///
         /// To implement this heuristic, each worker thread has a slot which
-        /// holds the task that should be polled next. In earlier versions of
-        /// Tokio, this slot could not be stolen by other worker threads, which
-        /// can result in lower total throughput when tasks tend to have longer
-        /// poll times.
+        /// holds the task that should be polled next. However, this slot cannot
+        /// be stolen by other worker threads, which can result in lower total
+        /// throughput when tasks tend to have longer poll times.
         ///
         /// This configuration option will disable this heuristic resulting in
-        /// all scheduled tasks being pushed into the worker-local queue. This
-        /// was intended as a workaround for the LIFO slot not being stealable.
-        /// As of Tokio 1.51, tasks can be stolen from the LIFO slot. In a
-        /// future version, this option may be deprecated.
+        /// all scheduled tasks being pushed into the worker-local queue, which
+        /// is stealable.
+        ///
+        /// Consider trying this option when the task "scheduled" time is high
+        /// but the runtime is underutilized. Use [tokio-rs/tokio-metrics] to
+        /// collect this data.
         ///
         /// # Unstable
         ///
-        /// This configuration option was considered a workaround for the LIFO
-        /// slot not being stealable. Since this is no longer the case, we will
+        /// This configuration option is considered a workaround for the LIFO
+        /// slot not being stealable. When the slot becomes stealable, we will
         /// revisit whether or not this option is necessary. See
         /// issue [tokio-rs/tokio#4941].
         ///

tokio/src/runtime/config.rs

@@ -34,13 +34,11 @@ pub(crate) struct Config {
 
     /// The multi-threaded scheduler includes a per-worker LIFO slot used to
     /// store the last scheduled task. This can improve certain usage patterns,
-    /// especially message passing between tasks.
+    /// especially message passing between tasks. However, this LIFO slot is not
+    /// currently stealable.
     ///
-    /// In Tokio versions before 1.51, tasks in the LIFO slot could not be
-    /// stolen, which could cause issues in applications with long poll times.
-    /// As a stop-gap, this unstable option lets users disable the LIFO task.
-    /// Now that the LIFO slot is stealable, we may remove this option in a
-    /// future version.
+    /// Eventually, the LIFO slot **will** become stealable, however as a
+    /// stop-gap, this unstable option lets users disable the LIFO task.
     pub(crate) disable_lifo_slot: bool,
 
     /// Random number generator seed to configure runtimes to act in a

tokio/src/runtime/mod.rs

@@ -367,8 +367,8 @@
 //! three times in a row, it is temporarily disabled until the worker thread has
 //! scheduled a task that didn't come from the lifo slot. The lifo slot can be
 //! disabled using the [`disable_lifo_slot`] setting. The lifo slot is separate
-//! from the local queue, and is stolen from by other worker threads only when
-//! a worker's local queue has been drained.
+//! from the local queue, so other worker threads cannot steal the task in the
+//! lifo slot.
 //!
 //! When a task is woken from a thread that is not a worker thread, then the
 //! task is placed in the global queue.

tokio/src/runtime/scheduler/multi_thread/queue.rs

@@ -52,13 +52,6 @@ pub(crate) struct Inner<T: 'static> {
     /// Only updated by producer thread but read by many threads.
     tail: AtomicUnsignedShort,
 
-    /// When a task is scheduled from a worker, it is stored in this slot. The
-    /// worker will check this slot for a task **before** checking the run
-    /// queue. This effectively results in the **last** scheduled task to be run
-    /// next (LIFO). This is an optimization for improving locality which
-    /// benefits message passing patterns and helps to reduce latency.
-    lifo: task::AtomicNotified<T>,
-
     /// Elements
     buffer: Box<[UnsafeCell<MaybeUninit<task::Notified<T>>>; LOCAL_QUEUE_CAPACITY]>,
 }
@@ -99,7 +92,6 @@ pub(crate) fn local<T: 'static>() -> (Steal<T>, Local<T>) {
     let inner = Arc::new(Inner {
         head: AtomicUnsignedLong::new(0),
         tail: AtomicUnsignedShort::new(0),
-        lifo: task::AtomicNotified::empty(),
         buffer: make_fixed_size(buffer.into_boxed_slice()),
     });
 
@@ -116,10 +108,9 @@ impl<T> Local<T> {
     /// Returns the number of entries in the queue
     pub(crate) fn len(&self) -> usize {
         let (_, head) = unpack(self.inner.head.load(Acquire));
-        let lifo = self.inner.lifo.is_some() as usize;
         // safety: this is the **only** thread that updates this cell.
         let tail = unsafe { self.inner.tail.unsync_load() };
-        len(head, tail) + lifo
+        len(head, tail)
     }
 
     /// How many tasks can be pushed into the queue
@@ -410,28 +401,14 @@ impl<T> Local<T> {
 
         Some(self.inner.buffer[idx].with(|ptr| unsafe { ptr::read(ptr).assume_init() }))
     }
-
-    /// Pushes a task to the LIFO slot, returning the task previously in the
-    /// LIFO slot (if there was one).
-    pub(crate) fn push_lifo(&self, task: task::Notified<T>) -> Option<task::Notified<T>> {
-        self.inner.lifo.swap(Some(task))
-    }
-
-    /// Pops the task currently held in the LIFO slot, if there is one;
-    /// otherwise, returns `None`.
-    pub(crate) fn pop_lifo(&self) -> Option<task::Notified<T>> {
-        // LIFO-suction!
-        self.inner.lifo.take()
-    }
 }
 
 impl<T> Steal<T> {
     /// Returns the number of entries in the queue
     pub(crate) fn len(&self) -> usize {
         let (_, head) = unpack(self.0.head.load(Acquire));
         let tail = self.0.tail.load(Acquire);
-        let lifo = self.0.lifo.is_some() as usize;
-        len(head, tail) + lifo
+        len(head, tail)
     }
 
     /// Return true if the queue is empty,
@@ -466,14 +443,8 @@ impl<T> Steal<T> {
         let mut n = self.steal_into2(dst, dst_tail);
 
         if n == 0 {
-            // If no tasks were stolen, let's see if there's one in the LIFO
-            // slot.
-            let lifo = self.0.lifo.take();
-            if lifo.is_some() {
-                dst_stats.incr_steal_count(1);
-                dst_stats.incr_steal_operations();
-            }
-            return lifo;
+            // No tasks were stolen
+            return None;
         }
 
         dst_stats.incr_steal_count(n as u16);
@@ -605,7 +576,6 @@ impl<T> Drop for Local<T> {
     fn drop(&mut self) {
         if !std::thread::panicking() {
             assert!(self.pop().is_none(), "queue not empty");
-            assert!(self.pop_lifo().is_none(), "LIFO slot not empty");
         }
     }
 }

tokio/src/runtime/scheduler/multi_thread/worker.rs

@@ -112,6 +112,13 @@ struct Core {
     /// Used to schedule bookkeeping tasks every so often.
     tick: u32,
 
+    /// When a task is scheduled from a worker, it is stored in this slot. The
+    /// worker will check this slot for a task **before** checking the run
+    /// queue. This effectively results in the **last** scheduled task to be run
+    /// next (LIFO). This is an optimization for improving locality which
+    /// benefits message passing patterns and helps to reduce latency.
+    lifo_slot: Option<Notified>,
+
     /// When `true`, locally scheduled tasks go to the LIFO slot. When `false`,
     /// they go to the back of the `run_queue`.
     lifo_enabled: bool,
@@ -282,6 +289,7 @@ pub(super) fn create(
 
         cores.push(Box::new(Core {
             tick: 0,
+            lifo_slot: None,
             lifo_enabled: !config.disable_lifo_slot,
             run_queue,
             #[cfg(all(tokio_unstable, feature = "time"))]
@@ -443,7 +451,7 @@ where
         // If we heavily call `spawn_blocking`, there might be no available thread to
         // run this core. Except for the task in the lifo_slot, all tasks can be
         // stolen, so we move the task out of the lifo_slot to the run_queue.
-        if let Some(task) = core.run_queue.pop_lifo() {
+        if let Some(task) = core.lifo_slot.take() {
             core.run_queue
                 .push_back_or_overflow(task, &*cx.worker.handle, &mut core.stats);
         }
@@ -696,7 +704,7 @@ impl Context {
                 };
 
                 // Check for a task in the LIFO slot
-                let task = match core.run_queue.pop_lifo() {
+                let task = match core.lifo_slot.take() {
                     Some(task) => task,
                     None => {
                         self.reset_lifo_enabled(&mut core);
@@ -1122,7 +1130,7 @@ impl Core {
     }
 
     fn next_local_task(&mut self) -> Option<Notified> {
-        self.run_queue.pop_lifo().or_else(|| self.run_queue.pop())
+        self.lifo_slot.take().or_else(|| self.run_queue.pop())
     }
 
     /// Function responsible for stealing tasks from another worker
@@ -1178,7 +1186,7 @@ impl Core {
     }
 
     fn has_tasks(&self) -> bool {
-        self.run_queue.has_tasks()
+        self.lifo_slot.is_some() || self.run_queue.has_tasks()
     }
 
     fn should_notify_others(&self) -> bool {
@@ -1187,7 +1195,7 @@ impl Core {
         if self.is_searching {
             return false;
         }
-        self.run_queue.len() > 1
+        self.lifo_slot.is_some() as usize + self.run_queue.len() > 1
     }
 
     /// Prepares the worker state for parking.
@@ -1349,23 +1357,29 @@ impl Handle {
         // task must always be pushed to the back of the queue, enabling other
         // tasks to be executed. If **not** a yield, then there is more
         // flexibility and the task may go to the front of the queue.
-        if is_yield || !core.lifo_enabled {
+        let should_notify = if is_yield || !core.lifo_enabled {
             core.run_queue
                 .push_back_or_overflow(task, self, &mut core.stats);
+            true
         } else {
             // Push to the LIFO slot
-            if let Some(prev) = core.run_queue.push_lifo(task) {
-                // There was a previous task in the LIFO slot which needs
-                // to be pushed to the back of the run queue.
+            let prev = core.lifo_slot.take();
+            let ret = prev.is_some();
+
+            if let Some(prev) = prev {
                 core.run_queue
                     .push_back_or_overflow(prev, self, &mut core.stats);
             }
+
+            core.lifo_slot = Some(task);
+
+            ret
         };
 
         // Only notify if not currently parked. If `park` is `None`, then the
         // scheduling is from a resource driver. As notifications often come in
         // batches, the notification is delayed until the park is complete.
-        if core.park.is_some() {
+        if should_notify && core.park.is_some() {
             self.notify_parked_local();
         }
     }

tokio/src/runtime/task/atomic_notified.rs

@@ -209,11 +209,6 @@ pub(crate) use self::raw::RawTask;
 mod state;
 use self::state::State;
 
-#[cfg(feature = "rt-multi-thread")]
-mod atomic_notified;
-#[cfg(feature = "rt-multi-thread")]
-pub(crate) use self::atomic_notified::AtomicNotified;
-
 mod waker;
 
 pub(crate) use self::spawn_location::SpawnLocation;