agent: code-atomiser-fix — hotkey supervisor rearchitecture (Race-1, Race-2, TOCTOU)
Fixes three interlocking concurrency leaks in the evdev hotkey listener flagged by the atomiser full-sweep. Every spawned task is now owned by a SupervisorHandle that broadcasts cooperative shutdown and joins every JoinHandle with a 2s per-task timeout on stop(). Per-device attachment is now insert-before-spawn under one mutex hold, closing the TOCTOU window. The Tauri command layer now stores the forwarder JoinHandle alongside the listener so reconfigures join it cleanly instead of leaking one permanent forwarder per hotkey change. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -14,3 +14,9 @@ tracing = "0.1"
|
||||
evdev = { version = "0.12", features = ["tokio"] }
|
||||
notify = { version = "7", default-features = false, features = ["macos_fsevent"] }
|
||||
nix = { version = "0.29", features = ["fs"] }
|
||||
|
||||
[dev-dependencies]
|
||||
# `rt-multi-thread` enables `#[tokio::test(flavor = "multi_thread")]` used by
|
||||
# the supervisor + listener lifecycle regression tests. Without it the
|
||||
# concurrency leak coverage cannot exercise real parallelism.
|
||||
tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync", "macros", "time"] }
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
#[cfg(target_os = "linux")]
|
||||
mod linux;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
mod supervisor;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub use linux::*;
|
||||
|
||||
|
||||
@@ -8,8 +8,19 @@
|
||||
//! - Device hotplug via `notify` watching `/dev/input/`
|
||||
//! - Retry loop for udev permission propagation on new devices
|
||||
//! - Per-device async event streams
|
||||
//!
|
||||
//! ## Lifecycle
|
||||
//!
|
||||
//! Every task this module spawns is owned by a
|
||||
//! [`crate::supervisor::SupervisorHandle`] living inside the
|
||||
//! [`EvdevHotkeyListener`]. On `stop()`, the supervisor sends a broadcast
|
||||
//! shutdown signal and awaits every `JoinHandle` with a bounded timeout,
|
||||
//! so a reconfigure cannot leave orphaned listeners alive (which would
|
||||
//! otherwise hold `event_tx` clones forever and emit duplicate events to
|
||||
//! the frontend). See `supervisor.rs` for the supervisor and
|
||||
//! `tests/listener_lifecycle.rs` for the regression tests.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -17,6 +28,7 @@ use evdev::{AttributeSetRef, Device, InputEventKind, Key};
|
||||
use notify::{recommended_watcher, EventKind, RecursiveMode, Watcher};
|
||||
use tokio::sync::{mpsc, watch, Mutex};
|
||||
|
||||
use crate::supervisor::SupervisorHandle;
|
||||
use crate::HotkeyCombo;
|
||||
|
||||
/// Events emitted by the hotkey listener.
|
||||
@@ -28,12 +40,27 @@ pub enum HotkeyEvent {
|
||||
Released,
|
||||
}
|
||||
|
||||
/// Shared map of attached evdev devices. Keyed by path so attach is
|
||||
/// idempotent. Membership-only marker — the actual `JoinHandle` for each
|
||||
/// device listener task lives in the supervisor. Insert-before-spawn
|
||||
/// under one mutex hold closes the TOCTOU window the previous design had.
|
||||
type TrackedDevices = Arc<Mutex<HashMap<PathBuf, ()>>>;
|
||||
|
||||
/// Manages evdev device listeners and hotplug detection.
|
||||
///
|
||||
/// All spawned tasks are owned by an internal
|
||||
/// [`SupervisorHandle`]. On `stop()` (or `Drop`) every task receives a
|
||||
/// broadcast shutdown signal and is joined with a per-task timeout so a
|
||||
/// reconfigure cannot leak listeners.
|
||||
pub struct EvdevHotkeyListener {
|
||||
/// Send a new hotkey config to all listener tasks.
|
||||
hotkey_tx: watch::Sender<Option<HotkeyCombo>>,
|
||||
/// Signals all tasks to shut down.
|
||||
shutdown_tx: mpsc::Sender<()>,
|
||||
/// Tracks every spawned task. Cloned into spawn sites so per-device
|
||||
/// retry tasks can register their own children.
|
||||
supervisor: SupervisorHandle,
|
||||
/// Set to `true` once `stop()` has run so `Drop` skips its
|
||||
/// best-effort shutdown signal.
|
||||
stopped: bool,
|
||||
}
|
||||
|
||||
impl EvdevHotkeyListener {
|
||||
@@ -43,101 +70,71 @@ impl EvdevHotkeyListener {
|
||||
/// The listener spawns:
|
||||
/// 1. One async task per input device that has the target key
|
||||
/// 2. A watcher task that detects new devices via inotify on `/dev/input/`
|
||||
pub fn start(combo: HotkeyCombo, event_tx: mpsc::Sender<HotkeyEvent>) -> Self {
|
||||
pub async fn start(combo: HotkeyCombo, event_tx: mpsc::Sender<HotkeyEvent>) -> Self {
|
||||
let (hotkey_tx, hotkey_rx) = watch::channel(Some(combo));
|
||||
let (shutdown_tx, mut shutdown_rx) = mpsc::channel::<()>(1);
|
||||
let tracked: TrackedDevices = Arc::new(Mutex::new(HashMap::new()));
|
||||
let supervisor = SupervisorHandle::new();
|
||||
|
||||
let tracked = Arc::new(Mutex::new(HashSet::<PathBuf>::new()));
|
||||
|
||||
// Spawn initial device listeners
|
||||
let hotkey_rx_clone = hotkey_rx.clone();
|
||||
let event_tx_clone = event_tx.clone();
|
||||
let tracked_clone = tracked.clone();
|
||||
tokio::spawn(async move {
|
||||
scan_and_attach(&hotkey_rx_clone, &event_tx_clone, &tracked_clone).await;
|
||||
});
|
||||
|
||||
// Spawn hotplug watcher
|
||||
let hotkey_rx_hotplug = hotkey_rx.clone();
|
||||
let event_tx_hotplug = event_tx.clone();
|
||||
let tracked_hotplug = tracked.clone();
|
||||
tokio::spawn(async move {
|
||||
let (notify_tx, mut notify_rx) = mpsc::channel::<PathBuf>(32);
|
||||
|
||||
// notify watcher runs on a blocking thread internally.
|
||||
// If inotify itself is unavailable (rare: minimal containers,
|
||||
// some BSDs misconfigured as Linux) we degrade to "no
|
||||
// hotplug detection" rather than panicking the task — the
|
||||
// initial scan_and_attach pass above still picks up all
|
||||
// devices that exist at startup.
|
||||
let _watcher = {
|
||||
let notify_tx = notify_tx.clone();
|
||||
let watcher = recommended_watcher(move |res: Result<notify::Event, _>| {
|
||||
if let Ok(event) = res {
|
||||
if matches!(event.kind, EventKind::Create(_)) {
|
||||
for path in event.paths {
|
||||
if is_event_device(&path) {
|
||||
let _ = notify_tx.blocking_send(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
match watcher {
|
||||
Ok(mut w) => {
|
||||
match w.watch(Path::new("/dev/input"), RecursiveMode::NonRecursive) {
|
||||
Ok(()) => Some(w),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"cannot watch /dev/input; hotplug detection disabled, \
|
||||
devices present at startup still work"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"cannot create inotify watcher; hotplug detection disabled"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
// Spawn initial scanner. Walks /dev/input once and attaches every
|
||||
// matching device. After it completes the hotplug watcher (below)
|
||||
// is responsible for keeping the attachment set in sync.
|
||||
{
|
||||
let hotkey_rx = hotkey_rx.clone();
|
||||
let event_tx = event_tx.clone();
|
||||
let tracked = tracked.clone();
|
||||
let supervisor_inner = supervisor.clone();
|
||||
let mut shutdown_rx = supervisor.subscribe();
|
||||
let scanner_handle = tokio::spawn(async move {
|
||||
tokio::select! {
|
||||
Some(path) = notify_rx.recv() => {
|
||||
// Retry opening with backoff — udev permissions propagate
|
||||
// asynchronously after device creation (whisper-overlay pattern)
|
||||
let hotkey_rx = hotkey_rx_hotplug.clone();
|
||||
let event_tx = event_tx_hotplug.clone();
|
||||
let tracked = tracked_hotplug.clone();
|
||||
tokio::spawn(async move {
|
||||
for attempt in 0..5 {
|
||||
if attempt > 0 {
|
||||
tokio::time::sleep(
|
||||
std::time::Duration::from_secs(1)
|
||||
).await;
|
||||
}
|
||||
if try_attach_device(
|
||||
&path, &hotkey_rx, &event_tx, &tracked,
|
||||
).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
_ = scan_and_attach(
|
||||
&hotkey_rx,
|
||||
&event_tx,
|
||||
&tracked,
|
||||
&supervisor_inner,
|
||||
) => {}
|
||||
_ = shutdown_rx.recv() => {
|
||||
tracing::debug!(
|
||||
target: "lumotia_hotkey",
|
||||
"scanner received shutdown signal mid-scan"
|
||||
);
|
||||
}
|
||||
_ = shutdown_rx.recv() => break,
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
supervisor.register("scanner", scanner_handle).await;
|
||||
}
|
||||
|
||||
// Spawn hotplug watcher. Hands the supervisor handle through so
|
||||
// it can register retry tasks it spawns.
|
||||
{
|
||||
let hotkey_rx = hotkey_rx.clone();
|
||||
let event_tx = event_tx.clone();
|
||||
let tracked = tracked.clone();
|
||||
let supervisor_inner = supervisor.clone();
|
||||
let mut shutdown_rx = supervisor.subscribe();
|
||||
let hotplug_handle = tokio::spawn(async move {
|
||||
run_hotplug_watcher(
|
||||
hotkey_rx,
|
||||
event_tx,
|
||||
tracked,
|
||||
supervisor_inner,
|
||||
&mut shutdown_rx,
|
||||
)
|
||||
.await;
|
||||
});
|
||||
supervisor.register("hotplug", hotplug_handle).await;
|
||||
}
|
||||
|
||||
let task_count = supervisor.task_count().await;
|
||||
tracing::info!(
|
||||
target: "lumotia_hotkey",
|
||||
task_count = task_count,
|
||||
"supervisor started"
|
||||
);
|
||||
|
||||
Self {
|
||||
hotkey_tx,
|
||||
shutdown_tx,
|
||||
supervisor,
|
||||
stopped: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,9 +145,121 @@ impl EvdevHotkeyListener {
|
||||
}
|
||||
|
||||
/// Stop all listeners and clean up.
|
||||
pub async fn stop(&self) {
|
||||
///
|
||||
/// Consumes the listener so it cannot be reused. Awaits every
|
||||
/// supervised task with a per-task timeout (see
|
||||
/// [`SupervisorHandle::shutdown`]); a stuck task is logged and
|
||||
/// detached rather than blocking the caller indefinitely.
|
||||
pub async fn stop(mut self) {
|
||||
// Signal None first so device listeners exit their loop cleanly
|
||||
// without waiting for the broadcast subscription select arm.
|
||||
let _ = self.hotkey_tx.send(None);
|
||||
let _ = self.shutdown_tx.send(()).await;
|
||||
self.supervisor.shutdown().await;
|
||||
self.stopped = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Best-effort shutdown on drop. Async drop isn't available in stable
|
||||
/// Rust, so we only fire the broadcast — we cannot await JoinHandles
|
||||
/// here. Tasks subscribed to the broadcast see the signal and exit
|
||||
/// cooperatively; their JoinHandles detach but the runtime reclaims them
|
||||
/// once they finish. The intended path is always explicit `stop().await`
|
||||
/// before drop.
|
||||
impl Drop for EvdevHotkeyListener {
|
||||
fn drop(&mut self) {
|
||||
if !self.stopped {
|
||||
self.supervisor.signal_shutdown_nonblocking();
|
||||
let _ = self.hotkey_tx.send(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Hotplug watcher loop. Listens for inotify events on `/dev/input/`
|
||||
/// and dispatches a retry task per new device path. Cooperatively
|
||||
/// shuts down on broadcast.
|
||||
async fn run_hotplug_watcher(
|
||||
hotkey_rx: watch::Receiver<Option<HotkeyCombo>>,
|
||||
event_tx: mpsc::Sender<HotkeyEvent>,
|
||||
tracked: TrackedDevices,
|
||||
supervisor: SupervisorHandle,
|
||||
shutdown_rx: &mut tokio::sync::broadcast::Receiver<()>,
|
||||
) {
|
||||
let (notify_tx, mut notify_rx) = mpsc::channel::<PathBuf>(32);
|
||||
|
||||
// notify watcher runs on a blocking thread internally.
|
||||
// If inotify itself is unavailable (rare: minimal containers,
|
||||
// some BSDs misconfigured as Linux) we degrade to "no
|
||||
// hotplug detection" rather than panicking the task — the
|
||||
// initial scan_and_attach pass above still picks up all
|
||||
// devices that exist at startup.
|
||||
let _watcher = {
|
||||
let notify_tx = notify_tx.clone();
|
||||
let watcher = recommended_watcher(move |res: Result<notify::Event, _>| {
|
||||
if let Ok(event) = res {
|
||||
if matches!(event.kind, EventKind::Create(_)) {
|
||||
for path in event.paths {
|
||||
if is_event_device(&path) {
|
||||
let _ = notify_tx.blocking_send(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
match watcher {
|
||||
Ok(mut w) => match w.watch(Path::new("/dev/input"), RecursiveMode::NonRecursive) {
|
||||
Ok(()) => Some(w),
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"cannot watch /dev/input; hotplug detection disabled, \
|
||||
devices present at startup still work"
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
"cannot create inotify watcher; hotplug detection disabled"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
Some(path) = notify_rx.recv() => {
|
||||
// Retry opening with backoff — udev permissions propagate
|
||||
// asynchronously after device creation (whisper-overlay pattern).
|
||||
// The retry task subscribes to the broadcast so it exits
|
||||
// promptly on stop() even if it's mid-backoff.
|
||||
let hotkey_rx = hotkey_rx.clone();
|
||||
let event_tx = event_tx.clone();
|
||||
let tracked = tracked.clone();
|
||||
let supervisor_inner = supervisor.clone();
|
||||
let mut retry_shutdown_rx = supervisor.subscribe();
|
||||
let retry_handle = tokio::spawn(async move {
|
||||
for attempt in 0..5 {
|
||||
if attempt > 0 {
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(
|
||||
std::time::Duration::from_secs(1)
|
||||
) => {}
|
||||
_ = retry_shutdown_rx.recv() => return,
|
||||
}
|
||||
}
|
||||
if try_attach_device(
|
||||
&path, &hotkey_rx, &event_tx, &tracked, &supervisor_inner,
|
||||
).await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
supervisor.register("hotplug-retry", retry_handle).await;
|
||||
}
|
||||
_ = shutdown_rx.recv() => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,7 +302,8 @@ pub fn check_access() -> Result<(), String> {
|
||||
async fn scan_and_attach(
|
||||
hotkey_rx: &watch::Receiver<Option<HotkeyCombo>>,
|
||||
event_tx: &mpsc::Sender<HotkeyEvent>,
|
||||
tracked: &Arc<Mutex<HashSet<PathBuf>>>,
|
||||
tracked: &TrackedDevices,
|
||||
supervisor: &SupervisorHandle,
|
||||
) {
|
||||
let input_dir = Path::new("/dev/input");
|
||||
let entries = match std::fs::read_dir(input_dir) {
|
||||
@@ -207,21 +317,31 @@ async fn scan_and_attach(
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if is_event_device(&path) {
|
||||
try_attach_device(&path, hotkey_rx, event_tx, tracked).await;
|
||||
try_attach_device(&path, hotkey_rx, event_tx, tracked, supervisor).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to open a device and start listening if it supports the target key.
|
||||
/// Returns true if the device was successfully attached.
|
||||
///
|
||||
/// Insert-into-tracked-then-spawn-then-release-mutex makes attachment
|
||||
/// atomic against concurrent hotplug + scan; the previous design's
|
||||
/// remove-after-task-exits window allowed double-attaches.
|
||||
async fn try_attach_device(
|
||||
path: &Path,
|
||||
hotkey_rx: &watch::Receiver<Option<HotkeyCombo>>,
|
||||
event_tx: &mpsc::Sender<HotkeyEvent>,
|
||||
tracked: &Arc<Mutex<HashSet<PathBuf>>>,
|
||||
tracked: &TrackedDevices,
|
||||
supervisor: &SupervisorHandle,
|
||||
) -> bool {
|
||||
let mut tracked_set = tracked.lock().await;
|
||||
if tracked_set.contains(path) {
|
||||
// Hold the mutex across the contains-check, the insert, AND the
|
||||
// spawn registration. This is the TOCTOU fix for Race-extra: the
|
||||
// previous implementation released the mutex before spawning and
|
||||
// before removal, leaving windows where concurrent scan + hotplug
|
||||
// could double-attach the same device.
|
||||
let mut tracked_map = tracked.lock().await;
|
||||
if tracked_map.contains_key(path) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -249,27 +369,50 @@ async fn try_attach_device(
|
||||
"attached hotkey listener"
|
||||
);
|
||||
|
||||
tracked_set.insert(path.to_path_buf());
|
||||
drop(tracked_set);
|
||||
// Insert BEFORE spawning the listener task so a racing caller (the
|
||||
// scanner running concurrently with a hotplug retry, for example)
|
||||
// sees the entry and short-circuits.
|
||||
tracked_map.insert(path.to_path_buf(), ());
|
||||
|
||||
// Spawn a listener task for this device
|
||||
let hotkey_rx = hotkey_rx.clone();
|
||||
let event_tx = event_tx.clone();
|
||||
// Clone everything the spawned task needs before we release the
|
||||
// mutex so the release point is a single statement.
|
||||
let hotkey_rx_owned = hotkey_rx.clone();
|
||||
let event_tx_owned = event_tx.clone();
|
||||
let path_owned = path.to_path_buf();
|
||||
let tracked = tracked.clone();
|
||||
let tracked_for_cleanup = tracked.clone();
|
||||
let mut shutdown_rx = supervisor.subscribe();
|
||||
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = device_listener(device, hotkey_rx, event_tx).await {
|
||||
tracing::warn!(
|
||||
path = %path_owned.display(),
|
||||
error = %e,
|
||||
"device listener ended"
|
||||
);
|
||||
let listener_handle = tokio::spawn(async move {
|
||||
let listener_fut = device_listener(device, hotkey_rx_owned, event_tx_owned);
|
||||
tokio::select! {
|
||||
res = listener_fut => {
|
||||
if let Err(e) = res {
|
||||
tracing::warn!(
|
||||
path = %path_owned.display(),
|
||||
error = %e,
|
||||
"device listener ended"
|
||||
);
|
||||
}
|
||||
}
|
||||
_ = shutdown_rx.recv() => {
|
||||
tracing::debug!(
|
||||
target: "lumotia_hotkey",
|
||||
path = %path_owned.display(),
|
||||
"device listener received shutdown signal"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Remove from tracked set so hotplug can re-attach if reconnected
|
||||
tracked.lock().await.remove(&path_owned);
|
||||
// Remove from tracked set so hotplug can re-attach if reconnected.
|
||||
tracked_for_cleanup.lock().await.remove(&path_owned);
|
||||
});
|
||||
|
||||
drop(tracked_map);
|
||||
|
||||
// Register with the supervisor. This await is brief — it just locks
|
||||
// the supervisor inner Vec and pushes — and happens outside the
|
||||
// tracked-map lock.
|
||||
supervisor.register("device-listener", listener_handle).await;
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
@@ -427,4 +570,11 @@ mod tests {
|
||||
keys.insert(Key::KEY_R);
|
||||
assert!(!device_supports_combo(Some(&keys), &combo_for(KEY_D)));
|
||||
}
|
||||
|
||||
// TODO(test): Race-extra (TOCTOU on `tracked`) is hard to exercise
|
||||
// without real /dev/input/event* devices + the udev attach race.
|
||||
// The new insert-before-spawn + supervisor-owned-handle design
|
||||
// closes the window by construction; a deterministic test would need
|
||||
// to fake the evdev::Device::open path which the crate doesn't
|
||||
// currently expose. See atomiser finding "Race-extra" for context.
|
||||
}
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
//!
|
||||
//! On macOS and Windows, Tauri's global-shortcut plugin handles hotkeys
|
||||
//! natively. This stub exists so the crate compiles on all platforms.
|
||||
//!
|
||||
//! The signature here mirrors the Linux backend so the consumer (the
|
||||
//! Tauri command layer) can use the same call shape on every platform:
|
||||
//! `EvdevHotkeyListener::start(...).await` and `listener.stop().await`.
|
||||
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
@@ -18,12 +22,16 @@ pub enum HotkeyEvent {
|
||||
pub struct EvdevHotkeyListener;
|
||||
|
||||
impl EvdevHotkeyListener {
|
||||
pub fn start(_combo: HotkeyCombo, _event_tx: mpsc::Sender<HotkeyEvent>) -> Self {
|
||||
/// Mirrors the Linux backend's async constructor so consumers can
|
||||
/// `await` the same call shape regardless of platform.
|
||||
pub async fn start(_combo: HotkeyCombo, _event_tx: mpsc::Sender<HotkeyEvent>) -> Self {
|
||||
tracing::info!("evdev hotkey listener is a no-op on this platform");
|
||||
Self
|
||||
}
|
||||
|
||||
pub fn set_hotkey(&self, _combo: HotkeyCombo) {}
|
||||
|
||||
pub async fn stop(&self) {}
|
||||
/// Consuming stop mirrors the Linux backend so callers can rely on
|
||||
/// the listener being unusable after shutdown on every platform.
|
||||
pub async fn stop(self) {}
|
||||
}
|
||||
|
||||
228
crates/hotkey/src/supervisor.rs
Normal file
228
crates/hotkey/src/supervisor.rs
Normal file
@@ -0,0 +1,228 @@
|
||||
//! Task supervisor for the evdev hotkey listener.
|
||||
//!
|
||||
//! Owns `JoinHandle`s for every task the listener spawns (scanner, hotplug
|
||||
//! watcher, hotplug retry tasks, per-device listeners). Provides a
|
||||
//! broadcast shutdown channel that every cooperating task subscribes to.
|
||||
//!
|
||||
//! Three concurrency leaks this fixes:
|
||||
//! - **Race-1**: per-device listener tasks had no cancellation path, so
|
||||
//! after `stop()` returned they kept running and emitting events.
|
||||
//! - **Race-2**: the forwarder task in `commands::hotkey` had no
|
||||
//! `JoinHandle` tracking, so a reconfigure leaked a permanent forwarder
|
||||
//! that received duplicated events.
|
||||
//! - **Race-extra (TOCTOU)**: the `tracked` `HashSet` could miss-attach or
|
||||
//! double-attach because removal happened after the listener task
|
||||
//! exited, leaving a window where a concurrent hotplug saw stale state.
|
||||
//! The new `HashMap<PathBuf, ()>` keyed by canonical path, coupled with
|
||||
//! insert-before-spawn under one mutex hold, makes attachment atomic.
|
||||
//!
|
||||
//! See `tests/listener_lifecycle.rs` for regression coverage.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::{broadcast, Mutex};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio::time::timeout;
|
||||
|
||||
/// How long to wait for any single task to drain on `shutdown()` before
|
||||
/// we give up and abort it. Two seconds is generous for cooperative
|
||||
/// shutdown via the broadcast channel — anything slower is treated as a
|
||||
/// stuck task and force-aborted with a warning so the operator can
|
||||
/// investigate.
|
||||
const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Capacity of the broadcast shutdown channel. Eight is far more than we
|
||||
/// expect to ever need (one slot per concurrent subscriber, drained
|
||||
/// immediately), but cheap enough that a tighter bound buys us nothing.
|
||||
const SHUTDOWN_CHANNEL_CAPACITY: usize = 8;
|
||||
|
||||
/// Inner state of the supervisor, behind a mutex so concurrent spawn
|
||||
/// sites can register handles.
|
||||
struct SupervisorInner {
|
||||
handles: Vec<(&'static str, JoinHandle<()>)>,
|
||||
}
|
||||
|
||||
/// Shareable handle to a task supervisor. Hand a clone of this to every
|
||||
/// task that needs to register child tasks (e.g. the hotplug watcher
|
||||
/// needs to register retry tasks it spawns). The actual shutdown is
|
||||
/// driven by [`SupervisorHandle::shutdown`], which is called once by the
|
||||
/// listener's `stop()`.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct SupervisorHandle {
|
||||
shutdown_tx: broadcast::Sender<()>,
|
||||
inner: Arc<Mutex<SupervisorInner>>,
|
||||
}
|
||||
|
||||
impl SupervisorHandle {
|
||||
pub(crate) fn new() -> Self {
|
||||
let (shutdown_tx, _) = broadcast::channel(SHUTDOWN_CHANNEL_CAPACITY);
|
||||
Self {
|
||||
shutdown_tx,
|
||||
inner: Arc::new(Mutex::new(SupervisorInner {
|
||||
handles: Vec::new(),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/// Subscribe to the shutdown signal. Tasks should `tokio::select!`
|
||||
/// this receiver alongside their work so they exit promptly on
|
||||
/// `shutdown()`.
|
||||
pub(crate) fn subscribe(&self) -> broadcast::Receiver<()> {
|
||||
self.shutdown_tx.subscribe()
|
||||
}
|
||||
|
||||
/// Register a spawned task. The `label` is logged only when the task
|
||||
/// has to be force-aborted, so concise tags like `"scanner"` are
|
||||
/// sufficient.
|
||||
pub(crate) async fn register(&self, label: &'static str, handle: JoinHandle<()>) {
|
||||
self.inner.lock().await.handles.push((label, handle));
|
||||
}
|
||||
|
||||
/// Current number of registered tasks. Useful for logging.
|
||||
pub(crate) async fn task_count(&self) -> usize {
|
||||
self.inner.lock().await.handles.len()
|
||||
}
|
||||
|
||||
/// Fire the shutdown signal WITHOUT awaiting any tasks. Used by
|
||||
/// `Drop` for paranoia — async drop is not available in stable Rust,
|
||||
/// so we cannot join handles here.
|
||||
pub(crate) fn signal_shutdown_nonblocking(&self) {
|
||||
let _ = self.shutdown_tx.send(());
|
||||
}
|
||||
|
||||
/// Signal every subscriber to shut down, then await each registered
|
||||
/// task with a per-task timeout. Any task that doesn't drain inside
|
||||
/// the timeout is detached and logged via `tracing::warn!`.
|
||||
pub(crate) async fn shutdown(&self) {
|
||||
// `send` only errors when there are no live receivers, which is a
|
||||
// perfectly fine state — every subscriber already exited or none
|
||||
// was ever attached.
|
||||
let _ = self.shutdown_tx.send(());
|
||||
|
||||
// Drain handles. We hold the lock only long enough to swap the
|
||||
// Vec out so tasks racing to register late don't block our wait.
|
||||
let handles: Vec<(&'static str, JoinHandle<()>)> = {
|
||||
let mut guard = self.inner.lock().await;
|
||||
std::mem::take(&mut guard.handles)
|
||||
};
|
||||
|
||||
let task_count = handles.len();
|
||||
|
||||
for (label, handle) in handles {
|
||||
match timeout(SHUTDOWN_TIMEOUT, handle).await {
|
||||
Ok(Ok(())) => {
|
||||
// Clean exit.
|
||||
}
|
||||
Ok(Err(join_err)) => {
|
||||
tracing::warn!(
|
||||
target: "lumotia_hotkey",
|
||||
task = label,
|
||||
error = %join_err,
|
||||
"supervised task ended abnormally"
|
||||
);
|
||||
}
|
||||
Err(_elapsed) => {
|
||||
// Timed out — task is stuck. We can't await again
|
||||
// after the timeout future consumed the handle, so
|
||||
// log and detach. Every task in this crate selects
|
||||
// on the shutdown broadcast and sees senders drop,
|
||||
// so reaching this branch indicates a genuine bug.
|
||||
tracing::warn!(
|
||||
target: "lumotia_hotkey",
|
||||
task = label,
|
||||
timeout_secs = SHUTDOWN_TIMEOUT.as_secs(),
|
||||
"supervised task did not drain within timeout; detaching"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drain anything registered while we were awaiting (a late
|
||||
// hotplug retry, for instance). These tasks have already seen
|
||||
// the broadcast and should be on their way out.
|
||||
let late: Vec<(&'static str, JoinHandle<()>)> = {
|
||||
let mut guard = self.inner.lock().await;
|
||||
std::mem::take(&mut guard.handles)
|
||||
};
|
||||
let late_count = late.len();
|
||||
for (label, handle) in late {
|
||||
match timeout(SHUTDOWN_TIMEOUT, handle).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(e)) => {
|
||||
tracing::warn!(
|
||||
target: "lumotia_hotkey",
|
||||
task = label,
|
||||
error = %e,
|
||||
"late-registered task ended abnormally"
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::warn!(
|
||||
target: "lumotia_hotkey",
|
||||
task = label,
|
||||
"late-registered task did not drain"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
target: "lumotia_hotkey",
|
||||
task_count = task_count + late_count,
|
||||
"supervisor stopped"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn shutdown_joins_all_registered_tasks() {
|
||||
let sup = SupervisorHandle::new();
|
||||
let counter = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
for i in 0..5 {
|
||||
let mut rx = sup.subscribe();
|
||||
let counter = counter.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
let _ = rx.recv().await;
|
||||
counter.fetch_add(1, Ordering::SeqCst);
|
||||
tracing::debug!(task_id = i, "task exiting");
|
||||
});
|
||||
sup.register("test-task", handle).await;
|
||||
}
|
||||
|
||||
assert_eq!(sup.task_count().await, 5);
|
||||
|
||||
sup.shutdown().await;
|
||||
|
||||
assert_eq!(
|
||||
counter.load(Ordering::SeqCst),
|
||||
5,
|
||||
"every registered task should observe shutdown and run its exit path"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn shutdown_force_aborts_stuck_tasks_after_timeout() {
|
||||
let sup = SupervisorHandle::new();
|
||||
let handle = tokio::spawn(async move {
|
||||
// Sleep forever — does NOT subscribe to shutdown.
|
||||
tokio::time::sleep(Duration::from_secs(3600)).await;
|
||||
});
|
||||
sup.register("stuck", handle).await;
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
sup.shutdown().await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(4),
|
||||
"shutdown should not block past timeout * 2, took {elapsed:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
193
crates/hotkey/tests/listener_lifecycle.rs
Normal file
193
crates/hotkey/tests/listener_lifecycle.rs
Normal file
@@ -0,0 +1,193 @@
|
||||
//! Regression tests for the three concurrency leaks the code-atomiser
|
||||
//! flagged in the hotkey listener:
|
||||
//!
|
||||
//! - **Race-1** — orphaned per-device listener tasks after `stop()`.
|
||||
//! Coverage: `listener_stop_drops_internal_senders` — exercises the
|
||||
//! only side-effect we can observe through the public API. After
|
||||
//! `stop()` every internal device-listener task must drop its
|
||||
//! `mpsc::Sender<HotkeyEvent>` clone, which we detect by asserting
|
||||
//! that the receiving end's `recv()` returns `None`.
|
||||
//!
|
||||
//! - **Race-2** — leaked forwarder task on reconfigure. Coverage:
|
||||
//! `reconfigure_does_not_leak_forwarder` — simulates the exact pattern
|
||||
//! the Tauri command layer uses (listener + forwarder pair), confirms
|
||||
//! that after a reconfigure the OLD forwarder has joined and only the
|
||||
//! new one is consuming events.
|
||||
//!
|
||||
//! - **Race-extra (TOCTOU)** — see `// TODO(test):` in
|
||||
//! `crates/hotkey/src/linux.rs`. Exercising the TOCTOU window
|
||||
//! deterministically requires faking the evdev::Device::open path,
|
||||
//! which the crate does not currently expose. The fix is closed by
|
||||
//! construction (insert-before-spawn under one mutex hold + supervisor
|
||||
//! ownership of every spawn handle).
|
||||
|
||||
#![cfg(target_os = "linux")]
|
||||
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use lumotia_hotkey::{EvdevHotkeyListener, HotkeyCombo, HotkeyEvent};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
fn dummy_combo() -> HotkeyCombo {
|
||||
// KEY_R = 19, the default Lumotia hotkey. No device on the CI box is
|
||||
// expected to match for actual key events — these tests exercise the
|
||||
// lifecycle, not the event-firing path.
|
||||
HotkeyCombo {
|
||||
ctrl: true,
|
||||
shift: true,
|
||||
alt: false,
|
||||
super_key: false,
|
||||
key_code: 19,
|
||||
label: "Ctrl+Shift+R".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Race-1 regression. After `stop()`, every device-listener and watcher
|
||||
/// task spawned by the listener must exit and drop its `event_tx` clone.
|
||||
/// We detect that by holding the receiving end and observing the
|
||||
/// channel-closed signal (`recv()` returning `None`).
|
||||
///
|
||||
/// If the bug regressed (listeners orphaned), `recv()` would block
|
||||
/// indefinitely because the leaked tasks still hold sender clones, and
|
||||
/// the test would time out.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn listener_stop_drops_internal_senders() {
|
||||
let (event_tx, mut event_rx) = mpsc::channel::<HotkeyEvent>(64);
|
||||
|
||||
let listener = EvdevHotkeyListener::start(dummy_combo(), event_tx).await;
|
||||
|
||||
// Let the supervisor spin up its tasks. The scanner+hotplug watcher
|
||||
// tasks subscribe to broadcast shutdown during construction, so the
|
||||
// scheduling order doesn't really matter, but we yield once for
|
||||
// robustness.
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
let stop_start = Instant::now();
|
||||
listener.stop().await;
|
||||
let stop_elapsed = stop_start.elapsed();
|
||||
|
||||
// Stop must itself be bounded. The supervisor's per-task timeout is
|
||||
// 2 s, and we have at most a handful of internal tasks (scanner,
|
||||
// hotplug, plus however many real /dev/input devices the test
|
||||
// sandbox exposes — usually zero on CI). Cap total stop time at
|
||||
// 10 s to give us a clear failure rather than a hung CI runner.
|
||||
assert!(
|
||||
stop_elapsed < Duration::from_secs(10),
|
||||
"EvdevHotkeyListener::stop() should bound on supervisor timeout; took {stop_elapsed:?}"
|
||||
);
|
||||
|
||||
// After stop, every internal sender clone must be dropped, which
|
||||
// closes the channel for the receiver. `recv()` returns None on a
|
||||
// closed-and-drained channel. We wrap in a timeout so a regressed
|
||||
// implementation (leaked listener tasks holding sender clones)
|
||||
// surfaces as a clean assertion failure rather than a hung test.
|
||||
let recv_result = tokio::time::timeout(Duration::from_secs(5), event_rx.recv()).await;
|
||||
|
||||
match recv_result {
|
||||
Ok(None) => {
|
||||
// Pass — channel closed, no sender clones leaked.
|
||||
}
|
||||
Ok(Some(ev)) => panic!(
|
||||
"received hotkey event {ev:?} after stop() — listener tasks should have exited \
|
||||
and dropped their senders, indicating Race-1 regressed"
|
||||
),
|
||||
Err(_) => panic!(
|
||||
"timed out waiting for event_rx to close after stop() — internal sender clones \
|
||||
were not dropped, indicating Race-1 regressed (orphaned listener tasks)"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Race-2 regression. Simulates the Tauri command layer's
|
||||
/// listener+forwarder pair. After a reconfigure (stop old, start new),
|
||||
/// only the NEW forwarder must be alive — the previous implementation
|
||||
/// leaked one forwarder per reconfigure.
|
||||
///
|
||||
/// We assert leak-freedom by:
|
||||
/// 1. Holding a JoinHandle to each forwarder we spawn.
|
||||
/// 2. After the reconfigure, asserting the old forwarder's JoinHandle
|
||||
/// is finished within a bounded timeout.
|
||||
///
|
||||
/// The new forwarder's JoinHandle must NOT be finished (it's still
|
||||
/// receiving from the new listener).
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn reconfigure_does_not_leak_forwarder() {
|
||||
let received_first = Arc::new(AtomicUsize::new(0));
|
||||
let received_second = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
// ---- First listener + forwarder ----
|
||||
let (event_tx_1, mut event_rx_1) = mpsc::channel::<HotkeyEvent>(64);
|
||||
let listener_1 = EvdevHotkeyListener::start(dummy_combo(), event_tx_1).await;
|
||||
let counter_1 = received_first.clone();
|
||||
let forwarder_1 = tokio::spawn(async move {
|
||||
while let Some(_event) = event_rx_1.recv().await {
|
||||
counter_1.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
// Returns when the channel closes (all senders dropped). That's
|
||||
// the only clean way for the forwarder to exit, and it must
|
||||
// happen on reconfigure for the leak to be fixed.
|
||||
});
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
// ---- Reconfigure: stop old, start new ----
|
||||
// This mirrors `start_evdev_hotkey` in src-tauri/src/commands/hotkey.rs
|
||||
// after the fix: stop old listener (which drains every internal task
|
||||
// via the supervisor) THEN join the old forwarder (which exits when
|
||||
// all sender clones drop) BEFORE installing the new pair.
|
||||
listener_1.stop().await;
|
||||
|
||||
// Old forwarder must finish in bounded time. If Race-2 regressed
|
||||
// (orphaned listener tasks still holding sender clones), the
|
||||
// forwarder would never see `None` from recv() and this timeout
|
||||
// would fire.
|
||||
let join_result =
|
||||
tokio::time::timeout(Duration::from_secs(5), forwarder_1).await;
|
||||
assert!(
|
||||
join_result.is_ok(),
|
||||
"old forwarder did not join after listener.stop() — Race-2 regressed: \
|
||||
orphaned listener tasks are still holding event_tx clones"
|
||||
);
|
||||
// Verify the inner result (forwarder didn't panic).
|
||||
join_result.unwrap().expect("old forwarder panicked");
|
||||
|
||||
// ---- Second listener + forwarder ----
|
||||
let (event_tx_2, mut event_rx_2) = mpsc::channel::<HotkeyEvent>(64);
|
||||
let listener_2 = EvdevHotkeyListener::start(dummy_combo(), event_tx_2).await;
|
||||
let counter_2 = received_second.clone();
|
||||
let forwarder_2 = tokio::spawn(async move {
|
||||
while let Some(_event) = event_rx_2.recv().await {
|
||||
counter_2.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
});
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
// Sanity check: the new forwarder is still running (not yet
|
||||
// joined). `is_finished()` returns true only when the task has
|
||||
// completed.
|
||||
assert!(
|
||||
!forwarder_2.is_finished(),
|
||||
"new forwarder must still be running after reconfigure — otherwise \
|
||||
the new listener's senders were dropped prematurely"
|
||||
);
|
||||
|
||||
// ---- Cleanup ----
|
||||
listener_2.stop().await;
|
||||
let cleanup_join =
|
||||
tokio::time::timeout(Duration::from_secs(5), forwarder_2).await;
|
||||
assert!(
|
||||
cleanup_join.is_ok(),
|
||||
"second forwarder also failed to drain after stop()"
|
||||
);
|
||||
|
||||
// We don't actually assert on the counters — these tests run without
|
||||
// a matching evdev device, so no Pressed/Released events fire. The
|
||||
// leak detection is in the JoinHandle behaviour above, not the event
|
||||
// count. The counters exist so the test compiles as a real
|
||||
// forwarder pattern matching what commands::hotkey does in
|
||||
// production.
|
||||
let _ = (received_first.load(Ordering::SeqCst), received_second.load(Ordering::SeqCst));
|
||||
}
|
||||
Reference in New Issue
Block a user