fix(vpn): self-recover from expired WG sessions and dedupe log spam

When boringtun returns ConnectionExpired, the daemon previously entered a
catatonic loop — the dead Tunn kept returning the same error on every tick
without ever re-handshaking. The tunnel is now rebuilt immediately on
ConnectionExpired (fix #1), eliminating the deadlock. A per-peer
consecutive-rebuild counter escalates to a node-key rotation signal after
three rebuilds without a successful handshake, covering the case where the
server-side session is also stuck and needs a NodeKey change to force a
PeersChanged push (fix #2). Per-peer warn rate-limiting suppresses the
30+ duplicate log lines that previously appeared within 100ms when
NoCurrentSession errors flooded decapsulate (fix #4). The Running daemon
status now carries last_handshake_fail so operators can see when the most
recent expiry occurred via sunbeam vpn status (fix #3).

Signed-off-by: Sienna Meridian Satterwhite <sienna@r3t.io>

Author: siennathesane

platform/cli/sunbeam-net/src/daemon/endpoint.rs

@@ -26,6 +26,12 @@ struct PeerEndpoint {
     last_call_me_maybe: Option<Instant>,
 }
 
+impl Default for EndpointTracker {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
 impl EndpointTracker {
     pub fn new() -> Self {
         Self {

platform/cli/sunbeam-net/src/daemon/ipc.rs

@@ -367,6 +367,7 @@ mod tests {
             peer_count: 3,
             derp_home: Some(1),
             socks_proxy_port: None,
+            last_handshake_fail: None,
         }));
 
         let server = IpcServer::new(

platform/cli/sunbeam-net/src/daemon/lifecycle.rs

@@ -51,11 +51,14 @@ async fn run_daemon_loop(
     // would see a stale socket file and report "stale socket".
     let _socket_guard = SocketGuard::new(config.control_socket.clone());
 
-    let keys = crate::keys::NodeKeys::load_or_generate(&config.state_dir)?;
     let mut attempt: u32 = 0;
     let max_backoff = Duration::from_secs(60);
 
     loop {
+        // Reload keys every iteration so a key rotation written in the previous
+        // session is picked up before re-registering.
+        let keys = crate::keys::NodeKeys::load_or_generate(&config.state_dir)?;
+
         set_status(&status, DaemonStatus::Connecting);
 
         let session = run_session(&config, &keys, &status, &shutdown);
@@ -71,6 +74,12 @@ async fn run_daemon_loop(
         };
 
         match session_result {
+            Ok(SessionExit::RotatedKey) => {
+                // Keys already written; reload happens at the top of the next
+                // iteration. Skip backoff — we want to re-register quickly.
+                attempt = 0;
+                continue;
+            }
             Ok(SessionExit::Disconnected) | Err(_) => {
                 if let Err(ref e) = session_result {
                     tracing::error!("session ended with error: {e:?}");
@@ -93,6 +102,9 @@ async fn run_daemon_loop(
 
 enum SessionExit {
     Disconnected,
+    /// The stuck-handshake watchdog rotated the node key. The outer loop must
+    /// reload keys before starting the next session.
+    RotatedKey,
 }
 
 /// RAII guard that removes a Unix socket file when dropped. Used by
@@ -592,12 +604,16 @@ async fn run_session(
     //     map_stream_loop into the WG loop so update_peers() is called
     //     on every netmap change, not just at startup.
     let (peer_update_tx, peer_update_rx) = mpsc::channel::<Vec<Node>>(16);
+    // Key-rotation signal: wg_loop → run_session. Capacity 1 — one pending
+    // rotation is enough; extras are silently dropped via try_send.
+    let (rotate_key_tx, mut rotate_key_rx) = mpsc::channel::<()>(1);
     let engine_to_wg_rx = channels.engine_to_wg_rx;
     let wg_to_engine_tx = channels.wg_to_engine_tx;
     let wg_cancel = cancel.clone();
     // Convert disco keys to crypto_box types for NaCl seal/open.
     let disco_secret = crypto_box::SecretKey::from(keys.disco_private.to_bytes());
     let my_disco_pub: [u8; 32] = *keys.disco_public.as_bytes();
+    let wg_status = status.clone();
     let wg_task = tokio::spawn(async move {
         run_wg_loop(
             wg_tunnel,
@@ -612,6 +628,8 @@ async fn run_session(
             disco_secret,
             my_disco_pub,
             stun_bcast_tx.clone(),
+            wg_status,
+            rotate_key_tx,
             wg_cancel,
         )
         .await
@@ -640,11 +658,13 @@ async fn run_session(
             peer_count,
             derp_home,
             socks_proxy_port,
+            last_handshake_fail: None,
         },
     );
 
     // 11. Run concurrent tasks
     let route_whitelist = config.route_whitelist.clone();
+    let state_dir = config.state_dir.clone();
     tokio::select! {
         result = map_stream_loop(&mut map_stream, status, &route_table, &route_whitelist, &peer_update_tx, &derp_cmd_tx) => {
             eprintln!("[session] map_stream_loop exited: {result:?}");
@@ -679,6 +699,17 @@ async fn run_session(
             cancel.cancel();
             result.map(|_| SessionExit::Disconnected)
         }
+        _ = rotate_key_rx.recv() => {
+            tracing::info!("rotating node key after stuck-handshake watchdog escalation");
+            cancel.cancel();
+            match crate::keys::NodeKeys::rotate_node_key(&state_dir) {
+                Ok(()) => Ok(SessionExit::RotatedKey),
+                Err(e) => {
+                    tracing::error!("node key rotation failed: {e:?}");
+                    Ok(SessionExit::Disconnected)
+                }
+            }
+        }
     }
 }
 
@@ -742,6 +773,8 @@ async fn run_wg_loop(
     disco_private: crypto_box::SecretKey,
     my_disco_pub: [u8; 32],
     stun_response_tx: tokio::sync::broadcast::Sender<(std::net::SocketAddr, Vec<u8>)>,
+    status: Arc<RwLock<DaemonStatus>>,
+    rotate_key_tx: mpsc::Sender<()>,
     cancel: tokio_util::sync::CancellationToken,
 ) {
     let mut tick_interval = tokio::time::interval(Duration::from_millis(250));
@@ -852,10 +885,21 @@ async fn run_wg_loop(
                 }
             }
             _ = tick_interval.tick() => {
-                let actions = tunnel.tick();
-                for ta in actions {
+                let tick_result = tunnel.tick();
+                for ta in tick_result.actions {
                     dispatch_encap(ta.action, &derp_out_tx, &udp_out_tx).await;
                 }
+                if tick_result.had_connection_expired {
+                    let ts = std::time::SystemTime::now()
+                        .duration_since(std::time::UNIX_EPOCH)
+                        .unwrap_or_default()
+                        .as_secs();
+                    update_last_handshake_fail(&status, ts);
+                }
+                if tick_result.needs_key_rotation {
+                    tracing::warn!("stuck-handshake watchdog: escalating to node-key rotation");
+                    let _ = rotate_key_tx.try_send(());
+                }
             }
         }
     }
@@ -1315,6 +1359,14 @@ fn update_peer_count(status: &Arc<RwLock<DaemonStatus>>, count: usize) {
     }
 }
 
+fn update_last_handshake_fail(status: &Arc<RwLock<DaemonStatus>>, ts_secs: u64) {
+    if let Ok(mut s) = status.write()
+        && let DaemonStatus::Running { last_handshake_fail, .. } = &mut *s
+    {
+        *last_handshake_fail = Some(ts_secs);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

platform/cli/sunbeam-net/src/daemon/state.rs

@@ -30,6 +30,12 @@ pub enum DaemonStatus {
         /// matching auth token from `{state_dir}/socks5.auth`.
         #[serde(default)]
         socks_proxy_port: Option<u16>,
+        /// Unix timestamp (seconds) of the most recent `ConnectionExpired`
+        /// from any peer's WireGuard session. `None` if no expiry has
+        /// occurred since the daemon last reached Running. Backwards-compatible:
+        /// old daemons omit this field and clients deserialize it as `None`.
+        #[serde(default)]
+        last_handshake_fail: Option<u64>,
     },
     /// Reconnecting after a connection loss.
     Reconnecting {
@@ -170,6 +176,7 @@ mod tests {
             peer_count: 3,
             derp_home: Some(1),
             socks_proxy_port: Some(16580),
+            last_handshake_fail: None,
         };
         assert_eq!(running.to_string(), "running (100.64.0.1), 3 peers");
     }
@@ -181,12 +188,49 @@ mod tests {
             peer_count: 5,
             derp_home: Some(2),
             socks_proxy_port: None,
+            last_handshake_fail: None,
         };
         let json = serde_json::to_string(&status).unwrap();
         let deserialized: DaemonStatus = serde_json::from_str(&json).unwrap();
         assert_eq!(status, deserialized);
     }
 
+    #[test]
+    fn daemon_status_serializes_optional_last_handshake_fail() {
+        // Round-trip with Some timestamp.
+        let now_secs: u64 = 1_745_000_000;
+        let with_fail = DaemonStatus::Running {
+            addresses: vec!["100.64.0.1".parse().unwrap()],
+            peer_count: 1,
+            derp_home: None,
+            socks_proxy_port: None,
+            last_handshake_fail: Some(now_secs),
+        };
+        let json = serde_json::to_string(&with_fail).unwrap();
+        let back: DaemonStatus = serde_json::from_str(&json).unwrap();
+        assert_eq!(with_fail, back);
+
+        // Round-trip with None.
+        let without_fail = DaemonStatus::Running {
+            addresses: vec![],
+            peer_count: 0,
+            derp_home: None,
+            socks_proxy_port: None,
+            last_handshake_fail: None,
+        };
+        let json2 = serde_json::to_string(&without_fail).unwrap();
+        let back2: DaemonStatus = serde_json::from_str(&json2).unwrap();
+        assert_eq!(without_fail, back2);
+
+        // Backwards-compat: JSON missing the field deserializes as None.
+        let legacy_json = r#"{"running":{"addresses":["100.64.0.2"],"peer_count":2,"derp_home":1,"socks_proxy_port":null}}"#;
+        let legacy: DaemonStatus = serde_json::from_str(legacy_json).unwrap();
+        assert!(matches!(
+            legacy,
+            DaemonStatus::Running { last_handshake_fail: None, .. }
+        ));
+    }
+
     #[test]
     fn daemon_handle_default_status() {
         let handle = DaemonHandle::new("/tmp/test.sock".into());

platform/cli/sunbeam-net/src/discovery/registry.rs

@@ -153,10 +153,10 @@ impl RegistryWatcher {
                 match tokio::fs::metadata(&path).await {
                     Ok(meta) => {
                         let mt = meta.modified().ok();
-                        if mt != last_mtime {
-                            if let Some(new_mt) = reload(&path, &registry) {
-                                last_mtime = Some(new_mt);
-                            }
+                        if mt != last_mtime
+                            && let Some(new_mt) = reload(&path, &registry)
+                        {
+                            last_mtime = Some(new_mt);
                         }
                     }
                     Err(e) if e.kind() == std::io::ErrorKind::NotFound => {

platform/cli/sunbeam-net/src/keys.rs

@@ -94,6 +94,25 @@ impl NodeKeys {
         Ok(keys)
     }
 
+    /// Rotate only the `node_private` key in `state_dir/keys.json`, preserving
+    /// the disco and wg keys. Writes atomically: temp file then rename so a
+    /// crash mid-write can't corrupt the existing key.
+    pub fn rotate_node_key(state_dir: &Path) -> crate::Result<()> {
+        let path = state_dir.join(KEYS_FILE);
+        let existing_data = std::fs::read_to_string(&path).ctx("reading keys for rotation")?;
+        let mut persisted: PersistedKeys =
+            serde_json::from_str(&existing_data).map_err(crate::Error::Json)?;
+        let new_node_private = StaticSecret::random_from_rng(OsRng);
+        persisted.node_private = hex::encode(new_node_private.as_bytes());
+        let new_data = serde_json::to_string_pretty(&persisted)?;
+        // Write to a temp file in the same directory then atomically rename.
+        let tmp_path = path.with_extension("json.tmp");
+        std::fs::write(&tmp_path, new_data).ctx("writing rotated key to temp file")?;
+        std::fs::rename(&tmp_path, &path).ctx("renaming rotated key file")?;
+        tracing::info!("node key rotated and persisted to {}", path.display());
+        Ok(())
+    }
+
     /// Tailscale-style node key string: `nodekey:<hex>`.
     pub fn node_key_str(&self) -> String {
         format!("nodekey:{}", hex::encode(self.node_public.as_bytes()))

platform/cli/sunbeam-net/src/noise/handshake.rs

@@ -438,7 +438,7 @@ mod tests {
         // Ciphertext should differ from plaintext.
         assert_ne!(&data[..], b"hello");
 
-        assert_eq!(result.protocol_version, 49);
+        assert_eq!(result.protocol_version, crate::CURRENT_CAP_VER);
 
         server_handle.await.unwrap();
     }