Add delays to jefe restarts

mkeeter · mkeeter · commit 7594da63fb19 · 2025-08-13T12:50:38.000-04:00
diff --git a/app/cosmo/base.toml b/app/cosmo/base.toml
@@ -21,10 +21,10 @@ size = 256
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
 features = ["dump"]
-stacksize = 1536
+stacksize = 2304
 notifications = ["fault", "timer"]
 extern-regions = ["sram1", "sram2", "sram3", "sram4"]
 
diff --git a/app/demo-stm32f4-discovery/app-f3.toml b/app/demo-stm32f4-discovery/app-f3.toml
@@ -12,9 +12,9 @@ features = ["stm32f3"]
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 8192, ram = 2048}
+max-sizes = {flash = 8192, ram = 4096}
 start = true
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 
 [tasks.rcc_driver]
diff --git a/app/demo-stm32f4-discovery/app.toml b/app/demo-stm32f4-discovery/app.toml
@@ -12,9 +12,9 @@ features = ["stm32f4"]
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 8192, ram = 2048}
+max-sizes = {flash = 8192, ram = 4096}
 start = true
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 
 [tasks.rcc_driver]
diff --git a/app/demo-stm32g0-nucleo/app-g070-mini.toml b/app/demo-stm32g0-nucleo/app-g070-mini.toml
@@ -21,7 +21,7 @@ name = "task-jefe"
 priority = 0
 max-sizes = {flash = 4096, ram = 512}
 start = true
-stacksize = 352
+stacksize = 400
 notifications = ["fault", "timer"]
 
 [tasks.idle]
diff --git a/app/demo-stm32g0-nucleo/app-g070.toml b/app/demo-stm32g0-nucleo/app-g070.toml
@@ -16,7 +16,7 @@ name = "task-jefe"
 priority = 0
 max-sizes = {flash = 4096, ram = 512}
 start = true
-stacksize = 352
+stacksize = 400
 notifications = ["fault", "timer"]
 
 [tasks.sys]
diff --git a/app/demo-stm32h7-nucleo/app-h743.toml b/app/demo-stm32h7-nucleo/app-h743.toml
@@ -13,10 +13,10 @@ features = ["h743", "dump"]
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
 features = ["dump"]
-stacksize = 1536
+stacksize = 2000
 notifications = ["fault", "timer"]
 extern-regions = [ "sram1", "sram2", "sram3", "sram4" ]
 
diff --git a/app/demo-stm32h7-nucleo/app-h753.toml b/app/demo-stm32h7-nucleo/app-h753.toml
@@ -13,10 +13,10 @@ features = ["h753", "dump"]
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
 features = ["dump"]
-stacksize = 1536
+stacksize = 2000
 notifications = ["fault", "timer"]
 extern-regions = [ "sram1", "sram2", "sram3", "sram4" ]
 
diff --git a/app/donglet/app-g031.toml b/app/donglet/app-g031.toml
@@ -15,7 +15,7 @@ name = "task-jefe"
 priority = 0
 max-sizes = {flash = 4096, ram = 512}
 start = true
-stacksize = 256
+stacksize = 384
 notifications = ["fault", "timer"]
 features = ["no-panic", "nano"]
 
diff --git a/app/gemini-bu/app.toml b/app/gemini-bu/app.toml
@@ -12,9 +12,9 @@ requires = {flash = 32768, ram = 8192}
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 8192, ram = 2048}
+max-sizes = {flash = 8192, ram = 4096}
 start = true
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 
 [tasks.jefe.config.allowed-callers]
diff --git a/app/gimlet/base.toml b/app/gimlet/base.toml
@@ -17,10 +17,10 @@ size = 256
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
 features = ["dump"]
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 extern-regions = ["sram1", "sram2", "sram3", "sram4"]
 
diff --git a/app/gimletlet/app-mgmt.toml b/app/gimletlet/app-mgmt.toml
@@ -12,9 +12,9 @@ requires = {flash = 32768, ram = 4096}
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 8192, ram = 2048}
+max-sizes = {flash = 8192, ram = 4096}
 start = true
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 
 [tasks.jefe.config.allowed-callers]
diff --git a/app/gimletlet/base-gimletlet2.toml b/app/gimletlet/base-gimletlet2.toml
@@ -14,9 +14,9 @@ requires = {flash = 32768, ram = 8192}
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 
 [tasks.jefe.config.allowed-callers]
diff --git a/app/grapefruit/base.toml b/app/grapefruit/base.toml
@@ -26,9 +26,9 @@ default = true
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
-stacksize = 1536
+stacksize = 3000
 notifications = ["fault", "timer"]
 features = ["dump"]
 extern-regions = ["sram1", "sram2", "sram3", "sram4"]
diff --git a/app/lpc55xpresso/app.toml b/app/lpc55xpresso/app.toml
@@ -18,10 +18,10 @@ size = 256
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
 features = ["dump"]
-stacksize = 1536
+stacksize = 2000
 notifications = ["fault", "timer"]
 extern-regions = ["sram2"]
 
diff --git a/app/medusa/base.toml b/app/medusa/base.toml
@@ -11,9 +11,9 @@ requires = {flash = 24600, ram = 6256}
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 extern-regions = ["sram1", "sram2", "sram3", "sram4"]
 
diff --git a/app/minibar/app.toml b/app/minibar/app.toml
@@ -13,9 +13,9 @@ requires = {flash = 26656, ram = 6256}
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 
 [tasks.jefe.config.allowed-callers]
diff --git a/app/psc/base.toml b/app/psc/base.toml
@@ -17,10 +17,10 @@ size = 256
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
 features = ["dump"]
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 extern-regions = ["sram1", "sram2", "sram3", "sram4"]
 
diff --git a/app/sidecar/base.toml b/app/sidecar/base.toml
@@ -17,10 +17,10 @@ size = 256
 [tasks.jefe]
 name = "task-jefe"
 priority = 0
-max-sizes = {flash = 16384, ram = 2048}
+max-sizes = {flash = 16384, ram = 4096}
 start = true
 features = ["dump"]
-stacksize = 1536
+stacksize = 2200
 notifications = ["fault", "timer"]
 extern-regions = ["sram1", "sram2", "sram3", "sram4"]
 
diff --git a/task/jefe/src/external.rs b/task/jefe/src/external.rs
@@ -43,7 +43,7 @@
 //! sands...
 //!
 
-use crate::{Disposition, TaskStatus};
+use crate::{Disposition, TaskState, TaskStatus};
 use core::sync::atomic::{AtomicU32, Ordering};
 
 // This trait may not be needed, if compiling for a non-armv6m target.
@@ -101,10 +101,10 @@ static JEFE_EXTERNAL_ERRORS: AtomicU32 = AtomicU32::new(0);
 /// potentially modifying the passed array.  Returns a boolean to indicate if
 /// a valid external request was received.
 ///
-pub(crate) fn check(states: &mut [TaskStatus]) {
+pub(crate) fn check(states: &mut [TaskStatus], now: u64) {
     // This wrapper is responsible for updating operation counters, and allowing
     // the inner function to use Result for convenience.
-    match check_inner(states) {
+    match check_inner(states, now) {
         Ok(true) => {
             JEFE_EXTERNAL_REQUESTS.fetch_add(1, Ordering::SeqCst);
         }
@@ -119,7 +119,7 @@ pub(crate) fn check(states: &mut [TaskStatus]) {
 }
 
 // Implementation factor of `check` that can use Result.
-fn check_inner(states: &mut [TaskStatus]) -> Result<bool, Error> {
+fn check_inner(states: &mut [TaskStatus], now: u64) -> Result<bool, Error> {
     if JEFE_EXTERNAL_KICK.swap(0, Ordering::SeqCst) == 0 {
         return Ok(false);
     }
@@ -169,8 +169,8 @@ fn check_inner(states: &mut [TaskStatus]) -> Result<bool, Error> {
             // not only the disposition change, but may also have to restart the
             // task to clear a held fault.
             state.disposition = Disposition::Restart;
-            if state.holding_fault {
-                state.holding_fault = false;
+            if matches!(state.state, TaskState::HoldFault) {
+                state.state = TaskState::Running { started_at: now };
                 kipc::reinit_task(ndx, true);
             }
         }
diff --git a/task/jefe/src/main.rs b/task/jefe/src/main.rs
@@ -54,6 +54,12 @@ pub enum Disposition {
 // notification, but can otherwise be arbitrary.
 const TIMER_INTERVAL: u32 = 100;
 
+/// Minimum amount of time a task must run before being restarted
+///
+/// If a task runs for *less* than this amount of time before crashing, its
+/// restart is delayed to hit this value.
+const MIN_RUN_TIME: u64 = 50;
+
 #[export_name = "main"]
 fn main() -> ! {
     let mut task_states = [TaskStatus::default(); hubris_num_tasks::NUM_TASKS];
@@ -70,6 +76,7 @@ fn main() -> ! {
         state: 0,
         deadline,
         task_states: &mut task_states,
+        any_tasks_in_timeout: false,
         reset_reason: ResetReason::Unknown,
 
         #[cfg(feature = "dump")]
@@ -89,6 +96,7 @@ struct ServerImpl<'s> {
     state: u32,
     task_states: &'s mut [TaskStatus; NUM_TASKS],
     deadline: u64,
+    any_tasks_in_timeout: bool,
     reset_reason: ResetReason,
 
     /// Base address for a linked list of dump areas
@@ -315,7 +323,25 @@ impl idl::InOrderJefeImpl for ServerImpl<'_> {
 #[derive(Copy, Clone, Debug, Default)]
 struct TaskStatus {
     disposition: Disposition,
-    holding_fault: bool,
+    state: TaskState,
+}
+
+#[derive(Copy, Clone, Debug)]
+enum TaskState {
+    Running {
+        /// Time at which the task started
+        started_at: u64,
+    },
+    HoldFault,
+    Timeout {
+        restart_at: u64,
+    },
+}
+
+impl Default for TaskState {
+    fn default() -> Self {
+        TaskState::Running { started_at: 0 }
+    }
 }
 
 impl idol_runtime::NotificationHandler for ServerImpl<'_> {
@@ -324,16 +350,30 @@ impl idol_runtime::NotificationHandler for ServerImpl<'_> {
     }
 
     fn handle_notification(&mut self, bits: u32) {
+        let now = userlib::sys_get_timer().now;
+
         // Handle any external (debugger) requests.
-        external::check(self.task_states);
+        external::check(self.task_states, now);
 
         if bits & notifications::TIMER_MASK != 0 {
             // If our timer went off, we need to reestablish it
-            if userlib::sys_get_timer().now >= self.deadline {
-                self.deadline = userlib::set_timer_relative(
-                    TIMER_INTERVAL,
-                    notifications::TIMER_MASK,
-                );
+            if now >= self.deadline {
+                self.deadline = now.wrapping_add(u64::from(TIMER_INTERVAL));
+            }
+            // Check for tasks in timeout, updating our timer deadline
+            if core::mem::take(&mut self.any_tasks_in_timeout) {
+                for (index, status) in self.task_states.iter_mut().enumerate() {
+                    if let TaskState::Timeout { restart_at } = &status.state {
+                        if *restart_at >= now {
+                            kipc::reinit_task(index, true);
+                            status.state =
+                                TaskState::Running { started_at: now };
+                        } else {
+                            self.any_tasks_in_timeout = true;
+                            self.deadline = self.deadline.min(*restart_at);
+                        }
+                    }
+                }
             }
         }
 
@@ -359,11 +399,11 @@ impl idol_runtime::NotificationHandler for ServerImpl<'_> {
                 let status =
                     unsafe { self.task_states.get_unchecked_mut(fault_index) };
 
-                // If we're aware that this task is in a fault state, don't
-                // bother making a syscall to enquire.
-                if status.holding_fault {
+                // If we're aware that this task is in a fault state (or waiting
+                // in timeout), don't bother making a syscall to enquire.
+                let TaskState::Running { started_at } = &status.state else {
                     continue;
-                }
+                };
 
                 #[cfg(feature = "dump")]
                 {
@@ -376,15 +416,27 @@ impl idol_runtime::NotificationHandler for ServerImpl<'_> {
                 }
 
                 if status.disposition == Disposition::Restart {
-                    // Stand it back up
-                    kipc::reinit_task(fault_index, true);
+                    let dt = now.wrapping_sub(*started_at).wrapping_add(1);
+                    if let Some(extra_delay) = MIN_RUN_TIME.checked_sub(dt) {
+                        // Put it into timeout to hit our minimum run time
+                        let restart_at = now.wrapping_add(extra_delay);
+                        status.state = TaskState::Timeout { restart_at };
+                        self.deadline = self.deadline.min(restart_at);
+                        self.any_tasks_in_timeout = true;
+                    } else {
+                        // Stand it back up immediately
+                        kipc::reinit_task(fault_index, true);
+                        status.state = TaskState::Running { started_at: now };
+                    }
                 } else {
                     // Mark this one off so we don't revisit it until
                     // requested.
-                    status.holding_fault = true;
+                    status.state = TaskState::HoldFault;
                 }
             }
         }
+
+        userlib::sys_set_timer(Some(self.deadline), notifications::TIMER_MASK);
     }
 }
 
