add peft scheduling

better deadlock report by pim simulator

backend-simulators/pim/pim-simulator/src/bin/pim-simulator/main.rs

@@ -67,7 +67,7 @@ fn main() -> Result<()> {
         .lock()
         .unwrap()
         .init(executor.cpu().num_core(), args.output.clone());
-    executor.execute();
+    executor.execute()?;
     dump_memory(executor, &args)?;
     Ok(())
 }

backend-simulators/pim/pim-simulator/src/lib/pimcore.rs

@@ -1,5 +1,6 @@
 #![allow(unused)]
 
+use anyhow::{Result, bail};
 use std::{
     collections::{HashMap, HashSet},
     time::{Duration, SystemTime},
@@ -87,6 +88,11 @@ pub struct Executable<'a> {
     send_recv: SendRecv,
 }
 
+struct DeadlockInfo {
+    cycle: String,
+    states: String,
+}
+
 fn print_status(core_instructions: &[CoreInstructions]) {
     let mut tot_instructions = 0;
     let mut progress = 0;
@@ -118,7 +124,7 @@ impl<'a> Executable<'a> {
         }
     }
 
-    pub fn execute<'b>(&'b mut self)
+    pub fn execute<'b>(&'b mut self) -> Result<()>
     where
         'a: 'b,
     {
@@ -153,7 +159,13 @@ impl<'a> Executable<'a> {
                 }
                 if (now.elapsed().unwrap() > Duration::from_secs(5)) {
                     print_status(cores_instructions);
-                    check_cycle(cpu, cores_instructions, send_recv);
+                    if let Some(deadlock) = detect_deadlock(cores_instructions) {
+                        bail!(
+                            "Deadlock cycle detected: {} [{}]",
+                            deadlock.cycle,
+                            deadlock.states
+                        );
+                    }
                     now = SystemTime::now();
                 }
             }
@@ -178,8 +190,23 @@ impl<'a> Executable<'a> {
         }
         print_status(cores_instructions);
 
+        if let Some(deadlock) = detect_deadlock(cores_instructions) {
+            bail!(
+                "Deadlock cycle detected: {} [{}]",
+                deadlock.cycle,
+                deadlock.states
+            );
+        }
+        if cores_instructions
+            .iter()
+            .any(|core_inst| core_inst.program_counter < core_inst.instructions.len())
+        {
+            bail!("Execution stalled with unfinished instructions");
+        }
+
         #[cfg(feature = "profile_time")]
         TRACER.lock().unwrap().report();
+        Ok(())
     }
 
     pub fn cpu(&self) -> &CPU<'a> {
@@ -201,12 +228,12 @@ impl<'a> Executable<'a> {
     }
 }
 
-fn check_cycle(cpu: &mut CPU, cores_instructions: &[CoreInstructions], send_recv: &mut SendRecv) {
+fn detect_deadlock(cores_instructions: &[CoreInstructions]) -> Option<DeadlockInfo> {
     #[derive(Debug, PartialEq, Eq)]
     enum CoreState {
-        SendingTo(i32),
-        ReceivingFrom(i32),
-        Working, 
+        SendingTo(i32, i32),
+        ReceivingFrom(i32, i32),
+        Working,
         Halted,
     }
 
@@ -223,9 +250,9 @@ fn check_cycle(cpu: &mut CPU, cores_instructions: &[CoreInstructions], send_recv
         let (this_core, target_core) = data.get_core_immcore();
 
         if isa_recv(functor_address) {
-            states.insert(this_core, CoreState::ReceivingFrom(target_core));
+            states.insert(this_core, CoreState::ReceivingFrom(target_core, data.imm_len()));
         } else if isa_send(functor_address) {
-            states.insert(this_core, CoreState::SendingTo(target_core));
+            states.insert(this_core, CoreState::SendingTo(target_core, data.imm_len()));
         } else {
             states.insert(this_core, CoreState::Working);
         }
@@ -235,15 +262,15 @@ fn check_cycle(cpu: &mut CPU, cores_instructions: &[CoreInstructions], send_recv
 
     for (&core_id, state) in states.iter() {
         match state {
-            CoreState::SendingTo(target_core) => {
+            CoreState::SendingTo(target_core, size) => {
                 let target_state = states.get(target_core).unwrap_or(&CoreState::Halted);
-                if target_state != &CoreState::ReceivingFrom(core_id) {
+                if target_state != &CoreState::ReceivingFrom(core_id, *size) {
                     wait_for.insert(core_id, *target_core);
                 }
             }
-            CoreState::ReceivingFrom(target_core) => {
+            CoreState::ReceivingFrom(target_core, size) => {
                 let target_state = states.get(target_core).unwrap_or(&CoreState::Halted);
-                if target_state != &CoreState::SendingTo(core_id) {
+                if target_state != &CoreState::SendingTo(core_id, *size) {
                     wait_for.insert(core_id, *target_core);
                 }
             }
@@ -279,11 +306,33 @@ fn check_cycle(cpu: &mut CPU, cores_instructions: &[CoreInstructions], send_recv
                     .collect::<Vec<_>>()
                     .join(" -> ");
 
+                let cycle = cycle
+                    .iter()
+                    .copied()
+                    .chain(std::iter::once(waiting_for))
+                    .collect::<Vec<_>>();
                 let cycle_msg = format!("{} -> {}", cycle_str, waiting_for);
+                let states_msg = cycle
+                    .iter()
+                    .filter_map(|core| {
+                        states.get(core).map(|state| match state {
+                            CoreState::SendingTo(target, size) => {
+                                format!("core {} send {}B -> {}", core, size, target)
+                            }
+                            CoreState::ReceivingFrom(source, size) => {
+                                format!("core {} recv {}B <- {}", core, size, source)
+                            }
+                            CoreState::Working => format!("core {} working", core),
+                            CoreState::Halted => format!("core {} halted", core),
+                        })
+                    })
+                    .collect::<Vec<_>>()
+                    .join(", ");
 
-                println!("Fatal: Deadlock cycle detected: {}", cycle_msg);
-                // bail!("Deadlock detected: {}", cycle_msg);
-                break; // Stop tracing
+                return Some(DeadlockInfo {
+                    cycle: cycle_msg,
+                    states: states_msg,
+                });
             }
 
             // Hit a known branch that didn't result in a cycle
@@ -294,6 +343,7 @@ fn check_cycle(cpu: &mut CPU, cores_instructions: &[CoreInstructions], send_recv
             current_core = waiting_for;
         }
     }
+    None
 }
 
 fn handle_wait_sync<'a, 'b, 'c>(