add peft scheduling

better deadlock report by pim simulator

validation/subprocess_utils.py

@@ -3,6 +3,7 @@ import os
 import pty
 import selectors
 import subprocess
+import time
 
 MAX_ERROR_OUTPUT_BYTES = 8192
 
@@ -16,16 +17,26 @@ def _read_chunk(fd, treat_eio_as_eof=False):
         raise
 
 
-def _stream_output(fd, process, reporter, treat_eio_as_eof=False, stream_output=True):
+def _stream_output(fd, process, reporter, treat_eio_as_eof=False, stream_output=True, timeout_sec=None):
     selector = selectors.DefaultSelector()
     recent_output = bytearray()
     captured_output = bytearray()
+    deadline = None if timeout_sec is None else time.monotonic() + timeout_sec
 
     try:
         selector.register(fd, selectors.EVENT_READ)
 
         while selector.get_map():
-            for key, _ in selector.select():
+            select_timeout = None
+            if deadline is not None:
+                remaining = deadline - time.monotonic()
+                if remaining <= 0:
+                    process.kill()
+                    process.wait()
+                    raise subprocess.TimeoutExpired(process.args, timeout_sec, output=bytes(captured_output))
+                select_timeout = min(1.0, remaining)
+
+            for key, _ in selector.select(select_timeout):
                 data = _read_chunk(key.fileobj, treat_eio_as_eof=treat_eio_as_eof)
                 if not data:
                     selector.unregister(key.fileobj)
@@ -53,7 +64,7 @@ def _stream_output(fd, process, reporter, treat_eio_as_eof=False, stream_output=
     return bytes(captured_output)
 
 
-def run_command_with_reporter(cmd, cwd=None, reporter=None, capture_output=False):
+def run_command_with_reporter(cmd, cwd=None, reporter=None, capture_output=False, timeout_sec=None):
     if reporter is None:
         if capture_output:
             completed = subprocess.run(
@@ -62,9 +73,10 @@ def run_command_with_reporter(cmd, cwd=None, reporter=None, capture_output=False
                 check=True,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
+                timeout=timeout_sec,
             )
             return completed.stdout.decode("utf-8", errors="replace")
-        subprocess.run(cmd, cwd=cwd, check=True)
+        subprocess.run(cmd, cwd=cwd, check=True, timeout=timeout_sec)
         return None
 
     stream_output = bool(getattr(reporter, "verbose", False))
@@ -74,6 +86,7 @@ def run_command_with_reporter(cmd, cwd=None, reporter=None, capture_output=False
             cwd=cwd,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
+            timeout=timeout_sec,
         )
         if completed.returncode != 0:
             raise subprocess.CalledProcessError(completed.returncode, completed.args, output=completed.stdout)
@@ -89,7 +102,7 @@ def run_command_with_reporter(cmd, cwd=None, reporter=None, capture_output=False
             stderr=subprocess.STDOUT,
         )
         assert process.stdout is not None
-        output = _stream_output(process.stdout.fileno(), process, reporter)
+        output = _stream_output(process.stdout.fileno(), process, reporter, timeout_sec=timeout_sec)
         return output.decode("utf-8", errors="replace") if capture_output else None
 
     try:
@@ -102,5 +115,5 @@ def run_command_with_reporter(cmd, cwd=None, reporter=None, capture_output=False
     finally:
         os.close(slave_fd)
 
-    output = _stream_output(master_fd, process, reporter, treat_eio_as_eof=True)
+    output = _stream_output(master_fd, process, reporter, treat_eio_as_eof=True, timeout_sec=timeout_sec)
     return output.decode("utf-8", errors="replace") if capture_output else None