remove duplicated code

quieter validation scripts (with optional verbose flag)

validation/subprocess_utils.py

@@ -16,7 +16,7 @@ def _read_chunk(fd, treat_eio_as_eof=False):
         raise
 
 
-def _stream_output(fd, process, reporter, treat_eio_as_eof=False):
+def _stream_output(fd, process, reporter, treat_eio_as_eof=False, stream_output=True):
     selector = selectors.DefaultSelector()
     recent_output = bytearray()
     captured_output = bytearray()
@@ -32,19 +32,22 @@ def _stream_output(fd, process, reporter, treat_eio_as_eof=False):
                     os.close(key.fileobj)
                     continue
 
-                reporter._clear()
-                os.write(1, data)
-                reporter._render()
+                if stream_output:
+                    reporter._clear()
+                    os.write(1, data)
+                    reporter._render()
                 captured_output.extend(data)
-                recent_output.extend(data)
-                if len(recent_output) > MAX_ERROR_OUTPUT_BYTES:
-                    del recent_output[:-MAX_ERROR_OUTPUT_BYTES]
+                if stream_output:
+                    recent_output.extend(data)
+                    if len(recent_output) > MAX_ERROR_OUTPUT_BYTES:
+                        del recent_output[:-MAX_ERROR_OUTPUT_BYTES]
     finally:
         selector.close()
 
     return_code = process.wait()
     if return_code != 0:
-        raise subprocess.CalledProcessError(return_code, process.args, output=bytes(recent_output))
+        error_output = captured_output if not stream_output else recent_output
+        raise subprocess.CalledProcessError(return_code, process.args, output=bytes(error_output))
     return bytes(captured_output)
 
 
@@ -62,6 +65,18 @@ def run_command_with_reporter(cmd, cwd=None, reporter=None, capture_output=False
         subprocess.run(cmd, cwd=cwd, check=True)
         return None
 
+    stream_output = bool(getattr(reporter, "verbose", False))
+    if not stream_output:
+        process = subprocess.Popen(
+            cmd,
+            cwd=cwd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+        )
+        assert process.stdout is not None
+        output = _stream_output(process.stdout.fileno(), process, reporter, stream_output=False)
+        return output.decode("utf-8", errors="replace") if capture_output else None
+
     try:
         master_fd, slave_fd = pty.openpty()
     except OSError:

validation/validate.py

@@ -27,6 +27,10 @@ def print_validation_error(reporter, rel, exc):
           file=sys.stderr, flush=True)
     if isinstance(exc, subprocess.CalledProcessError):
         print(format_return_status(exc.returncode), file=sys.stderr, flush=True)
+        if exc.output:
+            output_text = exc.output.decode("utf-8", errors="replace") if isinstance(exc.output, bytes) else str(exc.output)
+            if output_text:
+                print(output_text, file=sys.stderr, end="" if output_text.endswith("\n") else "\n", flush=True)
     else:
         print(f"{type(exc).__name__}: {exc}", file=sys.stderr, flush=True)
     print("=" * 72, file=sys.stderr, flush=True)
@@ -60,6 +64,8 @@ def main():
                     help="Core count to pass to Raptor. If omitted, Raptor uses its default.")
     ap.add_argument("--clean", action="store_true",
                     help="Remove generated validation artifacts under each model workspace and exit.")
+    ap.add_argument("--verbose", action="store_true",
+                    help="Print per-stage progress and subprocess logs for passing validations too.")
     a = ap.parse_args()
 
     script_dir = Path(__file__).parent.resolve()
@@ -101,7 +107,7 @@ def main():
     pass_timing_counts = {label: 0 for _, label in PIM_PASS_LABELS}
     total_timing_sum = 0.0
     timed_benchmark_count = 0
-    reporter = ProgressReporter(len(onnx_files))
+    reporter = ProgressReporter(len(onnx_files), verbose=a.verbose)
     for index, onnx_path in enumerate(onnx_files, start=1):
         rel = onnx_path.relative_to(operations_dir)
         try:
@@ -112,6 +118,7 @@ def main():
                 reporter=reporter,
                 model_index=index,
                 model_total=len(onnx_files),
+                verbose=a.verbose,
             )
             results[str(rel)] = result.passed
             if result.pim_pass_timings:
@@ -134,22 +141,25 @@ def main():
     # Summary
     n_passed = sum(1 for passed in results.values() if passed)
     n_total = len(results)
-    status_width = len("Result")
-    path_width = max(len("Operation"), *(len(rel) for rel in results))
-    separator = f"+-{'-' * path_width}-+-{'-' * status_width}-+"
-
     print("\n" + Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
-    print(separator)
-    print(f"| {'Operation'.ljust(path_width)} | {'Result'.ljust(status_width)} |")
-    print(separator)
-    for rel, passed in results.items():
-        plain_status = "PASS" if passed else "FAIL"
-        status = Fore.GREEN + plain_status.ljust(status_width) + Style.RESET_ALL if passed else \
-            Fore.RED + plain_status.ljust(status_width) + Style.RESET_ALL
-        print(f"| {rel.ljust(path_width)} | {status} |")
-    print(separator)
     print(Style.BRIGHT + f"Passed: {n_passed}" + Style.RESET_ALL)
     print(Style.BRIGHT + f"Failed: {n_total - n_passed}" + Style.RESET_ALL)
+    failing = [rel for rel, passed in results.items() if not passed]
+    if a.verbose or failing:
+        status_width = len("Result")
+        path_width = max(len("Operation"), *(len(rel) for rel in results))
+        separator = f"+-{'-' * path_width}-+-{'-' * status_width}-+"
+        print(separator)
+        print(f"| {'Operation'.ljust(path_width)} | {'Result'.ljust(status_width)} |")
+        print(separator)
+        for rel, passed in results.items():
+            if not a.verbose and passed:
+                continue
+            plain_status = "PASS" if passed else "FAIL"
+            status = Fore.GREEN + plain_status.ljust(status_width) + Style.RESET_ALL if passed else \
+                Fore.RED + plain_status.ljust(status_width) + Style.RESET_ALL
+            print(f"| {rel.ljust(path_width)} | {status} |")
+        print(separator)
     print_average_pim_pass_timings(
         pass_timing_sums,
         pass_timing_counts,

validation/validate_one.py

@@ -36,7 +36,7 @@ class ValidationResult:
 
 
 class ProgressReporter:
-    def __init__(self, total_models, stages_per_model=STAGE_COUNT, enabled=None):
+    def __init__(self, total_models, stages_per_model=STAGE_COUNT, enabled=None, verbose=False):
         self.total_models = total_models
         self.stages_per_model = stages_per_model
         self.total_steps = max(1, total_models * stages_per_model)
@@ -45,6 +45,7 @@ class ProgressReporter:
         self.failed_models = 0
         self.current_label = ""
         self.enabled = sys.stdout.isatty() if enabled is None else enabled
+        self.verbose = verbose
         self.columns = shutil.get_terminal_size((100, 20)).columns
         self.suspended = False
 
@@ -96,6 +97,8 @@ class ProgressReporter:
         sys.stdout.flush()
 
     def log(self, message="", color=None):
+        if not self.enabled and not self.verbose:
+            return
         if self.enabled:
             self._clear()
         if color:
@@ -228,7 +231,7 @@ def parse_pim_simulator_outputs(output_bin_path, outputs_descriptor):
     return arrays
 
 
-def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1e-3):
+def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1e-3, verbose=False):
     all_passed = True
     rows = []
     for sim_array, (oi, name, _, shape) in zip(sim_arrays, outputs_descriptor):
@@ -245,26 +248,27 @@ def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1
     result_width = len("Result")
     separator = f"  +-{'-' * name_width}-+-{'-' * diff_width}-+-{'-' * result_width}-+"
 
-    print(separator)
-    print(f"  | {'Output'.ljust(name_width)} | {'Max diff'.ljust(diff_width)} | {'Result'} |")
-    print(separator)
-    for name, diff_text, passed in rows:
-        status_text = ("PASS" if passed else "FAIL").ljust(result_width)
-        status = Fore.GREEN + status_text + Style.RESET_ALL if passed else Fore.RED + status_text + Style.RESET_ALL
-        print(f"  | {name.ljust(name_width)} | {diff_text.ljust(diff_width)} | {status} |")
-    print(separator)
+    if verbose or not all_passed:
+        print(separator)
+        print(f"  | {'Output'.ljust(name_width)} | {'Max diff'.ljust(diff_width)} | {'Result'} |")
+        print(separator)
+        for name, diff_text, passed in rows:
+            status_text = ("PASS" if passed else "FAIL").ljust(result_width)
+            status = Fore.GREEN + status_text + Style.RESET_ALL if passed else Fore.RED + status_text + Style.RESET_ALL
+            print(f"  | {name.ljust(name_width)} | {diff_text.ljust(diff_width)} | {status} |")
+        print(separator)
     return all_passed
 
 
 def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
                      simulator_dir, crossbar_size=64, crossbar_count=8, core_count=None, threshold=1e-3,
-                     reporter=None, model_index=1, model_total=1):
+                     reporter=None, model_index=1, model_total=1, verbose=False):
     network_onnx_path = Path(network_onnx_path).resolve()
     raptor_path = Path(raptor_path).resolve()
     onnx_include_dir = Path(onnx_include_dir).resolve()
     simulator_dir = Path(simulator_dir).resolve()
     owns_reporter = reporter is None
-    reporter = reporter or ProgressReporter(model_total)
+    reporter = reporter or ProgressReporter(model_total, verbose=verbose)
 
     workspace_dir = network_onnx_path.parent
     clean_workspace_artifacts(workspace_dir, network_onnx_path.stem)
@@ -331,7 +335,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
         print_stage(reporter, model_index, model_total, network_onnx_path.name, "Compare Outputs")
         sim_arrays = parse_pim_simulator_outputs(output_bin_path, outputs_descriptor)
         reporter.suspend()
-        passed = validate_outputs(sim_arrays, out_dir, outputs_descriptor, threshold)
+        passed = validate_outputs(sim_arrays, out_dir, outputs_descriptor, threshold, verbose=verbose)
         reporter.resume()
         reporter.advance()
         reporter.record_result(passed)