add throughput mode to validation scripts
make raptor also emit input sizes
This commit is contained in:
NiccoloN
@@ -156,6 +156,42 @@ def gen_random_inputs(
|
||||
return arrays_in_order, arrays_by_name
|
||||
|
||||
|
||||
def generate_input_batch(onnx_inputs, first_inputs, batch_size, seed):
|
||||
if batch_size < 1:
|
||||
raise ValueError("batch size must be at least 1")
|
||||
if not onnx_inputs:
|
||||
return [first_inputs] * batch_size
|
||||
|
||||
batch = [first_inputs]
|
||||
for index in range(1, batch_size):
|
||||
sample, _ = gen_random_inputs(onnx_inputs, seed=seed + index)
|
||||
if all(np.array_equal(left, right) for left, right in zip(sample, batch[-1])):
|
||||
sample[0] = sample[0].copy()
|
||||
if sample[0].size == 0:
|
||||
raise ValueError("throughput validation cannot distinguish empty input tensors")
|
||||
if np.issubdtype(sample[0].dtype, np.bool_):
|
||||
sample[0].flat[0] = not sample[0].flat[0]
|
||||
elif np.issubdtype(sample[0].dtype, np.integer):
|
||||
info = np.iinfo(sample[0].dtype)
|
||||
value = sample[0].flat[0]
|
||||
sample[0].flat[0] = value + 1 if value < info.max else value - 1
|
||||
else:
|
||||
sample[0].flat[0] += 1
|
||||
batch.append(sample)
|
||||
return batch
|
||||
|
||||
|
||||
def write_input_batch_csv(path, input_batch):
|
||||
path = pathlib.Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", newline="", encoding="utf-8") as output:
|
||||
writer = csv.writer(output)
|
||||
for sample in input_batch:
|
||||
writer.writerow(
|
||||
np.concatenate([array.reshape(-1) for array in sample]) if sample else ()
|
||||
)
|
||||
|
||||
|
||||
def save_inputs_to_files(onnx_path, arrays_in_order, out_dir):
|
||||
"""
|
||||
Save arrays to CSV files. Returns (flags, files) where flags is a list
|
||||
@@ -201,3 +237,22 @@ def write_inputs_to_memory_bin(memory_bin_path, config_json_path, arrays_in_orde
|
||||
native = arr.astype(arr.dtype.newbyteorder("="), copy=False)
|
||||
f.seek(addr)
|
||||
f.write(native.tobytes(order="C"))
|
||||
|
||||
|
||||
def write_inputs_binary(path, arrays_in_order):
|
||||
"""Write one simulator input in graph-input order."""
|
||||
with open(path, "wb") as f:
|
||||
for arr in arrays_in_order:
|
||||
native = arr.astype(arr.dtype.newbyteorder("="), copy=False)
|
||||
f.write(native.tobytes(order="C"))
|
||||
|
||||
|
||||
def write_input_batch_binaries(input_batch, output_dir, transform=None):
|
||||
output_dir = pathlib.Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
paths = []
|
||||
for index, sample in enumerate(input_batch):
|
||||
path = output_dir / f"input_{index}.bin"
|
||||
write_inputs_binary(path, [transform(sample[0])] if transform is not None else sample)
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
@@ -35,7 +35,7 @@ def parse_pimsim_nn_metrics(output):
|
||||
return metrics
|
||||
|
||||
|
||||
def export_raptor_latency_artifact(pim_dir, output_dir):
|
||||
def export_raptor_pimsim_artifact(pim_dir, output_dir):
|
||||
pim_dir = Path(pim_dir)
|
||||
output_dir = Path(output_dir)
|
||||
if output_dir.exists():
|
||||
|
||||
@@ -10,9 +10,18 @@ from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from colorama import Style, Fore
|
||||
from .gen_network_runner import gen_network_runner
|
||||
from .onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP
|
||||
from .onnx_utils import (
|
||||
_ONNX_TO_NP,
|
||||
gen_random_inputs,
|
||||
generate_input_batch,
|
||||
onnx_io,
|
||||
save_inputs_to_files,
|
||||
write_input_batch_binaries,
|
||||
write_input_batch_csv,
|
||||
write_inputs_to_memory_bin,
|
||||
)
|
||||
from .raptor import compile_with_raptor
|
||||
from .pimsim_nn import export_raptor_latency_artifact, parse_pimsim_nn_metrics, read_raptor_instruction_count
|
||||
from .pimsim_nn import export_raptor_pimsim_artifact, parse_pimsim_nn_metrics, read_raptor_instruction_count
|
||||
from .subprocess_utils import run_command_with_reporter
|
||||
|
||||
STAGE_TITLES = (
|
||||
@@ -36,7 +45,10 @@ STAGE_COLORS = {
|
||||
STAGE_TITLES[7]: Fore.BLUE,
|
||||
}
|
||||
STAGE_COUNT = len(STAGE_TITLES)
|
||||
GENERATED_DIR_NAMES = ("inputs", "outputs", "pimcomp", "raptor", "runner", "simulation")
|
||||
GENERATED_DIR_NAMES = (
|
||||
"inputs", "outputs", "pimcomp", "raptor", "runner", "simulation",
|
||||
"throughput_validation",
|
||||
)
|
||||
|
||||
MODE_FULL = "full"
|
||||
MODE_COMPILE_ONLY = "compile_only"
|
||||
@@ -77,11 +89,19 @@ def sanitize_output_name(name):
|
||||
@dataclass
|
||||
class ValidationResult:
|
||||
passed: bool
|
||||
latency_passed: bool | None = None
|
||||
throughput_passed: bool | None = None
|
||||
pim_pass_timings: dict[str, float] = field(default_factory=dict)
|
||||
pimsim_latency_ms: float | None = None
|
||||
pimsim_throughput_samples_s: float | None = None
|
||||
pimsim_power_mw: float | None = None
|
||||
pimsim_energy_pj: float | None = None
|
||||
pimsim_throughput_average_latency_ms: float | None = None
|
||||
pimsim_throughput_average_power_mw: float | None = None
|
||||
pimsim_throughput_average_energy_pj: float | None = None
|
||||
mode_metrics: dict[str, dict[str, float | int | None]] = field(default_factory=dict)
|
||||
pimsim_status: str = PIMSIM_SKIPPED
|
||||
throughput_pimsim_status: str = PIMSIM_SKIPPED
|
||||
compile_time_s: float | None = None
|
||||
host_memory_bytes: int | None = None
|
||||
cores_memory_bytes: int | None = None
|
||||
@@ -291,11 +311,16 @@ def pimcomp_compatibility_errors(config_path, *, core_count, crossbar_count, cro
|
||||
return errors
|
||||
|
||||
|
||||
def run_pimsim_nn(pimsim_nn_build_dir, pim_dir, config_path, reporter=None, timeout_sec=None):
|
||||
latency_artifact = export_raptor_latency_artifact(pim_dir, Path(pim_dir).parent / "pimsim_nn")
|
||||
def run_pimsim_nn(
|
||||
pimsim_nn_build_dir, pim_dir, config_path, execution_mode,
|
||||
reporter=None, timeout_sec=None, fast=True):
|
||||
pimsim_artifact = export_raptor_pimsim_artifact(pim_dir, Path(pim_dir).parent / "pimsim_nn")
|
||||
command = [pimsim_nn_build_dir / "ChipTest", pimsim_artifact, config_path, "--gui=false"]
|
||||
if fast:
|
||||
command.append("--fast")
|
||||
try:
|
||||
output = run_command(
|
||||
[pimsim_nn_build_dir / "ChipTest", latency_artifact, config_path, "--gui=false"],
|
||||
command,
|
||||
cwd=pimsim_nn_build_dir,
|
||||
reporter=reporter,
|
||||
timeout_sec=timeout_sec,
|
||||
@@ -307,10 +332,14 @@ def run_pimsim_nn(pimsim_nn_build_dir, pim_dir, config_path, reporter=None, time
|
||||
raise PimSimUnsupportedError(PIMSIM_UNSUPPORTED_VSOFTMAX) from exc
|
||||
raise
|
||||
metrics = parse_pimsim_nn_metrics(output)
|
||||
required = ("latency_ms", "average_power_mw", "average_energy_pj")
|
||||
required = (
|
||||
("latency_ms", "average_power_mw", "average_energy_pj")
|
||||
if execution_mode == "latency"
|
||||
else ("throughput", "average_latency_ms", "average_power_mw", "average_energy_pj")
|
||||
)
|
||||
if any(name not in metrics for name in required):
|
||||
raise RuntimeError("pimsim-nn output did not contain latency, average power, and average energy")
|
||||
return tuple(metrics[name] for name in required)
|
||||
raise RuntimeError(f"pimsim-nn output did not contain required {execution_mode} metrics")
|
||||
return metrics
|
||||
|
||||
|
||||
def clean_workspace_artifacts(workspace_dir, model_stem):
|
||||
@@ -327,6 +356,7 @@ def clean_workspace_artifacts(workspace_dir, model_stem):
|
||||
|
||||
for name in GENERATED_DIR_NAMES:
|
||||
remove_path(workspace_dir / name)
|
||||
remove_path(workspace_dir / "inputs.csv")
|
||||
|
||||
for suffix in (".onnx.mlir", ".so", ".tmp"):
|
||||
remove_path(workspace_dir / f"{model_stem}{suffix}")
|
||||
@@ -334,8 +364,10 @@ def clean_workspace_artifacts(workspace_dir, model_stem):
|
||||
return removed_paths
|
||||
|
||||
|
||||
def print_stage(reporter, model_index, model_total, model_name, title):
|
||||
color = STAGE_COLORS.get(title, Fore.WHITE)
|
||||
def print_stage(reporter, model_index, model_total, model_name, title, mode=None):
|
||||
if mode is not None:
|
||||
title = f"{title} ({mode.capitalize()})"
|
||||
color = STAGE_COLORS.get(title, STAGE_COLORS.get(title.split(" (", 1)[0], Fore.WHITE))
|
||||
reporter.log(Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL)
|
||||
reporter.set_stage(model_index, model_total, model_name, title)
|
||||
|
||||
@@ -374,11 +406,38 @@ def build_dump_ranges(config_path, outputs_descriptor):
|
||||
return ",".join(ranges)
|
||||
|
||||
|
||||
def run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges, reporter=None, timeout_sec=None):
|
||||
def build_pim_simulator_command(
|
||||
pim_dir, output_bin_path, dump_ranges, input_paths, mode="latency",
|
||||
batch_output_dir=None):
|
||||
if mode not in ("latency", "throughput"):
|
||||
raise ValueError(f"unknown simulator mode: {mode}")
|
||||
if not input_paths:
|
||||
raise ValueError("simulator requires at least one input")
|
||||
command = [
|
||||
"cargo", "run", "--no-default-features", "--release", "--package", "pim-simulator", "--bin", "pim-simulator",
|
||||
"--", "-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges,
|
||||
"--mode", mode, "--batch-size", str(len(input_paths)),
|
||||
]
|
||||
if batch_output_dir is not None:
|
||||
command += ["--batch-output-dir", str(batch_output_dir)]
|
||||
for path in input_paths:
|
||||
command += ["--input", str(path)]
|
||||
return command
|
||||
|
||||
|
||||
def run_pim_simulator(
|
||||
simulator_dir, pim_dir, output_bin_path, dump_ranges, reporter=None,
|
||||
timeout_sec=None, input_paths=(), mode="latency", batch_output_dir=None):
|
||||
command = build_pim_simulator_command(
|
||||
pim_dir,
|
||||
output_bin_path,
|
||||
dump_ranges,
|
||||
input_paths,
|
||||
mode=mode,
|
||||
batch_output_dir=batch_output_dir,
|
||||
)
|
||||
run_command(
|
||||
["cargo", "run", "--no-default-features", "--release", "--package", "pim-simulator", "--bin", "pim-simulator",
|
||||
"--",
|
||||
"-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges],
|
||||
command,
|
||||
cwd=simulator_dir,
|
||||
reporter=reporter,
|
||||
timeout_sec=timeout_sec,
|
||||
@@ -431,21 +490,121 @@ def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold,
|
||||
return all_passed
|
||||
|
||||
|
||||
def report_validation_failure(reporter, execution_name, stage, exc):
|
||||
reporter.suspend()
|
||||
print(
|
||||
Fore.RED + f"{execution_name.capitalize()} {stage} failed: "
|
||||
f"{type(exc).__name__}: {exc}" + Style.RESET_ALL,
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
reporter.resume()
|
||||
|
||||
|
||||
def validate_execution(
|
||||
execution, state, functional_data, workspace_dir, simulator_dir,
|
||||
pimsim_nn_build_dir, threshold, rtol, verbose, command_timeout_seconds,
|
||||
stage_context, pimsim_fast):
|
||||
reporter, model_index, model_total, model_name = stage_context
|
||||
name = execution["name"]
|
||||
pim_dir = execution["root"] / "pim"
|
||||
batch_size = execution["batch_size"]
|
||||
|
||||
if state["compiled"] and functional_data is not None:
|
||||
input_batch, input_paths, reference_dirs, outputs_descriptor = functional_data
|
||||
simulation_dir = workspace_dir / "simulation" / name
|
||||
try:
|
||||
print_stage(
|
||||
reporter, model_index, model_total, model_name,
|
||||
"Run Functional Simulation", name,
|
||||
)
|
||||
write_inputs_to_memory_bin(
|
||||
pim_dir / "memory.bin", pim_dir / "config.json", input_batch[0])
|
||||
simulation_dir.mkdir(parents=True, exist_ok=True)
|
||||
dump_ranges = build_dump_ranges(pim_dir / "config.json", outputs_descriptor)
|
||||
output_dir = simulation_dir / "outputs"
|
||||
run_pim_simulator(
|
||||
simulator_dir, pim_dir, simulation_dir / "out.bin", dump_ranges,
|
||||
reporter=reporter, timeout_sec=command_timeout_seconds,
|
||||
input_paths=input_paths[:batch_size], mode=name,
|
||||
batch_output_dir=output_dir)
|
||||
reporter.advance()
|
||||
|
||||
print_stage(
|
||||
reporter, model_index, model_total, model_name,
|
||||
"Compare Outputs", name,
|
||||
)
|
||||
reporter.suspend()
|
||||
try:
|
||||
iteration_results = [
|
||||
validate_outputs(
|
||||
parse_pim_simulator_outputs(
|
||||
output_dir / f"output_{index:06d}.bin", outputs_descriptor),
|
||||
reference_dirs[index], outputs_descriptor,
|
||||
threshold, rtol=rtol, verbose=verbose)
|
||||
for index in range(batch_size)
|
||||
]
|
||||
finally:
|
||||
reporter.resume()
|
||||
state["passed"] = all(iteration_results)
|
||||
reporter.advance()
|
||||
except Exception as exc:
|
||||
report_validation_failure(reporter, name, "functional validation", exc)
|
||||
|
||||
print_stage(
|
||||
reporter, model_index, model_total, model_name,
|
||||
"Run Non-functional Simulation", name,
|
||||
)
|
||||
config_path = execution["pimsim_config"]
|
||||
if state["compiled"] and pimsim_nn_build_dir is not None and config_path is not None:
|
||||
try:
|
||||
state["metrics"] = run_pimsim_nn(
|
||||
pimsim_nn_build_dir, pim_dir, config_path, name,
|
||||
reporter=reporter, timeout_sec=command_timeout_seconds,
|
||||
fast=pimsim_fast)
|
||||
state["pimsim_status"] = PIMSIM_DONE
|
||||
metric = (
|
||||
f"Latency: {state['metrics']['latency_ms']:.2f} ms"
|
||||
if name == "latency" else
|
||||
f"Throughput: {state['metrics']['throughput']:.2f} samples/s")
|
||||
energy_unit = "pJ" if name == "latency" else "pJ/it"
|
||||
print_info(
|
||||
reporter, f"{metric}, Power: {state['metrics']['average_power_mw']:.2f} mW, "
|
||||
f"Energy: {state['metrics']['average_energy_pj']:.2f} {energy_unit}")
|
||||
except PimSimUnsupportedError as exc:
|
||||
state["pimsim_status"] = PIMSIM_UNSUPPORTED
|
||||
print_info(reporter, str(exc))
|
||||
except Exception as exc:
|
||||
state["pimsim_status"] = PIMSIM_FAILED
|
||||
report_validation_failure(reporter, name, "non-functional validation", exc)
|
||||
elif not state["compiled"]:
|
||||
state["pimsim_status"] = PIMSIM_NOT_RUN
|
||||
else:
|
||||
print_info(reporter, "pimsim-nn non-functional simulation skipped")
|
||||
reporter.advance()
|
||||
|
||||
|
||||
def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
|
||||
simulator_dir, crossbar_size, crossbar_count, core_count,
|
||||
raptor_extra_args,
|
||||
pimsim_nn_build_dir, pimsim_config_path,
|
||||
threshold, rtol,
|
||||
seed, reporter, model_index, model_total, verbose,
|
||||
command_timeout_seconds, mode):
|
||||
command_timeout_seconds, mode, throughput_pipeline=None,
|
||||
throughput_batch_size=4, throughput_pimsim_config_path=None,
|
||||
pimsim_fast=True):
|
||||
if throughput_pipeline is not None and throughput_batch_size < 2:
|
||||
raise ValueError("throughput validation requires batch size greater than 1")
|
||||
network_onnx_path = Path(network_onnx_path).resolve()
|
||||
raptor_path = Path(raptor_path).resolve()
|
||||
onnx_include_dir = Path(onnx_include_dir).resolve()
|
||||
simulator_dir = Path(simulator_dir).resolve()
|
||||
pimsim_enabled = pimsim_nn_build_dir is not None and pimsim_config_path is not None
|
||||
if pimsim_enabled:
|
||||
if pimsim_nn_build_dir is not None:
|
||||
pimsim_nn_build_dir = Path(pimsim_nn_build_dir).resolve()
|
||||
if pimsim_config_path is not None:
|
||||
pimsim_config_path = Path(pimsim_config_path).resolve()
|
||||
if throughput_pimsim_config_path is not None:
|
||||
throughput_pimsim_config_path = Path(throughput_pimsim_config_path).resolve()
|
||||
compile_extra_args = list(raptor_extra_args or [])
|
||||
owns_reporter = reporter is None
|
||||
reporter = reporter or ProgressReporter(model_total, stages_per_model=len(MODE_STAGE_TITLES[mode]), verbose=verbose)
|
||||
@@ -454,189 +613,209 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
|
||||
raptor_dir = workspace_dir / "raptor"
|
||||
runner_dir = workspace_dir / "runner"
|
||||
runner_build_dir = runner_dir / "build"
|
||||
|
||||
if mode != MODE_RUN_ONLY:
|
||||
clean_workspace_artifacts(workspace_dir, network_onnx_path.stem)
|
||||
Path.mkdir(raptor_dir, exist_ok=True)
|
||||
Path.mkdir(raptor_dir, parents=True, exist_ok=True)
|
||||
Path.mkdir(runner_build_dir, parents=True, exist_ok=True)
|
||||
|
||||
reporter.log(Fore.CYAN + f"[{model_index}/{model_total}]" + Style.RESET_ALL +
|
||||
f" {Style.BRIGHT}Validating {network_onnx_path.name}{Style.RESET_ALL}")
|
||||
failed_with_exception = False
|
||||
stem = network_onnx_path.stem
|
||||
network_so_path = runner_dir / f"{stem}.so"
|
||||
network_mlir_path = raptor_dir / f"{stem}.onnx.mlir"
|
||||
runner_path = runner_build_dir / "runner"
|
||||
executions = [{
|
||||
"name": "latency",
|
||||
"root": raptor_dir,
|
||||
"batch_size": 1,
|
||||
"compile_args": compile_extra_args,
|
||||
"pimsim_config": pimsim_config_path,
|
||||
}]
|
||||
if throughput_pipeline is not None:
|
||||
throughput_args = [
|
||||
arg for arg in compile_extra_args if not str(arg).startswith("--pipeline=")
|
||||
] + [f"--pipeline={throughput_pipeline}"]
|
||||
executions.append({
|
||||
"name": "throughput",
|
||||
"root": raptor_dir / "throughput",
|
||||
"batch_size": throughput_batch_size,
|
||||
"compile_args": throughput_args,
|
||||
"pimsim_config": throughput_pimsim_config_path,
|
||||
})
|
||||
states = {
|
||||
execution["name"]: {
|
||||
"compiled": False,
|
||||
"passed": False,
|
||||
"metrics": {},
|
||||
"pimsim_status": PIMSIM_SKIPPED,
|
||||
"compile_time_s": 0.0,
|
||||
"resource_metrics": {},
|
||||
}
|
||||
for execution in executions
|
||||
}
|
||||
pim_pass_timings = {}
|
||||
compile_time_s = None
|
||||
compile_time_s = 0.0
|
||||
resource_metrics = {}
|
||||
|
||||
try:
|
||||
stem = network_onnx_path.stem
|
||||
network_so_path = runner_dir / f"{stem}.so"
|
||||
network_mlir_path = raptor_dir / f"{stem}.onnx.mlir"
|
||||
runner_path = runner_build_dir / "runner"
|
||||
pim_output_base = raptor_dir / stem
|
||||
|
||||
def compile_pim():
|
||||
nonlocal compile_time_s, resource_metrics
|
||||
started = time.perf_counter()
|
||||
timings = compile_with_raptor(
|
||||
network_onnx_path, raptor_path, pim_output_base, crossbar_size,
|
||||
crossbar_count, core_count=core_count,
|
||||
raptor_extra_args=compile_extra_args, cwd=raptor_dir,
|
||||
verbose=verbose, reporter=reporter,
|
||||
timeout_sec=command_timeout_seconds)
|
||||
compile_time_s = time.perf_counter() - started
|
||||
resource_metrics = collect_pim_resource_metrics(raptor_dir / "pim")
|
||||
return timings
|
||||
|
||||
reference_ready = False
|
||||
if mode != MODE_RUN_ONLY:
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Compile ONNX")
|
||||
network_so_path, network_mlir_path = compile_onnx_network(
|
||||
network_onnx_path, raptor_path, raptor_dir, runner_dir, reporter=reporter,
|
||||
timeout_sec=command_timeout_seconds)
|
||||
print_info(reporter, f"MLIR saved to {network_mlir_path}")
|
||||
print_info(reporter, f"Shared library saved to {network_so_path}")
|
||||
reporter.advance()
|
||||
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Build Runner")
|
||||
gen_network_runner(
|
||||
network_onnx_path,
|
||||
network_so_path,
|
||||
onnx_include_dir,
|
||||
entry="run_main_graph",
|
||||
out=runner_dir / "runner.c",
|
||||
verbose=False,
|
||||
)
|
||||
runner_path = build_onnx_runner(runner_dir, runner_build_dir, reporter=reporter,
|
||||
timeout_sec=command_timeout_seconds)
|
||||
print_info(reporter, f"Runner built at {runner_path}")
|
||||
reporter.advance()
|
||||
|
||||
if mode == MODE_COMPILE_ONLY:
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Compile PIM")
|
||||
pim_pass_timings = compile_pim()
|
||||
print_info(reporter, f"PIM artifacts saved to {raptor_dir / 'pim'}")
|
||||
reporter.advance()
|
||||
reporter.record_result(True)
|
||||
reporter.log(Style.BRIGHT + f"Result: {Fore.GREEN}PASS{Style.RESET_ALL}" + Style.RESET_ALL)
|
||||
return ValidationResult(
|
||||
passed=True, pim_pass_timings=pim_pass_timings,
|
||||
compile_time_s=compile_time_s, **resource_metrics)
|
||||
|
||||
if mode == MODE_RUN_ONLY:
|
||||
required_paths = [
|
||||
(network_so_path, "compiled reference shared library"),
|
||||
(network_mlir_path, "exported ONNX MLIR"),
|
||||
(runner_path, "built reference runner"),
|
||||
(raptor_dir / "pim" / "config.json", "compiled PIM artifacts"),
|
||||
]
|
||||
missing = [f"{description} at {path}" for path, description in required_paths if not path.exists()]
|
||||
if missing:
|
||||
raise FileNotFoundError("run-only mode requires existing artifacts:\n " + "\n ".join(missing))
|
||||
resource_metrics = collect_pim_resource_metrics(raptor_dir / "pim")
|
||||
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Generate Inputs")
|
||||
inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path)
|
||||
inputs_list, _inputs_dict = gen_random_inputs(inputs_descriptor, seed=seed)
|
||||
flags, _files = save_inputs_to_files(network_onnx_path, inputs_list, out_dir=workspace_dir / "inputs")
|
||||
print_info(reporter, f"Saved {len(inputs_list)} input file(s) to {workspace_dir / 'inputs'}")
|
||||
reporter.advance()
|
||||
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Run Reference")
|
||||
out_dir = workspace_dir / "outputs"
|
||||
Path.mkdir(out_dir, exist_ok=True)
|
||||
run_cmd = [runner_path, *flags]
|
||||
run_cmd += ["--save-csv-dir", f"{out_dir}"]
|
||||
run_command(run_cmd, cwd=runner_build_dir, reporter=reporter, timeout_sec=command_timeout_seconds)
|
||||
print_info(reporter, f"Reference outputs saved to {out_dir}")
|
||||
reporter.advance()
|
||||
|
||||
if mode != MODE_RUN_ONLY:
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Compile PIM")
|
||||
pim_pass_timings = compile_pim()
|
||||
print_info(reporter, f"PIM artifacts saved to {raptor_dir / 'pim'}")
|
||||
reporter.advance()
|
||||
|
||||
print_stage(
|
||||
reporter, model_index, model_total, network_onnx_path.name,
|
||||
"Run Functional Simulation")
|
||||
pim_dir = raptor_dir / "pim"
|
||||
write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", inputs_list)
|
||||
simulation_dir = workspace_dir / "simulation"
|
||||
Path.mkdir(simulation_dir, exist_ok=True)
|
||||
dump_ranges = build_dump_ranges(pim_dir / "config.json", outputs_descriptor)
|
||||
output_bin_path = simulation_dir / "out.bin"
|
||||
run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges, reporter=reporter,
|
||||
timeout_sec=command_timeout_seconds)
|
||||
print_info(reporter, f"Functional simulation output saved to {output_bin_path}")
|
||||
reporter.advance()
|
||||
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Compare Outputs")
|
||||
sim_arrays = parse_pim_simulator_outputs(output_bin_path, outputs_descriptor)
|
||||
reporter.suspend()
|
||||
passed = validate_outputs(sim_arrays, out_dir, outputs_descriptor, threshold, rtol=rtol, verbose=verbose)
|
||||
reporter.resume()
|
||||
reporter.advance()
|
||||
|
||||
print_stage(
|
||||
reporter, model_index, model_total, network_onnx_path.name,
|
||||
"Run Non-functional Simulation")
|
||||
pimsim_latency_ms = None
|
||||
pimsim_power_mw = None
|
||||
pimsim_energy_pj = None
|
||||
pimsim_status = PIMSIM_SKIPPED
|
||||
if pimsim_enabled:
|
||||
try:
|
||||
pimsim_latency_ms, pimsim_power_mw, pimsim_energy_pj = run_pimsim_nn(
|
||||
pimsim_nn_build_dir,
|
||||
pim_dir,
|
||||
pimsim_config_path,
|
||||
reporter=reporter,
|
||||
timeout_sec=command_timeout_seconds,
|
||||
)
|
||||
pimsim_status = PIMSIM_DONE
|
||||
print_info(
|
||||
reporter,
|
||||
f"Latency: {pimsim_latency_ms:.6f} ms, "
|
||||
f"Power: {pimsim_power_mw:.6f} mW, "
|
||||
f"Energy: {pimsim_energy_pj:.6f} pJ")
|
||||
except PimSimUnsupportedError as exc:
|
||||
pimsim_status = PIMSIM_UNSUPPORTED
|
||||
print_info(reporter, str(exc))
|
||||
except Exception as exc:
|
||||
pimsim_status = PIMSIM_FAILED
|
||||
reporter.suspend()
|
||||
print(
|
||||
Fore.RED
|
||||
+ f"pimsim-nn non-functional simulation failed: {type(exc).__name__}: {exc}"
|
||||
+ Style.RESET_ALL,
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
reporter.resume()
|
||||
else:
|
||||
print_info(reporter, "pimsim-nn non-functional simulation skipped")
|
||||
reporter.advance()
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Compile ONNX")
|
||||
network_so_path, network_mlir_path = compile_onnx_network(
|
||||
network_onnx_path, raptor_path, raptor_dir, runner_dir,
|
||||
reporter=reporter, timeout_sec=command_timeout_seconds)
|
||||
print_info(reporter, f"MLIR saved to {network_mlir_path}")
|
||||
print_info(reporter, f"Shared library saved to {network_so_path}")
|
||||
reporter.advance()
|
||||
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Build Runner")
|
||||
gen_network_runner(
|
||||
network_onnx_path, network_so_path, onnx_include_dir,
|
||||
entry="run_main_graph", out=runner_dir / "runner.c", verbose=False)
|
||||
runner_path = build_onnx_runner(
|
||||
runner_dir, runner_build_dir, reporter=reporter,
|
||||
timeout_sec=command_timeout_seconds)
|
||||
print_info(reporter, f"Runner built at {runner_path}")
|
||||
reporter.advance()
|
||||
reference_ready = True
|
||||
except Exception as exc:
|
||||
report_validation_failure(reporter, "reference", "compilation", exc)
|
||||
else:
|
||||
required_paths = (network_so_path, network_mlir_path, runner_path)
|
||||
reference_ready = all(path.exists() for path in required_paths)
|
||||
if not reference_ready:
|
||||
report_validation_failure(reporter, "reference", "artifact lookup", FileNotFoundError(
|
||||
"run-only mode requires the compiled shared library, ONNX MLIR, and runner"))
|
||||
|
||||
for execution in executions:
|
||||
name = execution["name"]
|
||||
root = execution["root"]
|
||||
pim_dir = root / "pim"
|
||||
if mode == MODE_RUN_ONLY:
|
||||
states[name]["compiled"] = (pim_dir / "config.json").exists()
|
||||
if not states[name]["compiled"]:
|
||||
report_validation_failure(reporter, name, "artifact lookup", FileNotFoundError(
|
||||
f"run-only mode requires compiled PIM artifacts at {pim_dir}"))
|
||||
else:
|
||||
states[name]["resource_metrics"] = collect_pim_resource_metrics(pim_dir)
|
||||
if name == "latency":
|
||||
resource_metrics = states[name]["resource_metrics"]
|
||||
continue
|
||||
try:
|
||||
print_stage(
|
||||
reporter, model_index, model_total, network_onnx_path.name,
|
||||
"Compile PIM", name,
|
||||
)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
started = time.perf_counter()
|
||||
timings = compile_with_raptor(
|
||||
network_onnx_path, raptor_path, root / stem, crossbar_size,
|
||||
crossbar_count, core_count=core_count,
|
||||
raptor_extra_args=execution["compile_args"], cwd=root,
|
||||
verbose=verbose, reporter=reporter,
|
||||
timeout_sec=command_timeout_seconds)
|
||||
elapsed = time.perf_counter() - started
|
||||
compile_time_s += elapsed
|
||||
states[name]["compile_time_s"] = elapsed
|
||||
for label, duration in timings.items():
|
||||
pim_pass_timings[label] = pim_pass_timings.get(label, 0) + duration
|
||||
states[name]["compiled"] = True
|
||||
states[name]["resource_metrics"] = collect_pim_resource_metrics(pim_dir)
|
||||
if name == "latency":
|
||||
resource_metrics = states[name]["resource_metrics"]
|
||||
print_info(reporter, f"PIM artifacts saved to {pim_dir}")
|
||||
except Exception as exc:
|
||||
report_validation_failure(reporter, name, "compilation", exc)
|
||||
reporter.advance()
|
||||
|
||||
if mode == MODE_COMPILE_ONLY:
|
||||
for state in states.values():
|
||||
state["passed"] = reference_ready and state["compiled"]
|
||||
else:
|
||||
input_batch = input_paths = reference_dirs = outputs_descriptor = None
|
||||
try:
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Generate Inputs")
|
||||
inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path)
|
||||
first_inputs, _ = gen_random_inputs(inputs_descriptor, seed=seed)
|
||||
input_batch = generate_input_batch(
|
||||
inputs_descriptor, first_inputs,
|
||||
max(execution["batch_size"] for execution in executions), seed)
|
||||
write_input_batch_csv(workspace_dir / "inputs.csv", input_batch)
|
||||
input_paths = write_input_batch_binaries(input_batch, workspace_dir / "simulation" / "inputs")
|
||||
input_flags = [
|
||||
save_inputs_to_files(
|
||||
network_onnx_path, inputs,
|
||||
out_dir=workspace_dir / "inputs" / f"{index:06d}")[0]
|
||||
for index, inputs in enumerate(input_batch)
|
||||
]
|
||||
print_info(reporter, f"Saved {len(input_batch)} input sample(s) to {workspace_dir / 'inputs.csv'}")
|
||||
reporter.advance()
|
||||
|
||||
if not reference_ready:
|
||||
raise FileNotFoundError("reference runner is unavailable")
|
||||
print_stage(reporter, model_index, model_total, network_onnx_path.name, "Run Reference")
|
||||
reference_dirs = []
|
||||
for index, flags in enumerate(input_flags):
|
||||
reference_dir = workspace_dir / "outputs" / f"{index:06d}"
|
||||
reference_dir.mkdir(parents=True, exist_ok=True)
|
||||
run_command(
|
||||
[runner_path, *flags, "--save-csv-dir", str(reference_dir)],
|
||||
cwd=runner_build_dir, reporter=reporter,
|
||||
timeout_sec=command_timeout_seconds)
|
||||
reference_dirs.append(reference_dir)
|
||||
print_info(reporter, f"Reference outputs saved for {len(reference_dirs)} sample(s)")
|
||||
reporter.advance()
|
||||
except Exception as exc:
|
||||
report_validation_failure(reporter, "reference", "execution", exc)
|
||||
|
||||
functional_data = None
|
||||
if all(value is not None for value in (
|
||||
input_batch, input_paths, reference_dirs, outputs_descriptor)):
|
||||
functional_data = input_batch, input_paths, reference_dirs, outputs_descriptor
|
||||
stage_context = reporter, model_index, model_total, network_onnx_path.name
|
||||
for execution in executions:
|
||||
validate_execution(
|
||||
execution, states[execution["name"]], functional_data,
|
||||
workspace_dir, simulator_dir, pimsim_nn_build_dir,
|
||||
threshold, rtol, verbose, command_timeout_seconds,
|
||||
stage_context, pimsim_fast)
|
||||
|
||||
latency = states["latency"]
|
||||
throughput = states.get("throughput")
|
||||
passed = all(state["passed"] for state in states.values())
|
||||
latency_metrics = latency["metrics"]
|
||||
throughput_metrics = throughput["metrics"] if throughput else {}
|
||||
reporter.record_result(passed)
|
||||
status = Fore.GREEN + "PASS" + Style.RESET_ALL if passed else Fore.RED + "FAIL" + Style.RESET_ALL
|
||||
reporter.log(Style.BRIGHT + f"Result: {status}" + Style.RESET_ALL)
|
||||
mode_metrics = {
|
||||
name: {
|
||||
"compile_time_s": state["compile_time_s"] or None,
|
||||
**state["resource_metrics"],
|
||||
}
|
||||
for name, state in states.items()
|
||||
}
|
||||
return ValidationResult(
|
||||
passed=passed,
|
||||
latency_passed=latency["passed"],
|
||||
throughput_passed=throughput["passed"] if throughput else None,
|
||||
pim_pass_timings=pim_pass_timings,
|
||||
pimsim_latency_ms=pimsim_latency_ms,
|
||||
pimsim_power_mw=pimsim_power_mw,
|
||||
pimsim_energy_pj=pimsim_energy_pj,
|
||||
pimsim_status=pimsim_status,
|
||||
compile_time_s=compile_time_s,
|
||||
pimsim_latency_ms=latency_metrics.get("latency_ms"),
|
||||
pimsim_throughput_samples_s=throughput_metrics.get("throughput"),
|
||||
pimsim_power_mw=latency_metrics.get("average_power_mw"),
|
||||
pimsim_energy_pj=latency_metrics.get("average_energy_pj"),
|
||||
pimsim_throughput_average_latency_ms=throughput_metrics.get("average_latency_ms"),
|
||||
pimsim_throughput_average_power_mw=throughput_metrics.get("average_power_mw"),
|
||||
pimsim_throughput_average_energy_pj=throughput_metrics.get("average_energy_pj"),
|
||||
mode_metrics=mode_metrics,
|
||||
pimsim_status=latency["pimsim_status"],
|
||||
throughput_pimsim_status=(
|
||||
throughput["pimsim_status"] if throughput else PIMSIM_SKIPPED),
|
||||
compile_time_s=compile_time_s or None,
|
||||
**resource_metrics,
|
||||
)
|
||||
except Exception:
|
||||
failed_with_exception = True
|
||||
reporter.record_result(False)
|
||||
reporter.log(Style.BRIGHT + Fore.RED + "Result: FAIL" + Style.RESET_ALL)
|
||||
reporter.suspend()
|
||||
raise
|
||||
finally:
|
||||
if not failed_with_exception:
|
||||
reporter.log("=" * 72)
|
||||
reporter.log("=" * 72)
|
||||
if owns_reporter:
|
||||
reporter.finish()
|
||||
|
||||
Reference in New Issue
Block a user