add throughput mode to validation scripts
make raptor also emit input sizes
This commit is contained in:
@@ -15,7 +15,7 @@ REPO_ROOT = VALIDATION_DIR.parent
|
||||
if str(VALIDATION_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(VALIDATION_DIR))
|
||||
|
||||
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_to_memory_bin
|
||||
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_binary, write_inputs_to_memory_bin
|
||||
from raptor_validation.validate_one import (
|
||||
MODE_COMPILE_ONLY,
|
||||
build_dump_ranges,
|
||||
@@ -222,6 +222,8 @@ def run_reference_and_simulator(args, model_path: Path, tensor: np.ndarray):
|
||||
subprocess.run(runner_cmd, cwd=runner_build_dir, check=True)
|
||||
|
||||
write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", [tensor])
|
||||
input_bin_path = simulation_dir / "input.bin"
|
||||
write_inputs_binary(input_bin_path, [tensor])
|
||||
dump_ranges = build_dump_ranges(pim_dir / "config.json", output_descriptors)
|
||||
output_bin_path = simulation_dir / "out.bin"
|
||||
run_pim_simulator(
|
||||
@@ -230,6 +232,7 @@ def run_reference_and_simulator(args, model_path: Path, tensor: np.ndarray):
|
||||
output_bin_path,
|
||||
dump_ranges,
|
||||
timeout_sec=args.command_timeout_seconds,
|
||||
input_paths=[input_bin_path],
|
||||
)
|
||||
|
||||
output_index, output_name, output_dtype_code, output_shape = output_descriptors[0]
|
||||
|
||||
@@ -33,18 +33,22 @@ sys.path.insert(0, str(VALIDATION_DIR))
|
||||
from raptor_validation.gen_network_runner import gen_network_runner # noqa: E402
|
||||
from raptor_validation.onnx_utils import ( # noqa: E402
|
||||
_ONNX_TO_NP,
|
||||
generate_input_batch,
|
||||
gen_random_inputs,
|
||||
onnx_io,
|
||||
save_inputs_to_files,
|
||||
write_input_batch_binaries,
|
||||
write_input_batch_csv,
|
||||
write_inputs_to_memory_bin,
|
||||
)
|
||||
from raptor_validation.raptor import compile_with_raptor # noqa: E402
|
||||
from raptor_validation.pimsim_nn import ( # noqa: E402
|
||||
export_raptor_latency_artifact,
|
||||
export_raptor_pimsim_artifact,
|
||||
parse_pimsim_nn_metrics,
|
||||
)
|
||||
from raptor_validation.validate_one import ( # noqa: E402
|
||||
STAGE_COLORS,
|
||||
build_pim_simulator_command,
|
||||
build_dump_ranges,
|
||||
parse_pim_simulator_outputs,
|
||||
)
|
||||
@@ -402,6 +406,29 @@ def generate_reference_outputs(
|
||||
return reference_dir
|
||||
|
||||
|
||||
def generate_reference_batch_outputs(
|
||||
runner_path: Path,
|
||||
runner_build_dir: Path,
|
||||
model_path: Path,
|
||||
input_batch: list[list[np.ndarray]],
|
||||
steps: list[StepRecord],
|
||||
args: argparse.Namespace,
|
||||
out_dir: Path,
|
||||
) -> list[Path]:
|
||||
return [
|
||||
generate_reference_outputs(
|
||||
runner_path,
|
||||
runner_build_dir,
|
||||
model_path,
|
||||
sample,
|
||||
steps,
|
||||
args,
|
||||
out_dir / f"batch_{index:06d}",
|
||||
)
|
||||
for index, sample in enumerate(input_batch)
|
||||
]
|
||||
|
||||
|
||||
def prepare_common_artifacts(
|
||||
args: argparse.Namespace,
|
||||
model_path: Path,
|
||||
@@ -500,31 +527,31 @@ def run_functional_validation(
|
||||
pim_dir: Path,
|
||||
config_path: Path,
|
||||
output_bin: Path,
|
||||
input_bins: list[Path],
|
||||
outputs_desc: list[tuple[int, str, int, list[int]]],
|
||||
reference_dir: Path,
|
||||
reference_dirs: list[Path],
|
||||
steps: list[StepRecord],
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
channel_last: bool = False,
|
||||
) -> CompareResult:
|
||||
dump_ranges = build_dump_ranges(config_path, outputs_desc)
|
||||
cmd = [
|
||||
"cargo",
|
||||
"run",
|
||||
"--no-default-features",
|
||||
"--release",
|
||||
"--package",
|
||||
"pim-simulator",
|
||||
"--bin",
|
||||
"pim-simulator",
|
||||
"--",
|
||||
"-f",
|
||||
str(pim_dir),
|
||||
"-o",
|
||||
str(output_bin),
|
||||
"-d",
|
||||
batch_size = len(input_bins)
|
||||
if batch_size == 0 or len(reference_dirs) != batch_size:
|
||||
raise ValueError(
|
||||
f"functional validation requires one input and reference per iteration, got "
|
||||
f"{batch_size} inputs and {len(reference_dirs)} references"
|
||||
)
|
||||
batch_output_dir = output_bin.parent / f"{output_bin.stem}_iterations"
|
||||
shutil.rmtree(batch_output_dir, ignore_errors=True)
|
||||
cmd = build_pim_simulator_command(
|
||||
pim_dir,
|
||||
output_bin,
|
||||
dump_ranges,
|
||||
]
|
||||
input_bins,
|
||||
args.pimsim_mode,
|
||||
batch_output_dir,
|
||||
)
|
||||
output_bin.parent.mkdir(parents=True, exist_ok=True)
|
||||
run_logged(
|
||||
label,
|
||||
@@ -534,13 +561,25 @@ def run_functional_validation(
|
||||
steps=steps,
|
||||
stage="Run Functional Simulation",
|
||||
)
|
||||
return compare_simulator_outputs(
|
||||
output_bin,
|
||||
outputs_desc,
|
||||
reference_dir,
|
||||
threshold=args.threshold,
|
||||
rtol=args.rtol,
|
||||
channel_last=channel_last,
|
||||
max_diffs: dict[str, float] = {}
|
||||
failed_iterations = []
|
||||
for index, reference_dir in enumerate(reference_dirs):
|
||||
result = compare_simulator_outputs(
|
||||
batch_output_dir / f"output_{index:06d}.bin",
|
||||
outputs_desc,
|
||||
reference_dir,
|
||||
threshold=args.threshold,
|
||||
rtol=args.rtol,
|
||||
channel_last=channel_last,
|
||||
)
|
||||
if not result.passed:
|
||||
failed_iterations.append(index)
|
||||
for name, diff in result.max_diffs.items():
|
||||
max_diffs[name] = max(max_diffs.get(name, 0.0), diff)
|
||||
return CompareResult(
|
||||
passed=not failed_iterations,
|
||||
max_diffs=max_diffs,
|
||||
error=(f"batch iterations failed: {failed_iterations}" if failed_iterations else None),
|
||||
)
|
||||
|
||||
|
||||
@@ -730,6 +769,7 @@ def export_pimcomp_for_rust(
|
||||
"adc_count": sim_info["config"]["adc_count"],
|
||||
"array_group_map": {},
|
||||
"inputs_addresses": [input_addr],
|
||||
"inputs_sizes": [input_tensor.nbytes],
|
||||
"outputs_addresses": [],
|
||||
}
|
||||
output_name_to_node = {node["name"]: node for node in node_list}
|
||||
@@ -887,6 +927,8 @@ def run_pimsim_nn(
|
||||
str(config_path),
|
||||
"--gui=false",
|
||||
]
|
||||
if not args.no_fast:
|
||||
cmd.append("--fast")
|
||||
output = run_logged(
|
||||
label,
|
||||
cmd,
|
||||
@@ -997,8 +1039,12 @@ def perf_status(perf: dict[str, Any]) -> str:
|
||||
return "DONE"
|
||||
|
||||
|
||||
def perf_value(perf: dict[str, Any], key: str) -> Any:
|
||||
return perf[key] if key in perf else "n/a"
|
||||
def perf_value(perf: dict[str, Any], key: str, unit: str = "") -> Any:
|
||||
value = perf.get(key)
|
||||
if value is None:
|
||||
return "n/a"
|
||||
formatted = f"{value:.2f}" if isinstance(value, float) else value
|
||||
return f"{formatted} {unit}" if unit else formatted
|
||||
|
||||
|
||||
def empty_instruction_summary(reason: str | None = None, error: str | None = None) -> dict[str, Any]:
|
||||
@@ -1170,24 +1216,24 @@ def write_report(
|
||||
if pimsim_mode == "throughput":
|
||||
lines.extend(
|
||||
[
|
||||
"| Compiler | Status | Throughput (samples/s) | Avg latency (ms) | Avg power (mW) | Avg energy (pJ/it) | Output count |",
|
||||
"| Compiler | Status | Avg latency | Throughput | Avg power | Avg energy | Output count |",
|
||||
"|---|---|---:|---:|---:|---:|---:|",
|
||||
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'throughput')} | {perf_value(raptor_perf, 'average_latency_ms')} | "
|
||||
f"{perf_value(raptor_perf, 'average_power_mw')} | {perf_value(raptor_perf, 'average_energy_pj')} | {perf_value(raptor_perf, 'output_count')} |",
|
||||
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'throughput')} | {perf_value(pimcomp_perf, 'average_latency_ms')} | "
|
||||
f"{perf_value(pimcomp_perf, 'average_power_mw')} | {perf_value(pimcomp_perf, 'average_energy_pj')} | {perf_value(pimcomp_perf, 'output_count')} |",
|
||||
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'average_latency_ms', 'ms')} | {perf_value(raptor_perf, 'throughput', 'samples/s')} | "
|
||||
f"{perf_value(raptor_perf, 'average_power_mw', 'mW')} | {perf_value(raptor_perf, 'average_energy_pj', 'pJ/it')} | {perf_value(raptor_perf, 'output_count')} |",
|
||||
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'average_latency_ms', 'ms')} | {perf_value(pimcomp_perf, 'throughput', 'samples/s')} | "
|
||||
f"{perf_value(pimcomp_perf, 'average_power_mw', 'mW')} | {perf_value(pimcomp_perf, 'average_energy_pj', 'pJ/it')} | {perf_value(pimcomp_perf, 'output_count')} |",
|
||||
"",
|
||||
]
|
||||
)
|
||||
else:
|
||||
lines.extend(
|
||||
[
|
||||
"| Compiler | Status | Latency (ms) | Avg power (mW) | Avg energy (pJ) |",
|
||||
"| Compiler | Status | Latency | Avg power | Avg energy |",
|
||||
"|---|---|---:|---:|---:|",
|
||||
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'latency_ms')} | "
|
||||
f"{perf_value(raptor_perf, 'average_power_mw')} | {perf_value(raptor_perf, 'average_energy_pj')} |",
|
||||
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'latency_ms')} | "
|
||||
f"{perf_value(pimcomp_perf, 'average_power_mw')} | {perf_value(pimcomp_perf, 'average_energy_pj')} |",
|
||||
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'latency_ms', 'ms')} | "
|
||||
f"{perf_value(raptor_perf, 'average_power_mw', 'mW')} | {perf_value(raptor_perf, 'average_energy_pj', 'pJ')} |",
|
||||
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'latency_ms', 'ms')} | "
|
||||
f"{perf_value(pimcomp_perf, 'average_power_mw', 'mW')} | {perf_value(pimcomp_perf, 'average_energy_pj', 'pJ')} |",
|
||||
"",
|
||||
]
|
||||
)
|
||||
@@ -1293,6 +1339,7 @@ def main():
|
||||
parser.add_argument("--mesh-cols", type=int)
|
||||
parser.add_argument("--pimsim-time-ms", type=int, default=1000)
|
||||
parser.add_argument("--pimsim-mode", choices=["latency", "throughput"], default="latency")
|
||||
parser.add_argument("--batch-size", type=int, default=128)
|
||||
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
|
||||
parser.add_argument("--pimcomp-model-name", help="Use a PIMCOMP built-in model name such as vgg16.")
|
||||
parser.add_argument(
|
||||
@@ -1316,6 +1363,11 @@ def main():
|
||||
help="Preserve the PIMCOMP side of an existing comparison report without rerunning it.",
|
||||
)
|
||||
parser.add_argument("--skip-pimsim-nn", action="store_true")
|
||||
parser.add_argument(
|
||||
"--no-fast",
|
||||
action="store_true",
|
||||
help="Disable fast pimsim-nn throughput convergence for authoritative experiments.",
|
||||
)
|
||||
parser.add_argument("--verbose-raptor-compile", action="store_true")
|
||||
parser.add_argument("--raptor-extra-arg", action="append", default=[])
|
||||
parser.add_argument(
|
||||
@@ -1328,6 +1380,10 @@ def main():
|
||||
parser.error("--reuse-pimcomp-dir and --reuse-pimcomp-report are mutually exclusive")
|
||||
if args.pimsim_time_ms <= 0:
|
||||
parser.error("--pimsim-time-ms must be positive")
|
||||
if args.batch_size <= 0:
|
||||
parser.error("--batch-size must be positive")
|
||||
if args.pimsim_mode == "throughput" and args.batch_size < 2:
|
||||
parser.error("throughput mode requires batch size greater than 1")
|
||||
if args.timeout_seconds < 0:
|
||||
parser.error("--timeout-seconds must be non-negative")
|
||||
if args.pimcomp_pipeline is None:
|
||||
@@ -1371,6 +1427,7 @@ def main():
|
||||
|
||||
runner_path: Path | None = None
|
||||
reference_dir: Path | None = None
|
||||
reference_dirs: list[Path] = []
|
||||
raptor_pim_dir: Path | None = None
|
||||
raptor_pimsim_dir: Path | None = None
|
||||
raptor_pass_timings: dict[str, float] = {}
|
||||
@@ -1506,6 +1563,33 @@ def main():
|
||||
"Reference outputs were skipped because the native runner or model inputs are not available.",
|
||||
)
|
||||
|
||||
input_batch = None
|
||||
raptor_input_bins: list[Path] = []
|
||||
pimcomp_input_bins: list[Path] = []
|
||||
if model_io is not None:
|
||||
batch_size = 1 if args.pimsim_mode == "latency" else args.batch_size
|
||||
input_batch = generate_input_batch(inputs_desc, runtime_inputs, batch_size, args.seed)
|
||||
write_input_batch_csv(out_dir / "inputs.csv", input_batch)
|
||||
raptor_input_bins = write_input_batch_binaries(input_batch, out_dir / "simulation/raptor_inputs")
|
||||
if args.pimsim_mode == "throughput":
|
||||
throughput_references = try_stage(
|
||||
failures,
|
||||
"Run throughput references",
|
||||
generate_reference_batch_outputs,
|
||||
runner_path,
|
||||
runner_path.parent,
|
||||
model_path,
|
||||
input_batch,
|
||||
steps,
|
||||
args,
|
||||
out_dir / "reference",
|
||||
) if runner_path is not None and runner_path.exists() else None
|
||||
if throughput_references is not None:
|
||||
reference_dirs = throughput_references
|
||||
reference_dir = out_dir / "reference"
|
||||
elif reference_dir is not None:
|
||||
reference_dirs = [reference_dir]
|
||||
|
||||
if not reuse_raptor and model_path.exists() and hardware["core_count"] > 0:
|
||||
compiled_raptor = try_stage(
|
||||
failures,
|
||||
@@ -1535,7 +1619,7 @@ def main():
|
||||
raptor_pim_dir / "config.json",
|
||||
runtime_inputs,
|
||||
)
|
||||
if wrote_inputs and reference_dir is not None and outputs_desc:
|
||||
if wrote_inputs and reference_dirs and outputs_desc:
|
||||
validation = try_stage(
|
||||
failures,
|
||||
"Functional Validation Raptor",
|
||||
@@ -1544,13 +1628,14 @@ def main():
|
||||
raptor_pim_dir,
|
||||
raptor_pim_dir / "config.json",
|
||||
out_dir / "simulation/out.bin",
|
||||
raptor_input_bins,
|
||||
outputs_desc,
|
||||
reference_dir,
|
||||
reference_dirs,
|
||||
steps,
|
||||
args,
|
||||
)
|
||||
raptor_validation = validation if validation is not None else failed_validation("Raptor validation failed")
|
||||
elif reference_dir is None:
|
||||
elif not reference_dirs:
|
||||
raptor_validation = skipped_validation("Reference outputs are not available")
|
||||
elif not outputs_desc:
|
||||
raptor_validation = skipped_validation("Output descriptors are not available")
|
||||
@@ -1616,7 +1701,14 @@ def main():
|
||||
"PIMCOMP functional export failed because model inputs are not available.",
|
||||
)
|
||||
|
||||
if not reuse_pimcomp and pimcomp_export_dir is not None and reference_dir is not None and outputs_desc:
|
||||
if input_batch is not None and pimcomp_export_dir is not None:
|
||||
pimcomp_input_bins = write_input_batch_binaries(
|
||||
input_batch,
|
||||
out_dir / "simulation/pimcomp_inputs",
|
||||
transform=flatten_pimcomp_input,
|
||||
)
|
||||
|
||||
if not reuse_pimcomp and pimcomp_export_dir is not None and reference_dirs and outputs_desc:
|
||||
validation = try_stage(
|
||||
failures,
|
||||
"Functional Validation PIMCOMP",
|
||||
@@ -1625,8 +1717,9 @@ def main():
|
||||
pimcomp_export_dir,
|
||||
pimcomp_export_dir / "config.json",
|
||||
out_dir / "simulation/pimcomp.out.bin",
|
||||
pimcomp_input_bins,
|
||||
outputs_desc,
|
||||
reference_dir,
|
||||
reference_dirs,
|
||||
steps,
|
||||
args,
|
||||
channel_last=True,
|
||||
@@ -1636,7 +1729,7 @@ def main():
|
||||
pass
|
||||
elif pimcomp_export_dir is None:
|
||||
pimcomp_validation = failed_validation("PIMCOMP functional export is not available")
|
||||
elif reference_dir is None:
|
||||
elif not reference_dirs:
|
||||
pimcomp_validation = failed_validation("Reference outputs are not available")
|
||||
else:
|
||||
pimcomp_validation = failed_validation("Output descriptors are not available")
|
||||
@@ -1673,7 +1766,7 @@ def main():
|
||||
raptor_pimsim_dir = try_stage(
|
||||
failures,
|
||||
"Export Raptor for pimsim-nn",
|
||||
export_raptor_latency_artifact,
|
||||
export_raptor_pimsim_artifact,
|
||||
raptor_pim_dir,
|
||||
out_dir / "raptor/pimsim_nn",
|
||||
)
|
||||
@@ -1777,6 +1870,8 @@ def main():
|
||||
"common_dir": str(common_dir),
|
||||
"reference_inputs": optional_path(common_dir / "inputs"),
|
||||
"reference_outputs": optional_path(reference_dir),
|
||||
"batch_inputs": optional_path(out_dir / "inputs.csv"),
|
||||
"batch_outputs": optional_path(out_dir / "simulation/out_iterations"),
|
||||
"reference_runner": optional_path(runner_path),
|
||||
"raptor_pim": optional_path(raptor_pim_dir),
|
||||
"raptor_pimsim_nn": optional_path(raptor_pimsim_dir),
|
||||
|
||||
@@ -108,10 +108,10 @@ def write_results_csv(
|
||||
"pimcomp_pipeline",
|
||||
"raptor_functional_validation",
|
||||
"pimcomp_functional_validation",
|
||||
"raptor_throughput_samples_s",
|
||||
"pimcomp_throughput_samples_s",
|
||||
"raptor_latency_ms",
|
||||
"pimcomp_latency_ms",
|
||||
"raptor_throughput_samples_s",
|
||||
"pimcomp_throughput_samples_s",
|
||||
"raptor_power_mw",
|
||||
"pimcomp_power_mw",
|
||||
"raptor_energy_pj",
|
||||
@@ -215,13 +215,13 @@ def write_results_csv(
|
||||
def performance_values(performance: dict) -> dict[str, float | None]:
|
||||
parsed = parse_pimsim_nn_metrics(performance.get("raw_output", ""))
|
||||
return {
|
||||
"throughput": performance.get("throughput") or parsed.get("throughput"),
|
||||
"latency": (
|
||||
performance.get("latency_ms")
|
||||
or performance.get("average_latency_ms")
|
||||
or parsed.get("latency_ms")
|
||||
or parsed.get("average_latency_ms")
|
||||
),
|
||||
"throughput": performance.get("throughput") or parsed.get("throughput"),
|
||||
"power": performance.get("average_power_mw") or parsed.get("average_power_mw"),
|
||||
"energy": performance.get("average_energy_pj") or parsed.get("average_energy_pj"),
|
||||
}
|
||||
@@ -261,7 +261,7 @@ def comparison_winner(mode: str, raptor: float, pimcomp: float) -> str:
|
||||
|
||||
|
||||
def format_value(value: float | None) -> str:
|
||||
return "NA" if value is None else f"{value:.6f}"
|
||||
return "NA" if value is None else f"{value:.2f}"
|
||||
|
||||
|
||||
def print_stage(title: str, color: str) -> None:
|
||||
@@ -323,7 +323,9 @@ def comparison_command(
|
||||
pipeline: int,
|
||||
pimcomp_pipeline: str,
|
||||
pimsim_time_ms: int,
|
||||
batch_size: int,
|
||||
timeout: float,
|
||||
fast: bool,
|
||||
reuse_raptor_report: Path | None = None,
|
||||
reuse_pimcomp_dir: Path | None = None,
|
||||
reuse_pimcomp_report: Path | None = None,
|
||||
@@ -351,6 +353,8 @@ def comparison_command(
|
||||
str(config),
|
||||
"--pimsim-mode",
|
||||
mode,
|
||||
"--batch-size",
|
||||
str(batch_size),
|
||||
*time_args,
|
||||
"--pimcomp-pipeline",
|
||||
pimcomp_pipeline,
|
||||
@@ -360,6 +364,7 @@ def comparison_command(
|
||||
"--timeout-seconds",
|
||||
str(timeout),
|
||||
"--fail-on-error",
|
||||
*([] if fast else ["--no-fast"]),
|
||||
*reuse_args,
|
||||
]
|
||||
|
||||
@@ -396,7 +401,9 @@ def comparison_command_for(
|
||||
spec.pipeline,
|
||||
spec.pimcomp_pipeline,
|
||||
args.pimsim_time_ms,
|
||||
args.batch_size,
|
||||
args.timeout_seconds,
|
||||
not args.no_fast,
|
||||
reuse_raptor_report=(
|
||||
report
|
||||
if args.only == "pimcomp"
|
||||
@@ -512,8 +519,14 @@ def main() -> int:
|
||||
parser.add_argument(
|
||||
"--pimsim-time-ms",
|
||||
type=int,
|
||||
default=100,
|
||||
help="throughput pimsim-nn horizon in ms (default: 100).",
|
||||
default=1000,
|
||||
help="throughput pimsim-nn convergence deadline in ms (default: 1000).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=128,
|
||||
help="functional throughput batch size (default: 128).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout-seconds",
|
||||
@@ -525,8 +538,8 @@ def main() -> int:
|
||||
"-j",
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=os.cpu_count() or 1,
|
||||
help="Number of comparisons to run in parallel (default: all available CPUs).",
|
||||
default=4,
|
||||
help="Number of comparisons to run in parallel (default: 4).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean",
|
||||
@@ -534,6 +547,11 @@ def main() -> int:
|
||||
help="Remove generated comparison artifacts and result summaries, then exit.",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print commands without modifying files.")
|
||||
parser.add_argument(
|
||||
"--no-fast",
|
||||
action="store_true",
|
||||
help="Disable fast pimsim-nn throughput convergence for authoritative experiments.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
|
||||
@@ -547,6 +565,8 @@ def main() -> int:
|
||||
parser.error("--jobs must be at least 1")
|
||||
if args.pimsim_time_ms <= 0:
|
||||
parser.error("--pimsim-time-ms must be positive")
|
||||
if args.batch_size <= 0:
|
||||
parser.error("--batch-size must be positive")
|
||||
if args.timeout_seconds < 0:
|
||||
parser.error("--timeout-seconds must be non-negative")
|
||||
comparisons_by_arch: dict[str, tuple[tuple[str, int, str], ...]] = {}
|
||||
|
||||
@@ -21,7 +21,7 @@ if sys.version_info < (3, 10):
|
||||
"Run it with a newer interpreter, for example your project venv Python."
|
||||
)
|
||||
|
||||
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_to_memory_bin
|
||||
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_binary, write_inputs_to_memory_bin
|
||||
from raptor_validation.validate_one import (
|
||||
MODE_COMPILE_ONLY,
|
||||
build_dump_ranges,
|
||||
@@ -137,6 +137,8 @@ def run_local_reference_and_simulator(args, network_dir: Path, network_onnx_path
|
||||
|
||||
tensor = np.loadtxt(paths["input_csv"], delimiter=",", dtype=np.float32).reshape(1, 3, 640, 640)
|
||||
write_inputs_to_memory_bin(paths["raptor_pim"] / "memory.bin", paths["raptor_pim"] / "config.json", [tensor])
|
||||
input_bin = paths["sim_dir"] / "input.bin"
|
||||
write_inputs_binary(input_bin, [tensor])
|
||||
|
||||
dump_ranges = build_dump_ranges(paths["raptor_pim"] / "config.json", output_descriptors)
|
||||
run_pim_simulator(
|
||||
@@ -145,6 +147,7 @@ def run_local_reference_and_simulator(args, network_dir: Path, network_onnx_path
|
||||
paths["sim_bin"],
|
||||
dump_ranges,
|
||||
timeout_sec=args.command_timeout_seconds,
|
||||
input_paths=[input_bin],
|
||||
)
|
||||
return paths, output_descriptors[0]
|
||||
|
||||
|
||||
@@ -272,6 +272,7 @@ def remote_case_paths(args, case_name: str):
|
||||
"input_csv": root / "real_image_validation" / "inputs" / f"{case_name}.csv",
|
||||
"ref_dir": root / "real_image_validation" / "reference" / case_name,
|
||||
"sim_dir": root / "real_image_validation" / "simulation" / case_name,
|
||||
"sim_input": root / "real_image_validation" / "simulation" / case_name / "input.bin",
|
||||
"sim_bin": root / "real_image_validation" / "simulation" / case_name / "out.bin",
|
||||
}
|
||||
|
||||
@@ -310,8 +311,10 @@ import numpy as np
|
||||
from pathlib import Path
|
||||
input_csv = Path({json.dumps(str(paths["input_csv"]))})
|
||||
pim_dir = Path({json.dumps(str(paths["raptor_pim"]))})
|
||||
input_bin = Path({json.dumps(str(paths["sim_input"]))})
|
||||
config = json.loads((pim_dir / "config.json").read_text())
|
||||
tensor = np.loadtxt(input_csv, delimiter=",", dtype=np.float32).reshape(1, 3, 640, 640)
|
||||
input_bin.write_bytes(tensor.tobytes(order="C"))
|
||||
with open(pim_dir / "memory.bin", "r+b") as f:
|
||||
f.seek(config["inputs_addresses"][0])
|
||||
f.write(tensor.tobytes(order="C"))
|
||||
@@ -327,7 +330,8 @@ PY
|
||||
f"export PATH=$HOME/.cargo/bin:$PATH && "
|
||||
f"cd {quoted_project}/backend-simulators/pim/pim-simulator && "
|
||||
f"cargo run --no-default-features --release --package pim-simulator --bin pim-simulator -- "
|
||||
f"-f {quoted_pim} -o {quoted_sim_bin} -d {dump_range}"
|
||||
f"-f {quoted_pim} -o {quoted_sim_bin} -d {dump_range} "
|
||||
f"--mode latency --input {shlex.quote(str(paths['sim_input']))}"
|
||||
)
|
||||
remote_bash(args.ssh_key, args.remote_host, sim_command)
|
||||
return paths
|
||||
|
||||
Reference in New Issue
Block a user