even better more structured throughput scheduling
Validate Operations / validate-operations (push) Has been cancelled

plot adjustments
validation adjustments
This commit is contained in:
NiccoloN
2026-10-27 17:46:46 +01:00
parent 49d6fb42cf
commit a77edafcbf
46 changed files with 3265 additions and 1043 deletions
+38 -12
View File
@@ -35,7 +35,7 @@ struct Notice { size_t arrival; int64_t source; };
struct Wait { size_t end; int64_t expected; };
struct Core {
size_t entry, tail, firstNetwork, lastNetwork;
Cost pending = 0, work = 0, instructions = 0;
Cost pending = 0, work = 0, instructions = 0, directWork = 0, constantWork = 0;
std::array<Cost, 7> service{};
std::array<Cost, 6> traceWork{};
llvm::SHA256 requests{}, blockingOrder{};
@@ -80,10 +80,13 @@ public:
stateWords = state.size();
return true;
}
// Snapshots are immutable; do not retain geometric construction capacity.
state.shrink_to_fit();
generations.shrink_to_fit();
size_t added = add(mul(add(state.capacity(), generations.capacity()), sizeof(uint64_t)), sizeof(Snapshot));
if (add(bytes, added) > 16 * 1024 * 1024) {
if (add(bytes, added) > 64 * 1024 * 1024) {
exhausted = true;
evidence.reason = "periodic-state history exceeds 16 MiB";
evidence.reason = "periodic-state history exceeds 64 MiB";
return false;
}
bytes += added;
@@ -141,7 +144,7 @@ PimQueuePolicyResult estimateQueuedPeriod(const std::vector<Event> &events,
pending.push({sub(time, service[node]), node, memoryAccess[node]});
};
for (size_t node = 0; node < nodes.size(); ++node) schedule(node);
const size_t maximumRounds = verifyState ? 256 : 64;
const size_t maximumRounds = verifyState ? 1024 : 64;
std::vector<Cost> roundEnd(maximumRounds);
std::vector<size_t> roundCount(maximumRounds);
std::map<int64_t, std::vector<Cost>> queueWait;
@@ -462,7 +465,7 @@ private:
// ROB retirement needs another edge; matrix also has AG/commit registers.
return issue ? mul(matrix ? 3 : 1, target.processorPeriodNs) : 0;
}
void memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes);
void memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes, bool constant = false);
bool communication(int64_t core, uint32_t pc, Opcode op,
int32_t peer, int32_t reg, int32_t count, Cost bytes);
};
@@ -518,14 +521,17 @@ bool Analysis::communication(int64_t core, uint32_t pc, Opcode op,
target.getLinkServiceNs(target.controlMessageBytes, core, peer, arithmetic));
receives[{peer, core}].push_back({complete, permission.arrival, permission.begin, bytes});
}
cores.at(core).work = add(cores.at(core).work, add(local(bytes, op == Opcode::recv), retirement()));
cores.at(core).service[2] = add(cores.at(core).service[2], add(local(bytes, op == Opcode::recv), retirement()));
opcodeService[static_cast<unsigned>(op)] = add(opcodeService[static_cast<unsigned>(op)], add(local(bytes, op == Opcode::recv), retirement()));
Cost endpointWork = add(local(bytes, op == Opcode::recv), retirement());
auto &state = cores.at(core);
state.work = add(state.work, endpointWork);
state.service[2] = add(state.service[2], endpointWork);
state.directWork = add(state.directWork, endpointWork);
opcodeService[static_cast<unsigned>(op)] = add(opcodeService[static_cast<unsigned>(op)], endpointWork);
}
return true;
}
void Analysis::memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes) {
void Analysis::memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes, bool constant) {
Core &state = cores.at(core);
recordBlocking(core, write ? Opcode::st : Opcode::ld, bytes);
if (options.workBreakdown) {
@@ -548,6 +554,7 @@ void Analysis::memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes) {
Cost cost = add(add(add(write ? local(bytes, false) : 0, demand.serviceNs), completion),
add(target.getSharedMemoryLinkServiceNs(write ? bytes : target.controlMessageBytes, core, false, arithmetic),
target.getSharedMemoryLinkServiceNs(!write && target.sharedMemoryReadReturnsPayload ? bytes : 0, core, true, arithmetic)));
if (constant && !write) state.constantWork = add(state.constantWork, cost);
state.service[write ? 4 : 3] = add(state.service[write ? 4 : 3], cost);
opcodeService[static_cast<unsigned>(write ? Opcode::st : Opcode::ld)] = add(opcodeService[static_cast<unsigned>(write ? Opcode::st : Opcode::ld)], cost);
if (memoryQueue) {
@@ -618,6 +625,8 @@ bool Analysis::readCore(int64_t core, llvm::StringRef data, llvm::ArrayRef<int64
int32_t b = llvm::support::endian::read32le(record + 12);
int32_t length = llvm::support::endian::read32le(record + 16);
++state.instructions;
// Pimsim-nn drops vsoftmax before fetch: no issue, memory, or execution service.
if (op == Opcode::vsoftmax) continue;
if (op <= Opcode::setbw) {
if (op == Opcode::sld) { error = "scalar memory loads are unsupported by this diagnostic"; return false; }
if (op == Opcode::setbw) {
@@ -635,7 +644,7 @@ bool Analysis::readCore(int64_t core, llvm::StringRef data, llvm::ArrayRef<int64
opcodeService[static_cast<unsigned>(op)] = add(opcodeService[static_cast<unsigned>(op)], cost);
continue;
}
if (length < 0 || op > Opcode::sync || op == Opcode::vsoftmax) {
if (length < 0 || op > Opcode::sync) {
error = "unsupported instruction opcode or negative transfer length"; return false;
}
Cost bytes = length;
@@ -781,7 +790,7 @@ bool Analysis::readExecution(int64_t core, const std::vector<PimExecutionEvent>
}
index = found->second.end - 1;
} else if (item.kind == PimExecutionEvent::Load || item.kind == PimExecutionEvent::Store) {
memoryAccess(core, position, item.kind == PimExecutionEvent::Store, item.amount);
memoryAccess(core, position, item.kind == PimExecutionEvent::Store, item.amount, item.constant);
} else {
for (int64_t value : {item.peer, item.reg, item.count})
if (value < 0 || value > std::numeric_limits<int32_t>::max()) {
@@ -839,7 +848,7 @@ EvaluatedAnalysis Analysis::evaluate() {
if (cores.empty()) result.error = "artifact has no active repeating instruction stream";
else if (!connect()) result.error = error;
if (!result.error.empty()) { result.cost.reason = result.error; return result; }
if (options.detailed) {
if (options.detailed || options.searchGuidance) {
auto recurrence = analyzePeriodicDependencies(events.size(), edges, result.error);
if (mlir::failed(recurrence)) { result.cost.reason = result.error; return result; }
result.recurrence = std::move(*recurrence);
@@ -859,6 +868,23 @@ EvaluatedAnalysis Analysis::evaluate() {
std::chrono::steady_clock::now() - start).count();
result.cost = queue.cost();
}
if (options.searchGuidance) {
result.cost.processorTaskWorkNs.resize(target.processorCount);
result.cost.processorCriticalDelayNs.resize(target.processorCount);
for (const auto &[core, state] : cores)
result.cost.processorTaskWorkNs[core] = add(state.constantWork,
add(add(state.service[0], state.service[1]),
add(sub(state.service[2], state.directWork), state.service[5])));
Cost distance = 0;
for (size_t index : result.recurrence->criticalCycle)
distance = add(distance, edges[index].distance);
if (distance)
for (size_t index : result.recurrence->criticalCycle) {
const auto &edge = edges[index];
result.cost.processorCriticalDelayNs[events[edge.source].core]
+= static_cast<double>(edge.delayNs) / distance;
}
}
return result;
}
+3
View File
@@ -89,8 +89,11 @@ struct PimPeriodicOptions {
bool instructionIssue = true;
bool scalarIssue = true;
bool ignoreLocalCopies = false;
// Spatial ablation mirrors codegen's omission of SYNC/WAIT instructions.
bool omitSynchronization = false;
bool workBreakdown = false;
bool measureTime = false;
bool searchGuidance = false;
// Optional ordered work/IO extraction for timing validation; no execution.
PimExecutionProgram *executionTrace = nullptr;
// Prove recurrence of the modeled state. Instruction diagnostics retain
+94 -40
View File
@@ -1,6 +1,8 @@
#include "PimPipelineSelection.hpp"
#include "PimCompilerOptions.hpp"
#include "PimCompilerUtils.hpp"
#include "PimSpatialCost.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/PipelineScheduling.hpp"
#include "src/Accelerators/PIM/Common/Support/DebugDump.hpp"
#include "src/Accelerators/PIM/Common/Support/FileSystemUtils.hpp"
#include "src/Accelerators/PIM/Common/Support/ReportUtils.hpp"
@@ -108,83 +110,133 @@ struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, Op
}
LogicalResult evaluateSpatialCandidates(ModuleOp checkpoint,
const PreparedSpatialSchedules &prepared, std::vector<WholeScheduleCost> &costs,
PreparedSpatialSchedules &prepared, std::vector<WholeScheduleCost> &costs,
llvm::json::Object &report) {
llvm::json::Array summaries;
double realization = 0, extraction = 0, expansion = 0, queue = 0;
std::string errors;
for (size_t i = 0; i < costs.size(); ++i) {
llvm::json::Object summary{{"index", static_cast<int64_t>(i)}};
double realization = 0, extraction = 0, expansion = 0, queue = 0, evaluation = 0;
auto evaluate = [&](const MergeScheduleResult &schedule) {
size_t index = summaries.size();
llvm::json::Object summary{{"index", static_cast<int64_t>(index)}};
OwningOpRef<ModuleOp> candidate;
WholeScheduleCost cost;
std::string scheduleReport;
{
ScopedCompilerOutputSuppression silence;
if (failed(realizeCandidate(checkpoint, prepared.candidates[i], candidate, scheduleReport, summary)))
costs[i].reason = "Spatial schedule realization failed verification";
else costs[i] = spatialCost(*candidate, summary);
if (failed(realizeCandidate(checkpoint, schedule, candidate, scheduleReport, summary)))
cost.reason = "Spatial schedule realization failed verification";
else cost = spatialCost(*candidate, summary);
}
realization += summary.getNumber("realization_seconds").value_or(0);
extraction += costs[i].timing.extractionSeconds;
expansion += costs[i].timing.expansionSeconds;
queue += costs[i].timing.evaluationSeconds;
bool supported = costs[i].loweringSupport == StaticLoweringSupport::Supported;
auto convergence = costs[i].convergenceProved();
auto support = costs[i].loweringSupported();
bool converged = convergence == true;
bool safe = costs[i].eligible();
summary["cost_available"] = costs[i].available();
evaluation += summary.getNumber("spatial_cost_seconds").value_or(0);
extraction += cost.timing.extractionSeconds;
expansion += cost.timing.expansionSeconds;
queue += cost.timing.evaluationSeconds;
auto convergence = cost.convergenceProved(), support = cost.loweringSupported();
summary["cost_available"] = cost.available();
summary["convergence_proved"] = convergence
? llvm::json::Value(*convergence) : llvm::json::Value(nullptr);
summary["lowering_supported"] = support
? llvm::json::Value(*support) : llvm::json::Value(nullptr);
summary["eligible"] = safe;
if (costs[i].primary) summary["interval_ns"] = *costs[i].primary;
if (costs[i].alternative) summary["alternative_tie_interval_ns"] = *costs[i].alternative;
if (!safe) {
std::string property;
if (!costs[i].available()) property += "unavailable timing; ";
if (!supported) property += "unsupported or unknown static lowering; ";
if (!converged) property += "unproved modeled-state convergence; ";
if (property.empty()) property = "unavailable selection margin";
summary["reason"] = property + ": " + costs[i].reason;
errors += "\ncandidate " + std::to_string(i) + ": " + property + ": " + costs[i].reason;
summary["eligible"] = cost.eligible();
if (cost.primary) summary["interval_ns"] = *cost.primary;
if (cost.alternative) summary["alternative_tie_interval_ns"] = *cost.alternative;
llvm::json::Array budgets, witness;
std::vector<size_t> cores(stages);
std::vector<double> pressure(stages);
for (size_t core = 0; core < schedule.processorStages.size(); ++core) {
size_t stage = schedule.processorStages[core];
++cores[stage];
if (core < cost.processorCriticalDelayNs.size())
pressure[stage] += cost.processorCriticalDelayNs[core];
}
for (size_t stage = 0; stage < stages; ++stage) {
budgets.push_back(static_cast<int64_t>(cores[stage]));
witness.push_back(pressure[stage]);
}
summary["stage_core_budgets"] = std::move(budgets);
summary["recurrence_stage_contributions_ns"] = std::move(witness);
llvm::json::Array runs;
for (size_t begin = 0; begin < prepared.graph.nodes.size();) {
auto stageFor = [&](size_t task) {
return schedule.processorStages[schedule.computeToCpuMap.lookup(
prepared.graph.nodes[task].instance)];
};
size_t end = begin + 1, stage = stageFor(begin);
while (end < prepared.graph.nodes.size() && stageFor(end) == stage) ++end;
runs.push_back(llvm::json::Array{static_cast<int64_t>(begin),
static_cast<int64_t>(end - begin), static_cast<int64_t>(stage)});
begin = end;
}
summary["task_stage_runs"] = std::move(runs);
if (!cost.eligible()) summary["reason"] = cost.reason;
summaries.push_back(std::move(summary));
}
report["candidates"] = std::move(summaries);
return cost;
};
std::string error;
auto start = Clock::now();
auto search = optimizePipelineSchedule(prepared.graph, prepared.candidates.front(),
stages, target, evaluate, error);
report["search_seconds"] = elapsed(start);
report["search_refinement_seconds"] = std::max(0.0, elapsed(start) - realization - evaluation);
report["cost_evaluation_seconds"] = evaluation;
report["candidate_realization_seconds"] = realization;
report["event_extraction_seconds"] = extraction;
report["event_expansion_seconds"] = expansion;
report["queue_seconds"] = queue;
if (!errors.empty()) {
report["candidates"] = std::move(summaries);
if (failed(search)) {
report["status"] = "incomplete-costs";
report["reason"] = errors;
checkpoint.emitError("Spatial pipeline cannot compare the complete candidate set:") << errors;
report["reason"] = error;
checkpoint.emitError("Spatial pipeline cannot compare the complete candidate set: ") << error;
return failure();
}
prepared.candidates = std::move(search->schedules);
costs = std::move(search->costs);
bool single = costs.empty();
if (single) {
report["candidates"] = llvm::json::Array{llvm::json::Object{{"index", 0}}};
} else {
auto *entries = report.getArray("candidates");
for (size_t i = 0; i < entries->size(); ++i)
(*entries)[i].getAsObject()->insert({"parent", static_cast<int64_t>(search->parents[i])});
}
report["generated_candidates"] = static_cast<int64_t>(prepared.candidates.size());
report["single_candidate_analysis_skipped"] = single;
report["search_rounds"] = static_cast<int64_t>(search->rounds);
report["packing_attempts"] = static_cast<int64_t>(search->packingAttempts);
report["considered_assignments"] = static_cast<int64_t>(search->consideredAssignments);
report["search_termination"] = search->termination;
report["queue_evaluation_budget"] = static_cast<int64_t>(kPipelineQueueEvaluationBudget);
report["beam_width"] = static_cast<int64_t>(kPipelineBeamWidth);
return success();
}
void runOnOperation() override {
ModuleOp checkpoint = getOperation();
auto start = Clock::now();
auto prepared = prepareSpatialSchedules(checkpoint, target, stages, /*collectCandidates=*/true);
auto prepared = prepareSpatialSchedules(checkpoint, target, stages);
if (failed(prepared)) { signalPassFailure(); return; }
if (prepared->candidates.empty()) {
checkpoint.emitError("Spatial pipeline requires at least one candidate schedule");
signalPassFailure(); return;
}
bool single = prepared->candidates.size() == 1;
llvm::json::Object report{{"strategy", "spatial-queue-aware"}, {"static_analysis", true},
{"candidate_generation_seconds", elapsed(start)},
{"generated_candidates", static_cast<int64_t>(prepared->candidates.size())},
{"single_candidate_analysis_skipped", single}, {"selection_margin", kPipelineSelectionMargin},
{"selection_margin", kPipelineSelectionMargin},
{"prediction_error_bound_available", false}, {"temporary_disk_bytes", 0}};
std::vector<WholeScheduleCost> costs(prepared->candidates.size());
if (!single && failed(evaluateSpatialCandidates(checkpoint, *prepared, costs, report))) {
if (pimDisableSynchronization) {
report["synchronization_ablation"] = true;
report["cost_model_program"] = "synchronized-reference";
llvm::errs() << "Synchronization ablation: keep the normal selected schedule; "
"report zero-cost SYNC/WAIT separately and omit them in code generation. "
"Buffer reuse and restart correctness are not guaranteed.\n";
}
std::vector<WholeScheduleCost> costs;
if (failed(evaluateSpatialCandidates(checkpoint, *prepared, costs, report))) {
writeSelectionReport(std::move(report)); signalPassFailure(); return;
}
if (single) report["candidates"] = llvm::json::Array{llvm::json::Object{{"index", 0}}};
bool single = costs.empty();
OwningOpRef<ModuleOp> winner;
std::string scheduleReport;
llvm::json::Array attempts;
@@ -204,6 +256,8 @@ struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, Op
summary["reason"] = "selected Spatial schedule failed deterministic realization";
summary["status"] = "realization-failed";
} else {
if (pimDisableSynchronization)
summary["synchronization_ablation_cost"] = analyzePimSpatialSynchronizationAblation(*candidate);
++lowered;
result = lowerPimPipelineCandidate(*candidate, resources, summary);
}
@@ -264,7 +318,7 @@ struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, Op
report["reason"] = reason;
report["finalization_seconds"] = elapsed(finalize);
report["total_seconds"] = elapsed(start);
llvm::errs() << "Spatial queue-aware pipeline: candidate " << *selected << " of " << costs.size()
llvm::errs() << "Spatial queue-aware pipeline: candidate " << *selected << " of " << prepared->candidates.size()
<< "; " << reason << "\n";
writeSelectionReport(std::move(report));
}
+42 -12
View File
@@ -47,13 +47,15 @@ class Extraction {
public:
const PimInstructionTiming &target;
bool diagnostics;
bool omitSynchronization;
PimExecutionProgram program;
std::string error, loweringReason;
StaticLoweringSupport loweringSupport = StaticLoweringSupport::Supported;
std::map<std::string, int64_t> gaps;
int64_t visited = 0, constantLoads = 0;
explicit Extraction(const PimInstructionTiming &target, bool diagnostics)
: target(target), diagnostics(diagnostics), program(target.processorCount) {}
explicit Extraction(const PimInstructionTiming &target, bool diagnostics, bool omitSynchronization)
: target(target), diagnostics(diagnostics), omitSynchronization(omitSynchronization),
program(target.processorCount) {}
bool build(ModuleOp module);
private:
llvm::SmallDenseSet<int64_t, 16> assignedCores;
@@ -135,7 +137,7 @@ Residence Extraction::storage(Value value) {
return Residence::Host;
}
if (isa_and_nonnull<SpatVMMOp, SpatVAddOp, SpatVSubOp, SpatVMulOp,
SpatVMaxOp, SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp,
SpatVMaxOp, SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp, SpatSoftmaxOp,
SpatChannelReceiveOp, SpatHostWaitLoadOp, SpatScheduledCompute,
SpatScheduledComputeBatch, SpatConcatOp, tensor::EmptyOp,
tensor::InsertSliceOp, tensor::PadOp, linalg::TransposeOp, scf::ForOp>(value.getDefiningOp()))
@@ -246,10 +248,13 @@ bool Extraction::operation(Operation &op, Sequence &sequence) {
return *result;
};
if (auto sync = dyn_cast<SpatSyncOp>(op)) {
sequence.push_back({Event::Signal, 0, index(sync.getTargetCoreId()), index(sync.getEventRegister())});
if (!omitSynchronization)
sequence.push_back({Event::Signal, 0, index(sync.getTargetCoreId()), index(sync.getEventRegister())});
} else if (auto wait = dyn_cast<SpatWaitOp>(op)) {
int64_t count = index(wait.getWaitValue());
if (count) sequence.push_back({Event::Wait, 0, 0, index(wait.getEventRegister()), count});
if (!omitSynchronization) {
int64_t count = index(wait.getWaitValue());
if (count) sequence.push_back({Event::Wait, 0, 0, index(wait.getEventRegister()), count});
}
} else if (auto send = dyn_cast<SpatChannelSendOp>(op)) {
materialize(send.getInput(), sequence);
sequence.push_back({Event::Send, bytes(send.getInput().getType()), index(send.getTargetCoreId())});
@@ -261,10 +266,13 @@ bool Extraction::operation(Operation &op, Sequence &sequence) {
sequence.push_back({Event::Repeat, static_cast<uint64_t>(plan->second.getRunCount()),
0, 0, 0, {{Event::Store, ceil(mul(plan->second.getRunElements(), target.computeBitwidth), 8)}}});
else sequence.push_back({Event::Store, bytes(store.getInput().getType())});
sequence.push_back({Event::Signal, 0, index(store.getTargetCoreId()), index(store.getEventRegister())});
if (!omitSynchronization)
sequence.push_back({Event::Signal, 0, index(store.getTargetCoreId()), index(store.getEventRegister())});
} else if (auto load = dyn_cast<SpatHostWaitLoadOp>(op)) {
int64_t count = index(load.getWaitValue());
if (count) sequence.push_back({Event::Wait, 0, 0, index(load.getEventRegister()), count});
if (!omitSynchronization) {
int64_t count = index(load.getWaitValue());
if (count) sequence.push_back({Event::Wait, 0, 0, index(load.getEventRegister()), count});
}
sequence.push_back({Event::Load, bytes(load.getOutput().getType())});
} else if (auto vmm = dyn_cast<SpatVMMOp>(op)) {
materialize(vmm.getInput(), sequence);
@@ -284,6 +292,9 @@ bool Extraction::operation(Operation &op, Sequence &sequence) {
pim_binary::kMatrixWeightPrecisionBits, weight.getDimSize(1) / target.matrixColumns,
ceil(target.computeBitwidth, 8));
work(sequence, getPimMatrixWorkNs(target, target.computeBitwidth, output), Event::Matrix);
} else if (auto softmax = dyn_cast<SpatSoftmaxOp>(op)) {
materialize(softmax.getInput(), sequence);
// Pimsim-nn omits vsoftmax itself. Keep any preceding input materialization.
} else if (isa<SpatVAddOp, SpatVSubOp, SpatVMulOp, SpatVMaxOp,
SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp>(op)) {
for (Value operand : op.getOperands()) materialize(operand, sequence);
@@ -464,7 +475,7 @@ PimPeriodicEstimate onnx_mlir::evaluatePimSpatialSchedule(ModuleOp module,
if (!target.hasPeriodicTransferTiming() || !target.computeBitwidth)
return unavailableCost("unsupported abstract timing contract");
auto start = std::chrono::steady_clock::now();
Extraction extraction(target, options.workBreakdown);
Extraction extraction(target, options.workBreakdown, options.omitSynchronization);
bool available = extraction.build(module);
double extractionMs = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - start).count();
@@ -587,8 +598,28 @@ llvm::json::Object onnx_mlir::analyzePimSpatialSchedule(ModuleOp module,
return evaluatePimSpatialSchedule(module, target, options).report;
}
llvm::json::Object onnx_mlir::analyzePimSpatialSynchronizationAblation(ModuleOp module) {
auto timing = getPimInstructionTiming();
auto contract = getPimPeriodicTimingContract();
if (!contract) return unavailableCost(llvm::toString(contract.takeError())).report;
timing.matrixCellBits = *contract;
PimPeriodicOptions options;
options.omitSynchronization = true;
options.verifyPeriodicState = true;
options.measureTime = true;
auto result = evaluatePimSpatialSchedule(module, timing, options);
result.report["diagnostic_only"] = true;
result.report["used_for_selection"] = false;
result.report["cost_model_program"] = "synchronization-omitted";
return std::move(result.report);
}
void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target,
PimInstructionTiming timing) {
// The stage-zero entry barrier joins the previous generation's work.
target.estimatePipelineDependencySpan = [](llvm::ArrayRef<Cost> spans) {
return spans.empty() ? Cost{0} : spans.front();
};
auto contract = getPimPeriodicTimingContract();
if (contract) timing.matrixCellBits = *contract;
auto hardware = std::make_shared<const PimInstructionTiming>(std::move(timing));
@@ -601,8 +632,6 @@ void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target,
if (!contract) target.wholeScheduleCostUnavailable = llvm::toString(contract.takeError());
if (!hardware->hasPeriodicTransferTiming())
target.wholeScheduleCostUnavailable = "target has no supported queue timing contract";
if (pimDisableSynchronization)
target.wholeScheduleCostUnavailable = "synchronization ablation has no periodic queue timing contract";
if (!target.wholeScheduleCostUnavailable.empty()) return;
target.estimateRealizedScheduleCost = [hardware](mlir::ModuleOp module,
llvm::json::Object &summary) {
@@ -610,6 +639,7 @@ void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target,
options.workBreakdown = pimPeriodicDiagnostics;
options.measureTime = true;
options.verifyPeriodicState = true;
options.searchGuidance = true;
auto estimate = evaluatePimSpatialSchedule(module, *hardware, options);
auto cost = std::move(estimate.cost);
summary = std::move(estimate.report);
+1
View File
@@ -12,5 +12,6 @@ PimPeriodicEstimate evaluatePimSpatialSchedule(mlir::ModuleOp module,
const PimInstructionTiming &target, PimPeriodicOptions options = {});
llvm::json::Object analyzePimSpatialSchedule(mlir::ModuleOp module,
const PimInstructionTiming &target, PimPeriodicOptions options = {});
llvm::json::Object analyzePimSpatialSynchronizationAblation(mlir::ModuleOp module);
} // namespace onnx_mlir
@@ -65,7 +65,7 @@ struct ScheduleAndRealizeSpatialPass final
signalPassFailure();
return;
}
auto prepared = prepareSpatialSchedules(moduleOp, target, pipelineStages, false);
auto prepared = prepareSpatialSchedules(moduleOp, target, pipelineStages);
if (failed(prepared) || failed(realizeSpatialSchedule(moduleOp,
prepared->candidates.front(), target, pipelineStages, exportStage,
periodicDiagnostics ? &prepared->graph : nullptr)))
@@ -83,7 +83,7 @@ private:
} // namespace
FailureOr<PreparedSpatialSchedules> prepareSpatialSchedules(ModuleOp moduleOp,
const SchedulingTarget &target, size_t pipelineStages, bool collectCandidates) {
const SchedulingTarget &target, size_t pipelineStages) {
if (!hasValidTarget(target) || !PipelineCoreLayout(target.processorCount, pipelineStages).isValid()
|| (pipelineStages > 1 && !target.synchronizationRegisterCount))
return moduleOp.emitError("Spatial scheduling requires an explicit valid target and pipeline layout"), failure();
@@ -122,7 +122,7 @@ FailureOr<PreparedSpatialSchedules> prepareSpatialSchedules(ModuleOp moduleOp,
break;
}
if (succeeded(runPipelineScheduler(
scheduledGraph, schedule, pipelineStages, target, pipelineError, collectCandidates ? &prepared.candidates : nullptr)))
scheduledGraph, schedule, pipelineStages, target, pipelineError)))
break;
moduleOp.emitError() << pipelineError;
return failure();
@@ -14,7 +14,7 @@ struct PreparedSpatialSchedules {
// Success always includes at least one candidate, including an empty compute graph.
mlir::FailureOr<PreparedSpatialSchedules> prepareSpatialSchedules(
mlir::ModuleOp module, const SchedulingTarget &target,
size_t pipelineStages, bool collectCandidates);
size_t pipelineStages);
mlir::LogicalResult realizeSpatialSchedule(mlir::ModuleOp module,
const MergeScheduleResult &schedule, const SchedulingTarget &target,
size_t pipelineStages, SpatialDataflowExportStage exportStage,
@@ -7,6 +7,7 @@
#include <algorithm>
#include <cstddef>
#include <functional>
#include <numeric>
#include <optional>
#include <string>
@@ -32,25 +33,6 @@ inline bool isBetterStagePlacement(const StagePlacementScore &candidate,
return candidate < current;
}
inline bool isBetterPipelineSchedule(const ScheduleCostSummary &candidate,
const ScheduleCostSummary &current) {
auto score = [](const ScheduleCostSummary &result) {
Cost stage = 0;
for (Cost span : result.stageSpanNs) stage = std::max(stage, span);
Cost memory = checkedAdd(result.baselineMemory.serviceNs, result.boundaryMemory.serviceNs);
return std::make_tuple(std::max(stage, memory), stage, memory);
};
auto currentScore = score(current);
auto candidateScore = score(candidate);
// Stage spans omit contention: do not exchange a larger compute regression
// for a smaller memory saving, even when memory remains the largest resource.
Cost currentDemand = checkedAdd(std::get<1>(currentScore), std::get<2>(currentScore));
Cost candidateDemand = checkedAdd(std::get<1>(candidateScore), std::get<2>(candidateScore));
Cost currentCoreDemand = checkedAdd(static_cast<Cost>(current.maximumCoreWorkNs), std::get<2>(currentScore));
Cost candidateCoreDemand = checkedAdd(static_cast<Cost>(candidate.maximumCoreWorkNs), std::get<2>(candidateScore));
return candidateScore < currentScore && candidateDemand <= currentDemand
&& candidateCoreDemand <= currentCoreDemand;
}
struct PipelineStageRange {
size_t begin;
@@ -119,13 +101,34 @@ private:
std::vector<size_t> physicalCoreStages;
};
// Produces a complete physical schedule directly from the shared compute graph.
mlir::LogicalResult runPipelineScheduler(const ComputeGraph& graph,
MergeScheduleResult& schedule,
size_t pipelineStages,
const SchedulingTarget& physicalTarget,
std::string& error,
std::vector<MergeScheduleResult> *candidates = nullptr);
// One initialization procedure, also used for intermediate emission.
mlir::LogicalResult runPipelineScheduler(const ComputeGraph &graph,
MergeScheduleResult &schedule, size_t pipelineStages,
const SchedulingTarget &target, std::string &error);
inline constexpr size_t kPipelineQueueEvaluationBudget = 32;
inline constexpr size_t kPipelineBeamWidth = 2;
struct PipelineSearchResult {
std::vector<MergeScheduleResult> schedules;
std::vector<WholeScheduleCost> costs;
std::vector<size_t> parents;
size_t rounds = 0, packingAttempts = 0, consideredAssignments = 0;
std::string termination;
};
// The callback realizes an independent Spatial clone; it must not mutate graph IR.
using PipelineCostQuery = std::function<WholeScheduleCost(const MergeScheduleResult &)>;
mlir::FailureOr<PipelineSearchResult> optimizePipelineSchedule(
const ComputeGraph &graph, const MergeScheduleResult &initial,
size_t pipelineStages, const SchedulingTarget &target,
const PipelineCostQuery &evaluate, std::string &error,
size_t evaluationBudget = kPipelineQueueEvaluationBudget);
// Constructive capacity/topology packing shared with the small-instance oracle.
mlir::LogicalResult packPipelineSchedule(const ComputeGraph &graph,
llvm::ArrayRef<size_t> taskStages, llvm::ArrayRef<size_t> stageSizes,
const SchedulingTarget &target, MergeScheduleResult &schedule, std::string &error);
enum class PipelineWorkloadPreparation {
Ready,
@@ -10,6 +10,77 @@
namespace onnx_mlir::spatial {
// Greedy maximal prefixes minimize ordered pack count: both constraints are
// closed under taking subranges. The optimum cost cannot increase with packs.
inline Cost estimateOrderedPackCost(
llvm::ArrayRef<Cost> taskCosts,
const std::vector<std::vector<size_t>> &taskWeightIds, size_t weightCount,
llvm::ArrayRef<size_t> tasks, size_t residentWeightCapacity,
size_t maximumPacks, std::vector<size_t> *taskPacks = nullptr) {
Cost low = 0;
Cost high = 0;
for (size_t task : tasks) {
low = std::max(low, taskCosts[task]);
high = checkedAdd(high, taskCosts[task]);
}
low = std::max(
low, high / maximumPacks + (high % maximumPacks != 0));
std::vector<size_t> seen(weightCount);
size_t generation = 0;
auto pack = [&](Cost maximumCost, std::vector<size_t> *assignment) {
size_t packs = 1;
Cost cost = 0;
size_t packWeightCount = 0;
++generation;
bool fits = true;
bool packEmpty = true;
for (size_t task : tasks) {
size_t addedWeights = 0;
for (size_t weight : taskWeightIds[task])
addedWeights += seen[weight] != generation;
Cost taskCost = taskCosts[task];
bool startsNewPack = !packEmpty
&& (cost > maximumCost - taskCost
|| packWeightCount + addedWeights > residentWeightCapacity);
if (startsNewPack) {
if (++packs > maximumPacks) {
fits = false;
break;
}
cost = 0;
packWeightCount = 0;
++generation;
packEmpty = true;
}
cost = checkedAdd(cost, taskCost);
for (size_t weight : taskWeightIds[task])
if (seen[weight] != generation) {
seen[weight] = generation;
++packWeightCount;
}
if (assignment)
assignment->push_back(packs - 1);
packEmpty = false;
}
return fits;
};
while (low < high) {
Cost middle = low + (high - low) / 2;
if (pack(middle, nullptr))
high = middle;
else
low = middle + 1;
}
if (taskPacks) {
taskPacks->clear();
if (!pack(low, taskPacks))
taskPacks->clear();
}
return low;
}
struct IndexedResidentWeights {
std::vector<std::vector<size_t>> taskWeightIds;
size_t weightCount = 0;
@@ -9,6 +9,7 @@
#include <vector>
#include "mlir/Support/LogicalResult.h"
#include "llvm/ADT/ArrayRef.h"
#include "Utils.hpp"
@@ -61,6 +62,9 @@ struct WholeScheduleCost {
std::optional<double> selectionMargin;
std::array<ScheduleRecurrenceEvidence, 2> recurrence;
StaticLoweringSupport loweringSupport = StaticLoweringSupport::Unknown;
// Optional task-attached work and recurrence contributions by physical processor.
// These guide search; they are not additional charges or hardware guarantees.
std::vector<double> processorTaskWorkNs, processorCriticalDelayNs;
struct AnalysisTiming {
double extractionSeconds = 0, expansionSeconds = 0, evaluationSeconds = 0;
} timing;
@@ -99,6 +103,9 @@ struct SchedulingTarget {
std::function<WholeScheduleCost(mlir::ModuleOp,
llvm::json::Object &)> estimateRealizedScheduleCost;
std::string wholeScheduleCostUnavailable;
// Optional recurrence priority over stage dependency spans. This screens
// moves only; the complete whole-schedule cost remains authoritative.
std::function<Cost(llvm::ArrayRef<Cost>)> estimatePipelineDependencySpan;
std::function<void(const ComputeGraph &,
const ScheduledComputeMaterializationResult &, const DeferredTransferPlan &,
const DeferredBoundaryPlan &)> reportPeriodicSchedule;