even better more structured throughput scheduling
Validate Operations / validate-operations (push) Has been cancelled
Validate Operations / validate-operations (push) Has been cancelled
plot adjustments validation adjustments
This commit is contained in:
@@ -35,7 +35,7 @@ struct Notice { size_t arrival; int64_t source; };
|
||||
struct Wait { size_t end; int64_t expected; };
|
||||
struct Core {
|
||||
size_t entry, tail, firstNetwork, lastNetwork;
|
||||
Cost pending = 0, work = 0, instructions = 0;
|
||||
Cost pending = 0, work = 0, instructions = 0, directWork = 0, constantWork = 0;
|
||||
std::array<Cost, 7> service{};
|
||||
std::array<Cost, 6> traceWork{};
|
||||
llvm::SHA256 requests{}, blockingOrder{};
|
||||
@@ -80,10 +80,13 @@ public:
|
||||
stateWords = state.size();
|
||||
return true;
|
||||
}
|
||||
// Snapshots are immutable; do not retain geometric construction capacity.
|
||||
state.shrink_to_fit();
|
||||
generations.shrink_to_fit();
|
||||
size_t added = add(mul(add(state.capacity(), generations.capacity()), sizeof(uint64_t)), sizeof(Snapshot));
|
||||
if (add(bytes, added) > 16 * 1024 * 1024) {
|
||||
if (add(bytes, added) > 64 * 1024 * 1024) {
|
||||
exhausted = true;
|
||||
evidence.reason = "periodic-state history exceeds 16 MiB";
|
||||
evidence.reason = "periodic-state history exceeds 64 MiB";
|
||||
return false;
|
||||
}
|
||||
bytes += added;
|
||||
@@ -141,7 +144,7 @@ PimQueuePolicyResult estimateQueuedPeriod(const std::vector<Event> &events,
|
||||
pending.push({sub(time, service[node]), node, memoryAccess[node]});
|
||||
};
|
||||
for (size_t node = 0; node < nodes.size(); ++node) schedule(node);
|
||||
const size_t maximumRounds = verifyState ? 256 : 64;
|
||||
const size_t maximumRounds = verifyState ? 1024 : 64;
|
||||
std::vector<Cost> roundEnd(maximumRounds);
|
||||
std::vector<size_t> roundCount(maximumRounds);
|
||||
std::map<int64_t, std::vector<Cost>> queueWait;
|
||||
@@ -462,7 +465,7 @@ private:
|
||||
// ROB retirement needs another edge; matrix also has AG/commit registers.
|
||||
return issue ? mul(matrix ? 3 : 1, target.processorPeriodNs) : 0;
|
||||
}
|
||||
void memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes);
|
||||
void memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes, bool constant = false);
|
||||
bool communication(int64_t core, uint32_t pc, Opcode op,
|
||||
int32_t peer, int32_t reg, int32_t count, Cost bytes);
|
||||
};
|
||||
@@ -518,14 +521,17 @@ bool Analysis::communication(int64_t core, uint32_t pc, Opcode op,
|
||||
target.getLinkServiceNs(target.controlMessageBytes, core, peer, arithmetic));
|
||||
receives[{peer, core}].push_back({complete, permission.arrival, permission.begin, bytes});
|
||||
}
|
||||
cores.at(core).work = add(cores.at(core).work, add(local(bytes, op == Opcode::recv), retirement()));
|
||||
cores.at(core).service[2] = add(cores.at(core).service[2], add(local(bytes, op == Opcode::recv), retirement()));
|
||||
opcodeService[static_cast<unsigned>(op)] = add(opcodeService[static_cast<unsigned>(op)], add(local(bytes, op == Opcode::recv), retirement()));
|
||||
Cost endpointWork = add(local(bytes, op == Opcode::recv), retirement());
|
||||
auto &state = cores.at(core);
|
||||
state.work = add(state.work, endpointWork);
|
||||
state.service[2] = add(state.service[2], endpointWork);
|
||||
state.directWork = add(state.directWork, endpointWork);
|
||||
opcodeService[static_cast<unsigned>(op)] = add(opcodeService[static_cast<unsigned>(op)], endpointWork);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void Analysis::memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes) {
|
||||
void Analysis::memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes, bool constant) {
|
||||
Core &state = cores.at(core);
|
||||
recordBlocking(core, write ? Opcode::st : Opcode::ld, bytes);
|
||||
if (options.workBreakdown) {
|
||||
@@ -548,6 +554,7 @@ void Analysis::memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes) {
|
||||
Cost cost = add(add(add(write ? local(bytes, false) : 0, demand.serviceNs), completion),
|
||||
add(target.getSharedMemoryLinkServiceNs(write ? bytes : target.controlMessageBytes, core, false, arithmetic),
|
||||
target.getSharedMemoryLinkServiceNs(!write && target.sharedMemoryReadReturnsPayload ? bytes : 0, core, true, arithmetic)));
|
||||
if (constant && !write) state.constantWork = add(state.constantWork, cost);
|
||||
state.service[write ? 4 : 3] = add(state.service[write ? 4 : 3], cost);
|
||||
opcodeService[static_cast<unsigned>(write ? Opcode::st : Opcode::ld)] = add(opcodeService[static_cast<unsigned>(write ? Opcode::st : Opcode::ld)], cost);
|
||||
if (memoryQueue) {
|
||||
@@ -618,6 +625,8 @@ bool Analysis::readCore(int64_t core, llvm::StringRef data, llvm::ArrayRef<int64
|
||||
int32_t b = llvm::support::endian::read32le(record + 12);
|
||||
int32_t length = llvm::support::endian::read32le(record + 16);
|
||||
++state.instructions;
|
||||
// Pimsim-nn drops vsoftmax before fetch: no issue, memory, or execution service.
|
||||
if (op == Opcode::vsoftmax) continue;
|
||||
if (op <= Opcode::setbw) {
|
||||
if (op == Opcode::sld) { error = "scalar memory loads are unsupported by this diagnostic"; return false; }
|
||||
if (op == Opcode::setbw) {
|
||||
@@ -635,7 +644,7 @@ bool Analysis::readCore(int64_t core, llvm::StringRef data, llvm::ArrayRef<int64
|
||||
opcodeService[static_cast<unsigned>(op)] = add(opcodeService[static_cast<unsigned>(op)], cost);
|
||||
continue;
|
||||
}
|
||||
if (length < 0 || op > Opcode::sync || op == Opcode::vsoftmax) {
|
||||
if (length < 0 || op > Opcode::sync) {
|
||||
error = "unsupported instruction opcode or negative transfer length"; return false;
|
||||
}
|
||||
Cost bytes = length;
|
||||
@@ -781,7 +790,7 @@ bool Analysis::readExecution(int64_t core, const std::vector<PimExecutionEvent>
|
||||
}
|
||||
index = found->second.end - 1;
|
||||
} else if (item.kind == PimExecutionEvent::Load || item.kind == PimExecutionEvent::Store) {
|
||||
memoryAccess(core, position, item.kind == PimExecutionEvent::Store, item.amount);
|
||||
memoryAccess(core, position, item.kind == PimExecutionEvent::Store, item.amount, item.constant);
|
||||
} else {
|
||||
for (int64_t value : {item.peer, item.reg, item.count})
|
||||
if (value < 0 || value > std::numeric_limits<int32_t>::max()) {
|
||||
@@ -839,7 +848,7 @@ EvaluatedAnalysis Analysis::evaluate() {
|
||||
if (cores.empty()) result.error = "artifact has no active repeating instruction stream";
|
||||
else if (!connect()) result.error = error;
|
||||
if (!result.error.empty()) { result.cost.reason = result.error; return result; }
|
||||
if (options.detailed) {
|
||||
if (options.detailed || options.searchGuidance) {
|
||||
auto recurrence = analyzePeriodicDependencies(events.size(), edges, result.error);
|
||||
if (mlir::failed(recurrence)) { result.cost.reason = result.error; return result; }
|
||||
result.recurrence = std::move(*recurrence);
|
||||
@@ -859,6 +868,23 @@ EvaluatedAnalysis Analysis::evaluate() {
|
||||
std::chrono::steady_clock::now() - start).count();
|
||||
result.cost = queue.cost();
|
||||
}
|
||||
if (options.searchGuidance) {
|
||||
result.cost.processorTaskWorkNs.resize(target.processorCount);
|
||||
result.cost.processorCriticalDelayNs.resize(target.processorCount);
|
||||
for (const auto &[core, state] : cores)
|
||||
result.cost.processorTaskWorkNs[core] = add(state.constantWork,
|
||||
add(add(state.service[0], state.service[1]),
|
||||
add(sub(state.service[2], state.directWork), state.service[5])));
|
||||
Cost distance = 0;
|
||||
for (size_t index : result.recurrence->criticalCycle)
|
||||
distance = add(distance, edges[index].distance);
|
||||
if (distance)
|
||||
for (size_t index : result.recurrence->criticalCycle) {
|
||||
const auto &edge = edges[index];
|
||||
result.cost.processorCriticalDelayNs[events[edge.source].core]
|
||||
+= static_cast<double>(edge.delayNs) / distance;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -89,8 +89,11 @@ struct PimPeriodicOptions {
|
||||
bool instructionIssue = true;
|
||||
bool scalarIssue = true;
|
||||
bool ignoreLocalCopies = false;
|
||||
// Spatial ablation mirrors codegen's omission of SYNC/WAIT instructions.
|
||||
bool omitSynchronization = false;
|
||||
bool workBreakdown = false;
|
||||
bool measureTime = false;
|
||||
bool searchGuidance = false;
|
||||
// Optional ordered work/IO extraction for timing validation; no execution.
|
||||
PimExecutionProgram *executionTrace = nullptr;
|
||||
// Prove recurrence of the modeled state. Instruction diagnostics retain
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#include "PimPipelineSelection.hpp"
|
||||
#include "PimCompilerOptions.hpp"
|
||||
#include "PimCompilerUtils.hpp"
|
||||
#include "PimSpatialCost.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/PipelineScheduling.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/DebugDump.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/FileSystemUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/ReportUtils.hpp"
|
||||
@@ -108,83 +110,133 @@ struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, Op
|
||||
}
|
||||
|
||||
LogicalResult evaluateSpatialCandidates(ModuleOp checkpoint,
|
||||
const PreparedSpatialSchedules &prepared, std::vector<WholeScheduleCost> &costs,
|
||||
PreparedSpatialSchedules &prepared, std::vector<WholeScheduleCost> &costs,
|
||||
llvm::json::Object &report) {
|
||||
llvm::json::Array summaries;
|
||||
double realization = 0, extraction = 0, expansion = 0, queue = 0;
|
||||
std::string errors;
|
||||
for (size_t i = 0; i < costs.size(); ++i) {
|
||||
llvm::json::Object summary{{"index", static_cast<int64_t>(i)}};
|
||||
double realization = 0, extraction = 0, expansion = 0, queue = 0, evaluation = 0;
|
||||
auto evaluate = [&](const MergeScheduleResult &schedule) {
|
||||
size_t index = summaries.size();
|
||||
llvm::json::Object summary{{"index", static_cast<int64_t>(index)}};
|
||||
OwningOpRef<ModuleOp> candidate;
|
||||
WholeScheduleCost cost;
|
||||
std::string scheduleReport;
|
||||
{
|
||||
ScopedCompilerOutputSuppression silence;
|
||||
if (failed(realizeCandidate(checkpoint, prepared.candidates[i], candidate, scheduleReport, summary)))
|
||||
costs[i].reason = "Spatial schedule realization failed verification";
|
||||
else costs[i] = spatialCost(*candidate, summary);
|
||||
if (failed(realizeCandidate(checkpoint, schedule, candidate, scheduleReport, summary)))
|
||||
cost.reason = "Spatial schedule realization failed verification";
|
||||
else cost = spatialCost(*candidate, summary);
|
||||
}
|
||||
realization += summary.getNumber("realization_seconds").value_or(0);
|
||||
extraction += costs[i].timing.extractionSeconds;
|
||||
expansion += costs[i].timing.expansionSeconds;
|
||||
queue += costs[i].timing.evaluationSeconds;
|
||||
bool supported = costs[i].loweringSupport == StaticLoweringSupport::Supported;
|
||||
auto convergence = costs[i].convergenceProved();
|
||||
auto support = costs[i].loweringSupported();
|
||||
bool converged = convergence == true;
|
||||
bool safe = costs[i].eligible();
|
||||
summary["cost_available"] = costs[i].available();
|
||||
evaluation += summary.getNumber("spatial_cost_seconds").value_or(0);
|
||||
extraction += cost.timing.extractionSeconds;
|
||||
expansion += cost.timing.expansionSeconds;
|
||||
queue += cost.timing.evaluationSeconds;
|
||||
auto convergence = cost.convergenceProved(), support = cost.loweringSupported();
|
||||
summary["cost_available"] = cost.available();
|
||||
summary["convergence_proved"] = convergence
|
||||
? llvm::json::Value(*convergence) : llvm::json::Value(nullptr);
|
||||
summary["lowering_supported"] = support
|
||||
? llvm::json::Value(*support) : llvm::json::Value(nullptr);
|
||||
summary["eligible"] = safe;
|
||||
if (costs[i].primary) summary["interval_ns"] = *costs[i].primary;
|
||||
if (costs[i].alternative) summary["alternative_tie_interval_ns"] = *costs[i].alternative;
|
||||
if (!safe) {
|
||||
std::string property;
|
||||
if (!costs[i].available()) property += "unavailable timing; ";
|
||||
if (!supported) property += "unsupported or unknown static lowering; ";
|
||||
if (!converged) property += "unproved modeled-state convergence; ";
|
||||
if (property.empty()) property = "unavailable selection margin";
|
||||
summary["reason"] = property + ": " + costs[i].reason;
|
||||
errors += "\ncandidate " + std::to_string(i) + ": " + property + ": " + costs[i].reason;
|
||||
summary["eligible"] = cost.eligible();
|
||||
if (cost.primary) summary["interval_ns"] = *cost.primary;
|
||||
if (cost.alternative) summary["alternative_tie_interval_ns"] = *cost.alternative;
|
||||
llvm::json::Array budgets, witness;
|
||||
std::vector<size_t> cores(stages);
|
||||
std::vector<double> pressure(stages);
|
||||
for (size_t core = 0; core < schedule.processorStages.size(); ++core) {
|
||||
size_t stage = schedule.processorStages[core];
|
||||
++cores[stage];
|
||||
if (core < cost.processorCriticalDelayNs.size())
|
||||
pressure[stage] += cost.processorCriticalDelayNs[core];
|
||||
}
|
||||
for (size_t stage = 0; stage < stages; ++stage) {
|
||||
budgets.push_back(static_cast<int64_t>(cores[stage]));
|
||||
witness.push_back(pressure[stage]);
|
||||
}
|
||||
summary["stage_core_budgets"] = std::move(budgets);
|
||||
summary["recurrence_stage_contributions_ns"] = std::move(witness);
|
||||
llvm::json::Array runs;
|
||||
for (size_t begin = 0; begin < prepared.graph.nodes.size();) {
|
||||
auto stageFor = [&](size_t task) {
|
||||
return schedule.processorStages[schedule.computeToCpuMap.lookup(
|
||||
prepared.graph.nodes[task].instance)];
|
||||
};
|
||||
size_t end = begin + 1, stage = stageFor(begin);
|
||||
while (end < prepared.graph.nodes.size() && stageFor(end) == stage) ++end;
|
||||
runs.push_back(llvm::json::Array{static_cast<int64_t>(begin),
|
||||
static_cast<int64_t>(end - begin), static_cast<int64_t>(stage)});
|
||||
begin = end;
|
||||
}
|
||||
summary["task_stage_runs"] = std::move(runs);
|
||||
if (!cost.eligible()) summary["reason"] = cost.reason;
|
||||
summaries.push_back(std::move(summary));
|
||||
}
|
||||
report["candidates"] = std::move(summaries);
|
||||
return cost;
|
||||
};
|
||||
std::string error;
|
||||
auto start = Clock::now();
|
||||
auto search = optimizePipelineSchedule(prepared.graph, prepared.candidates.front(),
|
||||
stages, target, evaluate, error);
|
||||
report["search_seconds"] = elapsed(start);
|
||||
report["search_refinement_seconds"] = std::max(0.0, elapsed(start) - realization - evaluation);
|
||||
report["cost_evaluation_seconds"] = evaluation;
|
||||
report["candidate_realization_seconds"] = realization;
|
||||
report["event_extraction_seconds"] = extraction;
|
||||
report["event_expansion_seconds"] = expansion;
|
||||
report["queue_seconds"] = queue;
|
||||
if (!errors.empty()) {
|
||||
report["candidates"] = std::move(summaries);
|
||||
if (failed(search)) {
|
||||
report["status"] = "incomplete-costs";
|
||||
report["reason"] = errors;
|
||||
checkpoint.emitError("Spatial pipeline cannot compare the complete candidate set:") << errors;
|
||||
report["reason"] = error;
|
||||
checkpoint.emitError("Spatial pipeline cannot compare the complete candidate set: ") << error;
|
||||
return failure();
|
||||
}
|
||||
prepared.candidates = std::move(search->schedules);
|
||||
costs = std::move(search->costs);
|
||||
bool single = costs.empty();
|
||||
if (single) {
|
||||
report["candidates"] = llvm::json::Array{llvm::json::Object{{"index", 0}}};
|
||||
} else {
|
||||
auto *entries = report.getArray("candidates");
|
||||
for (size_t i = 0; i < entries->size(); ++i)
|
||||
(*entries)[i].getAsObject()->insert({"parent", static_cast<int64_t>(search->parents[i])});
|
||||
}
|
||||
report["generated_candidates"] = static_cast<int64_t>(prepared.candidates.size());
|
||||
report["single_candidate_analysis_skipped"] = single;
|
||||
report["search_rounds"] = static_cast<int64_t>(search->rounds);
|
||||
report["packing_attempts"] = static_cast<int64_t>(search->packingAttempts);
|
||||
report["considered_assignments"] = static_cast<int64_t>(search->consideredAssignments);
|
||||
report["search_termination"] = search->termination;
|
||||
report["queue_evaluation_budget"] = static_cast<int64_t>(kPipelineQueueEvaluationBudget);
|
||||
report["beam_width"] = static_cast<int64_t>(kPipelineBeamWidth);
|
||||
return success();
|
||||
}
|
||||
|
||||
void runOnOperation() override {
|
||||
ModuleOp checkpoint = getOperation();
|
||||
auto start = Clock::now();
|
||||
auto prepared = prepareSpatialSchedules(checkpoint, target, stages, /*collectCandidates=*/true);
|
||||
auto prepared = prepareSpatialSchedules(checkpoint, target, stages);
|
||||
if (failed(prepared)) { signalPassFailure(); return; }
|
||||
if (prepared->candidates.empty()) {
|
||||
checkpoint.emitError("Spatial pipeline requires at least one candidate schedule");
|
||||
signalPassFailure(); return;
|
||||
}
|
||||
bool single = prepared->candidates.size() == 1;
|
||||
llvm::json::Object report{{"strategy", "spatial-queue-aware"}, {"static_analysis", true},
|
||||
{"candidate_generation_seconds", elapsed(start)},
|
||||
{"generated_candidates", static_cast<int64_t>(prepared->candidates.size())},
|
||||
{"single_candidate_analysis_skipped", single}, {"selection_margin", kPipelineSelectionMargin},
|
||||
{"selection_margin", kPipelineSelectionMargin},
|
||||
{"prediction_error_bound_available", false}, {"temporary_disk_bytes", 0}};
|
||||
std::vector<WholeScheduleCost> costs(prepared->candidates.size());
|
||||
if (!single && failed(evaluateSpatialCandidates(checkpoint, *prepared, costs, report))) {
|
||||
if (pimDisableSynchronization) {
|
||||
report["synchronization_ablation"] = true;
|
||||
report["cost_model_program"] = "synchronized-reference";
|
||||
llvm::errs() << "Synchronization ablation: keep the normal selected schedule; "
|
||||
"report zero-cost SYNC/WAIT separately and omit them in code generation. "
|
||||
"Buffer reuse and restart correctness are not guaranteed.\n";
|
||||
}
|
||||
std::vector<WholeScheduleCost> costs;
|
||||
if (failed(evaluateSpatialCandidates(checkpoint, *prepared, costs, report))) {
|
||||
writeSelectionReport(std::move(report)); signalPassFailure(); return;
|
||||
}
|
||||
if (single) report["candidates"] = llvm::json::Array{llvm::json::Object{{"index", 0}}};
|
||||
bool single = costs.empty();
|
||||
OwningOpRef<ModuleOp> winner;
|
||||
std::string scheduleReport;
|
||||
llvm::json::Array attempts;
|
||||
@@ -204,6 +256,8 @@ struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, Op
|
||||
summary["reason"] = "selected Spatial schedule failed deterministic realization";
|
||||
summary["status"] = "realization-failed";
|
||||
} else {
|
||||
if (pimDisableSynchronization)
|
||||
summary["synchronization_ablation_cost"] = analyzePimSpatialSynchronizationAblation(*candidate);
|
||||
++lowered;
|
||||
result = lowerPimPipelineCandidate(*candidate, resources, summary);
|
||||
}
|
||||
@@ -264,7 +318,7 @@ struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, Op
|
||||
report["reason"] = reason;
|
||||
report["finalization_seconds"] = elapsed(finalize);
|
||||
report["total_seconds"] = elapsed(start);
|
||||
llvm::errs() << "Spatial queue-aware pipeline: candidate " << *selected << " of " << costs.size()
|
||||
llvm::errs() << "Spatial queue-aware pipeline: candidate " << *selected << " of " << prepared->candidates.size()
|
||||
<< "; " << reason << "\n";
|
||||
writeSelectionReport(std::move(report));
|
||||
}
|
||||
|
||||
@@ -47,13 +47,15 @@ class Extraction {
|
||||
public:
|
||||
const PimInstructionTiming ⌖
|
||||
bool diagnostics;
|
||||
bool omitSynchronization;
|
||||
PimExecutionProgram program;
|
||||
std::string error, loweringReason;
|
||||
StaticLoweringSupport loweringSupport = StaticLoweringSupport::Supported;
|
||||
std::map<std::string, int64_t> gaps;
|
||||
int64_t visited = 0, constantLoads = 0;
|
||||
explicit Extraction(const PimInstructionTiming &target, bool diagnostics)
|
||||
: target(target), diagnostics(diagnostics), program(target.processorCount) {}
|
||||
explicit Extraction(const PimInstructionTiming &target, bool diagnostics, bool omitSynchronization)
|
||||
: target(target), diagnostics(diagnostics), omitSynchronization(omitSynchronization),
|
||||
program(target.processorCount) {}
|
||||
bool build(ModuleOp module);
|
||||
private:
|
||||
llvm::SmallDenseSet<int64_t, 16> assignedCores;
|
||||
@@ -135,7 +137,7 @@ Residence Extraction::storage(Value value) {
|
||||
return Residence::Host;
|
||||
}
|
||||
if (isa_and_nonnull<SpatVMMOp, SpatVAddOp, SpatVSubOp, SpatVMulOp,
|
||||
SpatVMaxOp, SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp,
|
||||
SpatVMaxOp, SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp, SpatSoftmaxOp,
|
||||
SpatChannelReceiveOp, SpatHostWaitLoadOp, SpatScheduledCompute,
|
||||
SpatScheduledComputeBatch, SpatConcatOp, tensor::EmptyOp,
|
||||
tensor::InsertSliceOp, tensor::PadOp, linalg::TransposeOp, scf::ForOp>(value.getDefiningOp()))
|
||||
@@ -246,10 +248,13 @@ bool Extraction::operation(Operation &op, Sequence &sequence) {
|
||||
return *result;
|
||||
};
|
||||
if (auto sync = dyn_cast<SpatSyncOp>(op)) {
|
||||
sequence.push_back({Event::Signal, 0, index(sync.getTargetCoreId()), index(sync.getEventRegister())});
|
||||
if (!omitSynchronization)
|
||||
sequence.push_back({Event::Signal, 0, index(sync.getTargetCoreId()), index(sync.getEventRegister())});
|
||||
} else if (auto wait = dyn_cast<SpatWaitOp>(op)) {
|
||||
int64_t count = index(wait.getWaitValue());
|
||||
if (count) sequence.push_back({Event::Wait, 0, 0, index(wait.getEventRegister()), count});
|
||||
if (!omitSynchronization) {
|
||||
int64_t count = index(wait.getWaitValue());
|
||||
if (count) sequence.push_back({Event::Wait, 0, 0, index(wait.getEventRegister()), count});
|
||||
}
|
||||
} else if (auto send = dyn_cast<SpatChannelSendOp>(op)) {
|
||||
materialize(send.getInput(), sequence);
|
||||
sequence.push_back({Event::Send, bytes(send.getInput().getType()), index(send.getTargetCoreId())});
|
||||
@@ -261,10 +266,13 @@ bool Extraction::operation(Operation &op, Sequence &sequence) {
|
||||
sequence.push_back({Event::Repeat, static_cast<uint64_t>(plan->second.getRunCount()),
|
||||
0, 0, 0, {{Event::Store, ceil(mul(plan->second.getRunElements(), target.computeBitwidth), 8)}}});
|
||||
else sequence.push_back({Event::Store, bytes(store.getInput().getType())});
|
||||
sequence.push_back({Event::Signal, 0, index(store.getTargetCoreId()), index(store.getEventRegister())});
|
||||
if (!omitSynchronization)
|
||||
sequence.push_back({Event::Signal, 0, index(store.getTargetCoreId()), index(store.getEventRegister())});
|
||||
} else if (auto load = dyn_cast<SpatHostWaitLoadOp>(op)) {
|
||||
int64_t count = index(load.getWaitValue());
|
||||
if (count) sequence.push_back({Event::Wait, 0, 0, index(load.getEventRegister()), count});
|
||||
if (!omitSynchronization) {
|
||||
int64_t count = index(load.getWaitValue());
|
||||
if (count) sequence.push_back({Event::Wait, 0, 0, index(load.getEventRegister()), count});
|
||||
}
|
||||
sequence.push_back({Event::Load, bytes(load.getOutput().getType())});
|
||||
} else if (auto vmm = dyn_cast<SpatVMMOp>(op)) {
|
||||
materialize(vmm.getInput(), sequence);
|
||||
@@ -284,6 +292,9 @@ bool Extraction::operation(Operation &op, Sequence &sequence) {
|
||||
pim_binary::kMatrixWeightPrecisionBits, weight.getDimSize(1) / target.matrixColumns,
|
||||
ceil(target.computeBitwidth, 8));
|
||||
work(sequence, getPimMatrixWorkNs(target, target.computeBitwidth, output), Event::Matrix);
|
||||
} else if (auto softmax = dyn_cast<SpatSoftmaxOp>(op)) {
|
||||
materialize(softmax.getInput(), sequence);
|
||||
// Pimsim-nn omits vsoftmax itself. Keep any preceding input materialization.
|
||||
} else if (isa<SpatVAddOp, SpatVSubOp, SpatVMulOp, SpatVMaxOp,
|
||||
SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp>(op)) {
|
||||
for (Value operand : op.getOperands()) materialize(operand, sequence);
|
||||
@@ -464,7 +475,7 @@ PimPeriodicEstimate onnx_mlir::evaluatePimSpatialSchedule(ModuleOp module,
|
||||
if (!target.hasPeriodicTransferTiming() || !target.computeBitwidth)
|
||||
return unavailableCost("unsupported abstract timing contract");
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
Extraction extraction(target, options.workBreakdown);
|
||||
Extraction extraction(target, options.workBreakdown, options.omitSynchronization);
|
||||
bool available = extraction.build(module);
|
||||
double extractionMs = std::chrono::duration<double, std::milli>(
|
||||
std::chrono::steady_clock::now() - start).count();
|
||||
@@ -587,8 +598,28 @@ llvm::json::Object onnx_mlir::analyzePimSpatialSchedule(ModuleOp module,
|
||||
return evaluatePimSpatialSchedule(module, target, options).report;
|
||||
}
|
||||
|
||||
llvm::json::Object onnx_mlir::analyzePimSpatialSynchronizationAblation(ModuleOp module) {
|
||||
auto timing = getPimInstructionTiming();
|
||||
auto contract = getPimPeriodicTimingContract();
|
||||
if (!contract) return unavailableCost(llvm::toString(contract.takeError())).report;
|
||||
timing.matrixCellBits = *contract;
|
||||
PimPeriodicOptions options;
|
||||
options.omitSynchronization = true;
|
||||
options.verifyPeriodicState = true;
|
||||
options.measureTime = true;
|
||||
auto result = evaluatePimSpatialSchedule(module, timing, options);
|
||||
result.report["diagnostic_only"] = true;
|
||||
result.report["used_for_selection"] = false;
|
||||
result.report["cost_model_program"] = "synchronization-omitted";
|
||||
return std::move(result.report);
|
||||
}
|
||||
|
||||
void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target,
|
||||
PimInstructionTiming timing) {
|
||||
// The stage-zero entry barrier joins the previous generation's work.
|
||||
target.estimatePipelineDependencySpan = [](llvm::ArrayRef<Cost> spans) {
|
||||
return spans.empty() ? Cost{0} : spans.front();
|
||||
};
|
||||
auto contract = getPimPeriodicTimingContract();
|
||||
if (contract) timing.matrixCellBits = *contract;
|
||||
auto hardware = std::make_shared<const PimInstructionTiming>(std::move(timing));
|
||||
@@ -601,8 +632,6 @@ void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target,
|
||||
if (!contract) target.wholeScheduleCostUnavailable = llvm::toString(contract.takeError());
|
||||
if (!hardware->hasPeriodicTransferTiming())
|
||||
target.wholeScheduleCostUnavailable = "target has no supported queue timing contract";
|
||||
if (pimDisableSynchronization)
|
||||
target.wholeScheduleCostUnavailable = "synchronization ablation has no periodic queue timing contract";
|
||||
if (!target.wholeScheduleCostUnavailable.empty()) return;
|
||||
target.estimateRealizedScheduleCost = [hardware](mlir::ModuleOp module,
|
||||
llvm::json::Object &summary) {
|
||||
@@ -610,6 +639,7 @@ void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target,
|
||||
options.workBreakdown = pimPeriodicDiagnostics;
|
||||
options.measureTime = true;
|
||||
options.verifyPeriodicState = true;
|
||||
options.searchGuidance = true;
|
||||
auto estimate = evaluatePimSpatialSchedule(module, *hardware, options);
|
||||
auto cost = std::move(estimate.cost);
|
||||
summary = std::move(estimate.report);
|
||||
|
||||
@@ -12,5 +12,6 @@ PimPeriodicEstimate evaluatePimSpatialSchedule(mlir::ModuleOp module,
|
||||
const PimInstructionTiming &target, PimPeriodicOptions options = {});
|
||||
llvm::json::Object analyzePimSpatialSchedule(mlir::ModuleOp module,
|
||||
const PimInstructionTiming &target, PimPeriodicOptions options = {});
|
||||
llvm::json::Object analyzePimSpatialSynchronizationAblation(mlir::ModuleOp module);
|
||||
|
||||
} // namespace onnx_mlir
|
||||
|
||||
+3
-3
@@ -65,7 +65,7 @@ struct ScheduleAndRealizeSpatialPass final
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
auto prepared = prepareSpatialSchedules(moduleOp, target, pipelineStages, false);
|
||||
auto prepared = prepareSpatialSchedules(moduleOp, target, pipelineStages);
|
||||
if (failed(prepared) || failed(realizeSpatialSchedule(moduleOp,
|
||||
prepared->candidates.front(), target, pipelineStages, exportStage,
|
||||
periodicDiagnostics ? &prepared->graph : nullptr)))
|
||||
@@ -83,7 +83,7 @@ private:
|
||||
} // namespace
|
||||
|
||||
FailureOr<PreparedSpatialSchedules> prepareSpatialSchedules(ModuleOp moduleOp,
|
||||
const SchedulingTarget &target, size_t pipelineStages, bool collectCandidates) {
|
||||
const SchedulingTarget &target, size_t pipelineStages) {
|
||||
if (!hasValidTarget(target) || !PipelineCoreLayout(target.processorCount, pipelineStages).isValid()
|
||||
|| (pipelineStages > 1 && !target.synchronizationRegisterCount))
|
||||
return moduleOp.emitError("Spatial scheduling requires an explicit valid target and pipeline layout"), failure();
|
||||
@@ -122,7 +122,7 @@ FailureOr<PreparedSpatialSchedules> prepareSpatialSchedules(ModuleOp moduleOp,
|
||||
break;
|
||||
}
|
||||
if (succeeded(runPipelineScheduler(
|
||||
scheduledGraph, schedule, pipelineStages, target, pipelineError, collectCandidates ? &prepared.candidates : nullptr)))
|
||||
scheduledGraph, schedule, pipelineStages, target, pipelineError)))
|
||||
break;
|
||||
moduleOp.emitError() << pipelineError;
|
||||
return failure();
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ struct PreparedSpatialSchedules {
|
||||
// Success always includes at least one candidate, including an empty compute graph.
|
||||
mlir::FailureOr<PreparedSpatialSchedules> prepareSpatialSchedules(
|
||||
mlir::ModuleOp module, const SchedulingTarget &target,
|
||||
size_t pipelineStages, bool collectCandidates);
|
||||
size_t pipelineStages);
|
||||
mlir::LogicalResult realizeSpatialSchedule(mlir::ModuleOp module,
|
||||
const MergeScheduleResult &schedule, const SchedulingTarget &target,
|
||||
size_t pipelineStages, SpatialDataflowExportStage exportStage,
|
||||
|
||||
+601
-576
File diff suppressed because it is too large
Load Diff
+29
-26
@@ -7,6 +7,7 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <numeric>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
@@ -32,25 +33,6 @@ inline bool isBetterStagePlacement(const StagePlacementScore &candidate,
|
||||
return candidate < current;
|
||||
}
|
||||
|
||||
inline bool isBetterPipelineSchedule(const ScheduleCostSummary &candidate,
|
||||
const ScheduleCostSummary ¤t) {
|
||||
auto score = [](const ScheduleCostSummary &result) {
|
||||
Cost stage = 0;
|
||||
for (Cost span : result.stageSpanNs) stage = std::max(stage, span);
|
||||
Cost memory = checkedAdd(result.baselineMemory.serviceNs, result.boundaryMemory.serviceNs);
|
||||
return std::make_tuple(std::max(stage, memory), stage, memory);
|
||||
};
|
||||
auto currentScore = score(current);
|
||||
auto candidateScore = score(candidate);
|
||||
// Stage spans omit contention: do not exchange a larger compute regression
|
||||
// for a smaller memory saving, even when memory remains the largest resource.
|
||||
Cost currentDemand = checkedAdd(std::get<1>(currentScore), std::get<2>(currentScore));
|
||||
Cost candidateDemand = checkedAdd(std::get<1>(candidateScore), std::get<2>(candidateScore));
|
||||
Cost currentCoreDemand = checkedAdd(static_cast<Cost>(current.maximumCoreWorkNs), std::get<2>(currentScore));
|
||||
Cost candidateCoreDemand = checkedAdd(static_cast<Cost>(candidate.maximumCoreWorkNs), std::get<2>(candidateScore));
|
||||
return candidateScore < currentScore && candidateDemand <= currentDemand
|
||||
&& candidateCoreDemand <= currentCoreDemand;
|
||||
}
|
||||
|
||||
struct PipelineStageRange {
|
||||
size_t begin;
|
||||
@@ -119,13 +101,34 @@ private:
|
||||
std::vector<size_t> physicalCoreStages;
|
||||
};
|
||||
|
||||
// Produces a complete physical schedule directly from the shared compute graph.
|
||||
mlir::LogicalResult runPipelineScheduler(const ComputeGraph& graph,
|
||||
MergeScheduleResult& schedule,
|
||||
size_t pipelineStages,
|
||||
const SchedulingTarget& physicalTarget,
|
||||
std::string& error,
|
||||
std::vector<MergeScheduleResult> *candidates = nullptr);
|
||||
// One initialization procedure, also used for intermediate emission.
|
||||
mlir::LogicalResult runPipelineScheduler(const ComputeGraph &graph,
|
||||
MergeScheduleResult &schedule, size_t pipelineStages,
|
||||
const SchedulingTarget &target, std::string &error);
|
||||
|
||||
inline constexpr size_t kPipelineQueueEvaluationBudget = 32;
|
||||
inline constexpr size_t kPipelineBeamWidth = 2;
|
||||
|
||||
struct PipelineSearchResult {
|
||||
std::vector<MergeScheduleResult> schedules;
|
||||
std::vector<WholeScheduleCost> costs;
|
||||
std::vector<size_t> parents;
|
||||
size_t rounds = 0, packingAttempts = 0, consideredAssignments = 0;
|
||||
std::string termination;
|
||||
};
|
||||
|
||||
// The callback realizes an independent Spatial clone; it must not mutate graph IR.
|
||||
using PipelineCostQuery = std::function<WholeScheduleCost(const MergeScheduleResult &)>;
|
||||
mlir::FailureOr<PipelineSearchResult> optimizePipelineSchedule(
|
||||
const ComputeGraph &graph, const MergeScheduleResult &initial,
|
||||
size_t pipelineStages, const SchedulingTarget &target,
|
||||
const PipelineCostQuery &evaluate, std::string &error,
|
||||
size_t evaluationBudget = kPipelineQueueEvaluationBudget);
|
||||
|
||||
// Constructive capacity/topology packing shared with the small-instance oracle.
|
||||
mlir::LogicalResult packPipelineSchedule(const ComputeGraph &graph,
|
||||
llvm::ArrayRef<size_t> taskStages, llvm::ArrayRef<size_t> stageSizes,
|
||||
const SchedulingTarget &target, MergeScheduleResult &schedule, std::string &error);
|
||||
|
||||
enum class PipelineWorkloadPreparation {
|
||||
Ready,
|
||||
|
||||
+71
@@ -10,6 +10,77 @@
|
||||
|
||||
namespace onnx_mlir::spatial {
|
||||
|
||||
// Greedy maximal prefixes minimize ordered pack count: both constraints are
|
||||
// closed under taking subranges. The optimum cost cannot increase with packs.
|
||||
inline Cost estimateOrderedPackCost(
|
||||
llvm::ArrayRef<Cost> taskCosts,
|
||||
const std::vector<std::vector<size_t>> &taskWeightIds, size_t weightCount,
|
||||
llvm::ArrayRef<size_t> tasks, size_t residentWeightCapacity,
|
||||
size_t maximumPacks, std::vector<size_t> *taskPacks = nullptr) {
|
||||
Cost low = 0;
|
||||
Cost high = 0;
|
||||
for (size_t task : tasks) {
|
||||
low = std::max(low, taskCosts[task]);
|
||||
high = checkedAdd(high, taskCosts[task]);
|
||||
}
|
||||
low = std::max(
|
||||
low, high / maximumPacks + (high % maximumPacks != 0));
|
||||
|
||||
std::vector<size_t> seen(weightCount);
|
||||
size_t generation = 0;
|
||||
auto pack = [&](Cost maximumCost, std::vector<size_t> *assignment) {
|
||||
size_t packs = 1;
|
||||
Cost cost = 0;
|
||||
size_t packWeightCount = 0;
|
||||
++generation;
|
||||
bool fits = true;
|
||||
bool packEmpty = true;
|
||||
for (size_t task : tasks) {
|
||||
size_t addedWeights = 0;
|
||||
for (size_t weight : taskWeightIds[task])
|
||||
addedWeights += seen[weight] != generation;
|
||||
Cost taskCost = taskCosts[task];
|
||||
bool startsNewPack = !packEmpty
|
||||
&& (cost > maximumCost - taskCost
|
||||
|| packWeightCount + addedWeights > residentWeightCapacity);
|
||||
if (startsNewPack) {
|
||||
if (++packs > maximumPacks) {
|
||||
fits = false;
|
||||
break;
|
||||
}
|
||||
cost = 0;
|
||||
packWeightCount = 0;
|
||||
++generation;
|
||||
packEmpty = true;
|
||||
}
|
||||
cost = checkedAdd(cost, taskCost);
|
||||
for (size_t weight : taskWeightIds[task])
|
||||
if (seen[weight] != generation) {
|
||||
seen[weight] = generation;
|
||||
++packWeightCount;
|
||||
}
|
||||
if (assignment)
|
||||
assignment->push_back(packs - 1);
|
||||
packEmpty = false;
|
||||
}
|
||||
return fits;
|
||||
};
|
||||
while (low < high) {
|
||||
Cost middle = low + (high - low) / 2;
|
||||
if (pack(middle, nullptr))
|
||||
high = middle;
|
||||
else
|
||||
low = middle + 1;
|
||||
}
|
||||
if (taskPacks) {
|
||||
taskPacks->clear();
|
||||
if (!pack(low, taskPacks))
|
||||
taskPacks->clear();
|
||||
}
|
||||
return low;
|
||||
}
|
||||
|
||||
|
||||
struct IndexedResidentWeights {
|
||||
std::vector<std::vector<size_t>> taskWeightIds;
|
||||
size_t weightCount = 0;
|
||||
|
||||
+7
@@ -9,6 +9,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include "mlir/Support/LogicalResult.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
|
||||
#include "Utils.hpp"
|
||||
|
||||
@@ -61,6 +62,9 @@ struct WholeScheduleCost {
|
||||
std::optional<double> selectionMargin;
|
||||
std::array<ScheduleRecurrenceEvidence, 2> recurrence;
|
||||
StaticLoweringSupport loweringSupport = StaticLoweringSupport::Unknown;
|
||||
// Optional task-attached work and recurrence contributions by physical processor.
|
||||
// These guide search; they are not additional charges or hardware guarantees.
|
||||
std::vector<double> processorTaskWorkNs, processorCriticalDelayNs;
|
||||
struct AnalysisTiming {
|
||||
double extractionSeconds = 0, expansionSeconds = 0, evaluationSeconds = 0;
|
||||
} timing;
|
||||
@@ -99,6 +103,9 @@ struct SchedulingTarget {
|
||||
std::function<WholeScheduleCost(mlir::ModuleOp,
|
||||
llvm::json::Object &)> estimateRealizedScheduleCost;
|
||||
std::string wholeScheduleCostUnavailable;
|
||||
// Optional recurrence priority over stage dependency spans. This screens
|
||||
// moves only; the complete whole-schedule cost remains authoritative.
|
||||
std::function<Cost(llvm::ArrayRef<Cost>)> estimatePipelineDependencySpan;
|
||||
std::function<void(const ComputeGraph &,
|
||||
const ScheduledComputeMaterializationResult &, const DeferredTransferPlan &,
|
||||
const DeferredBoundaryPlan &)> reportPeriodicSchedule;
|
||||
|
||||
Reference in New Issue
Block a user