better throughput scheduling with steady state cost model, now limited to spatial

This commit is contained in:
NiccoloN
2026-10-26 12:07:01 +01:00
parent fd493ae6e8
commit 4b53f6cf10
62 changed files with 4293 additions and 871 deletions
+1
View File
@@ -3,6 +3,7 @@ add_pim_library(OMPimCommon
IR/AddressAnalysis.cpp
IR/BatchCoreUtils.cpp
IR/ConstantUtils.cpp
IR/CopyAccessPlan.cpp
IR/CoreBlockUtils.cpp
IR/EntryPointUtils.cpp
IR/IndexingUtils.cpp
+6 -2
View File
@@ -171,7 +171,9 @@ static llvm::FailureOr<int64_t> resolveConstantGlobalLoad(mlir::memref::LoadOp l
if (indices.size() != static_cast<size_t>(globalType.getRank()))
return mlir::failure();
auto strides = computeRowMajorStrides(globalType.getShape());
auto checkedStrides = computeRowMajorStridesChecked(globalType.getShape());
if (failed(checkedStrides)) return mlir::failure();
auto strides = std::move(*checkedStrides);
int64_t linearIndex = linearizeIndex(indices, strides);
if (linearIndex < 0 || linearIndex >= globalType.getNumElements())
return mlir::failure();
@@ -308,7 +310,9 @@ llvm::FailureOr<CompiledIndexExpr> compileConstantGlobalLoad(mlir::memref::LoadO
CompiledIndexExprNode expr;
expr.kind = CompiledIndexExprNode::Kind::ConstantGlobalLoad;
expr.globalOp = globalOp;
expr.globalStrides = computeRowMajorStrides(globalType.getShape());
auto checkedStrides = computeRowMajorStridesChecked(globalType.getShape());
if (failed(checkedStrides)) return mlir::failure();
expr.globalStrides = std::move(*checkedStrides);
expr.operands.reserve(loadOp.getIndices().size());
for (mlir::Value index : loadOp.getIndices()) {
auto compiledIndex = compileIndexValueImpl(index);
+12
View File
@@ -10,6 +10,18 @@ using namespace mlir;
namespace onnx_mlir {
bool isZeroBitPatternSplat(Attribute value) {
auto dense = dyn_cast_or_null<DenseElementsAttr>(value);
if (!dense || !dense.isSplat())
return false;
Attribute element = dense.getSplatValue<Attribute>();
if (auto floating = dyn_cast<FloatAttr>(element))
return floating.getValue().isZero() && !floating.getValue().isNegative();
if (auto integer = dyn_cast<IntegerAttr>(element))
return integer.getValue().isZero();
return false;
}
ConstantPool::ConstantPool(Operation *constantAnchor, OpBuilder &builder)
: anchor(constantAnchor), block(getConstantInsertionBlock(constantAnchor)),
builder(builder) {
+3
View File
@@ -12,6 +12,9 @@
namespace onnx_mlir {
// Negative floating zero has a distinct representation and must not match.
bool isZeroBitPatternSplat(mlir::Attribute value);
class ConstantPool {
public:
ConstantPool(mlir::Operation *constantAnchor, mlir::OpBuilder &builder);
+74
View File
@@ -0,0 +1,74 @@
#include "CopyAccessPlan.hpp"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/CheckedArithmetic.h"
#include <algorithm>
namespace onnx_mlir {
mlir::FailureOr<int64_t> getContiguousSuffixRank(
llvm::ArrayRef<int64_t> shape, llvm::ArrayRef<int64_t> elementStrides) {
if (shape.size() != elementStrides.size()
|| llvm::any_of(shape, [](int64_t size) { return size <= 0; }))
return mlir::failure();
int64_t expectedStride = 1, rank = 0;
for (size_t dim = shape.size(); dim-- > 0;) {
if (shape[dim] != 1 && elementStrides[dim] != expectedStride)
break;
++rank;
auto next = llvm::checkedMul(expectedStride, shape[dim]);
if (!next) return mlir::failure();
expectedStride = *next;
}
return rank;
}
mlir::FailureOr<CopyAccessPlan> CopyAccessPlan::build(
llvm::ArrayRef<int64_t> shape, llvm::ArrayRef<int64_t> sourceElementStrides,
llvm::ArrayRef<int64_t> targetElementStrides) {
auto sourceSuffix = getContiguousSuffixRank(shape, sourceElementStrides);
auto targetSuffix = getContiguousSuffixRank(shape, targetElementStrides);
if (mlir::failed(sourceSuffix) || mlir::failed(targetSuffix))
return mlir::failure();
size_t outerRank = shape.size() - std::min(*sourceSuffix, *targetSuffix);
CopyAccessPlan plan;
plan.outerShape.assign(shape.begin(), shape.begin() + outerRank);
plan.sourceOuterStrides.assign(sourceElementStrides.begin(), sourceElementStrides.begin() + outerRank);
plan.targetOuterStrides.assign(targetElementStrides.begin(), targetElementStrides.begin() + outerRank);
int64_t elements = 1, maximumSource = 0, maximumTarget = 0;
for (size_t dim = 0; dim < shape.size(); ++dim) {
if (dim < outerRank && (sourceElementStrides[dim] < 0 || targetElementStrides[dim] < 0))
return mlir::failure();
int64_t &product = dim < outerRank ? plan.runCount : plan.runElements;
auto next = llvm::checkedMul(product, shape[dim]);
auto total = llvm::checkedMul(elements, shape[dim]);
if (!next || !total) return mlir::failure();
product = *next;
elements = *total;
// Unit dimensions never access their stride, including negative strides.
if (shape[dim] != 1) {
auto sourceEnd = llvm::checkedMulAdd(shape[dim] - 1, sourceElementStrides[dim], maximumSource);
auto targetEnd = llvm::checkedMulAdd(shape[dim] - 1, targetElementStrides[dim], maximumTarget);
if (!sourceEnd || !targetEnd) return mlir::failure();
maximumSource = *sourceEnd;
maximumTarget = *targetEnd;
}
}
return plan;
}
mlir::FailureOr<std::pair<int64_t, int64_t>> CopyAccessPlan::getRunOffsets(int64_t run) const {
if (run < 0 || run >= runCount) return mlir::failure();
int64_t source = 0, target = 0;
for (size_t dim = outerShape.size(); dim-- > 0;) {
int64_t coordinate = run % outerShape[dim];
run /= outerShape[dim];
auto nextSource = llvm::checkedMulAdd(coordinate, sourceOuterStrides[dim], source);
auto nextTarget = llvm::checkedMulAdd(coordinate, targetOuterStrides[dim], target);
if (!nextSource || !nextTarget) return mlir::failure();
source = *nextSource;
target = *nextTarget;
}
return std::pair{source, target};
}
} // namespace onnx_mlir
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include "mlir/Support/LogicalResult.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVector.h"
#include <cstdint>
#include <utility>
namespace onnx_mlir {
// Geometry only. Callers must prove storage layout, aliasing, and copy legality.
mlir::FailureOr<int64_t> getContiguousSuffixRank(
llvm::ArrayRef<int64_t> shape, llvm::ArrayRef<int64_t> elementStrides);
class CopyAccessPlan {
public:
// All accessed relative element offsets must fit int64_t, including run tails.
// Byte scaling and addition of base addresses remain caller responsibilities.
static mlir::FailureOr<CopyAccessPlan> build(llvm::ArrayRef<int64_t> shape,
llvm::ArrayRef<int64_t> sourceElementStrides,
llvm::ArrayRef<int64_t> targetElementStrides);
int64_t getRunCount() const { return runCount; }
int64_t getRunElements() const { return runElements; }
llvm::ArrayRef<int64_t> getOuterShape() const { return outerShape; }
llvm::ArrayRef<int64_t> getSourceOuterStrides() const { return sourceOuterStrides; }
llvm::ArrayRef<int64_t> getTargetOuterStrides() const { return targetOuterStrides; }
// Source and target element offsets, relative to their separately proven bases.
mlir::FailureOr<std::pair<int64_t, int64_t>> getRunOffsets(int64_t run) const;
private:
int64_t runCount = 1;
int64_t runElements = 1;
llvm::SmallVector<int64_t> outerShape, sourceOuterStrides, targetOuterStrides;
};
} // namespace onnx_mlir
+14 -2
View File
@@ -1,6 +1,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/CheckedArithmetic.h"
#include <functional>
@@ -9,9 +10,20 @@
namespace onnx_mlir {
llvm::SmallVector<int64_t> computeRowMajorStrides(llvm::ArrayRef<int64_t> shape) {
auto strides = computeRowMajorStridesChecked(shape);
if (mlir::failed(strides))
llvm::report_fatal_error("row-major strides are not representable in int64_t");
return std::move(*strides);
}
mlir::FailureOr<llvm::SmallVector<int64_t>>
computeRowMajorStridesChecked(llvm::ArrayRef<int64_t> shape) {
llvm::SmallVector<int64_t> strides(shape.size(), 1);
for (int64_t dim = static_cast<int64_t>(shape.size()) - 2; dim >= 0; --dim)
strides[dim] = strides[dim + 1] * shape[dim + 1];
for (int64_t dim = static_cast<int64_t>(shape.size()) - 2; dim >= 0; --dim) {
auto stride = llvm::checkedMul(strides[dim + 1], shape[dim + 1]);
if (!stride) return mlir::failure();
strides[dim] = *stride;
}
return strides;
}
+4
View File
@@ -19,7 +19,11 @@ namespace onnx_mlir {
using HSliceId = size_t;
using CoreId = size_t;
// Invariant form fails explicitly on overflow, including assertion-free builds.
// Optional analyses and rewrites must use the checked form.
llvm::SmallVector<int64_t> computeRowMajorStrides(llvm::ArrayRef<int64_t> shape);
mlir::FailureOr<llvm::SmallVector<int64_t>>
computeRowMajorStridesChecked(llvm::ArrayRef<int64_t> shape);
struct TransposeRun {
size_t start;
+3 -1
View File
@@ -191,7 +191,9 @@ resolveWeightView(mlir::Operation* coreLikeOp, mlir::Value weight, const StaticV
ResolvedWeightView view;
view.globalOp = globalOp;
view.shape.assign(denseAttr.getType().getShape().begin(), denseAttr.getType().getShape().end());
view.strides = computeRowMajorStrides(view.shape);
auto checkedStrides = computeRowMajorStridesChecked(view.shape);
if (failed(checkedStrides)) return mlir::failure();
view.strides = std::move(*checkedStrides);
CompiledIndexExpr offsetExpr = makeConstantExpr(0);
for (mlir::Operation* viewOp : llvm::reverse(viewOps)) {
+15
View File
@@ -4,6 +4,21 @@
namespace onnx_mlir::pim {
static constexpr llvm::StringLiteral candidateResourceFailure = "pim.candidate-resource-limit";
mlir::InFlightDiagnostic emitCandidateResourceError(mlir::Operation *op, llvm::StringRef message) {
auto diagnostic = op->emitError(message);
diagnostic.getUnderlyingDiagnostic()->getMetadata().emplace_back(llvm::StringRef(candidateResourceFailure));
return diagnostic;
}
bool isCandidateResourceError(mlir::Diagnostic &diagnostic) {
return llvm::any_of(diagnostic.getMetadata(), [](const mlir::DiagnosticArgument &argument) {
return argument.getKind() == mlir::DiagnosticArgument::DiagnosticArgumentKind::String
&& argument.getAsString() == candidateResourceFailure;
});
}
mlir::InFlightDiagnostic emitUnsupportedStaticShapeDiagnostic(mlir::Operation* op, llvm::StringRef valueDescription) {
return op->emitOpError() << "requires statically shaped " << valueDescription;
}
+4
View File
@@ -12,6 +12,10 @@
namespace onnx_mlir::pim {
/// Only a backend owner can classify a failed schedule as resource-infeasible.
mlir::InFlightDiagnostic emitCandidateResourceError(mlir::Operation *op, llvm::StringRef message);
bool isCandidateResourceError(mlir::Diagnostic &diagnostic);
struct CappedDiagnosticReporter {
explicit CappedDiagnosticReporter(int64_t maxReportedFailures = 8)
: maxReportedFailures(maxReportedFailures) {}
+2
View File
@@ -17,6 +17,8 @@ add_pim_library(OMPimCompilerUtils
PimCompilerUtils.cpp
PimScheduleDiagnostic.cpp
PimPeriodicAnalysis.cpp
PimQueueReporting.cpp
PimSpatialCost.cpp
PimPipelineSelection.cpp
PimArtifactWriter.cpp
PimCodeGen.cpp
+1
View File
@@ -14,6 +14,7 @@ namespace onnx_mlir::pim_binary {
inline constexpr char kMagic[4] = {'P', 'I', 'M', 'B'};
inline constexpr uint32_t kVersion = 1;
inline constexpr int32_t kMatrixWeightPrecisionBits = 8;
inline constexpr uint64_t kCountOffset = 8;
inline constexpr size_t kHeaderSize = 12;
inline constexpr size_t kRecordSize = 20;
+18 -67
View File
@@ -22,7 +22,6 @@
#include <absl/types/compare.h>
#include <algorithm>
#include <cassert>
#include <chrono>
#include <cstdint>
#include <fstream>
#include <limits>
@@ -32,6 +31,7 @@
#include <utility>
#include "Common/IR/CompactAsmUtils.hpp"
#include "Common/IR/ConstantUtils.hpp"
#include "Common/PimCommon.hpp"
#include "Common/Support/CheckedArithmetic.hpp"
#include "Common/Support/Diagnostics.hpp"
@@ -42,7 +42,6 @@
#include "src/Accelerators/PIM/Common/Support/FileSystemUtils.hpp"
#include "src/Accelerators/PIM/Compiler/PimArtifactWriter.hpp"
#include "src/Accelerators/PIM/Compiler/PimBinaryFormat.hpp"
#include "src/Accelerators/PIM/Compiler/PimPeriodicAnalysis.hpp"
#include "src/Accelerators/PIM/Compiler/PimCodeGen.hpp"
#include "src/Accelerators/PIM/Compiler/PimCompilerOptions.hpp"
#include "src/Accelerators/PIM/Compiler/PimCoreProgram.hpp"
@@ -116,15 +115,7 @@ static bool isZeroSplatGlobal(mlir::Value value) {
auto globalOp = moduleOp ? lookupGlobalForGetGlobal(moduleOp, getGlobalOp) : memref::GlobalOp();
if (!globalOp || !globalOp.getConstant() || !globalOp.getInitialValue())
return false;
auto denseAttr = dyn_cast<DenseElementsAttr>(*globalOp.getInitialValue());
if (!denseAttr || !denseAttr.isSplat())
return false;
Attribute valueAttr = denseAttr.getSplatValue<Attribute>();
if (auto floatAttr = dyn_cast<FloatAttr>(valueAttr))
return floatAttr.getValue().isZero() && !floatAttr.getValue().isNegative();
if (auto integerAttr = dyn_cast<IntegerAttr>(valueAttr))
return integerAttr.getValue().isZero();
return false;
return isZeroBitPatternSplat(*globalOp.getInitialValue());
}
// Pim instruction immediates are serialized as signed int32_t fields today
@@ -613,7 +604,7 @@ void PimCodeGen::emitMvmOp(size_t groupId, size_t rdAddr, size_t rdOffset, size_
instruction.opcode = pim_binary::Opcode::mvmul;
instruction.rd = 0;
instruction.r1 = 1;
instruction.r2OrImm = 8;
instruction.r2OrImm = pim_binary::kMatrixWeightPrecisionBits;
instruction.generic1 = 0;
instruction.generic2 = pim::checkedI32OrCrash(groupId, "mvm group id");
emitInstruction(instruction);
@@ -895,7 +886,6 @@ struct CoreEmissionResult {
OnnxMlirCompilerErrorCodes status = CompilerSuccess;
MemoryReportRow reportRow;
std::string instructions;
llvm::SmallVector<ResolvedWeightView, 8> usedWeights;
llvm::SmallVector<DiagnosticRecord, kMaxStoredCodegenDiagnostics> diagnostics;
size_t diagnosticCount = 0;
@@ -1150,10 +1140,8 @@ static OnnxMlirCompilerErrorCodes emitEmptyCoreArtifacts(StringRef outputDirPath
return CompilerSuccess;
}
OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::string& outputDirPath,
PimInstructionProgram *instructions, PimCodegenTiming *timing) {
auto start = std::chrono::steady_clock::now();
if (!instructions && !outputDirPath.empty()) {
OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::string& outputDirPath) {
if (!outputDirPath.empty()) {
if (auto error = sys::fs::create_directory(outputDirPath)) {
errs() << "Error creating output directory: " << outputDirPath << ": " << error.message() << '\n';
return InvalidOutputFileAccess;
@@ -1169,9 +1157,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
memory.hostMem.allocateHost(moduleOp, funcOp);
memory.reportHost();
if (!instructions)
if (auto err = writeMemoryBinary(moduleOp, funcOp, memory, outputDirPath))
return err;
if (auto err = writeMemoryBinary(moduleOp, funcOp, memory, outputDirPath))
return err;
json::Object xbarsPerArrayGroup;
size_t maxCoreId = 0;
@@ -1301,22 +1288,16 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
std::error_code errorCode;
auto outputCorePath = outputDirPath + "/core_" + std::to_string(job.physicalCoreId) + ".pim";
llvm::SmallVector<char, 0> binary;
raw_svector_ostream memoryStream(binary);
std::unique_ptr<raw_fd_ostream> fileStream;
if (!instructions) {
fileStream = std::make_unique<raw_fd_ostream>(outputCorePath, errorCode, sys::fs::OF_None);
if (errorCode) {
errs() << "Error opening core file: " << outputCorePath << ": " << errorCode.message() << '\n';
result.status = InvalidOutputFileAccess;
return result;
}
auto fileStream = std::make_unique<raw_fd_ostream>(outputCorePath, errorCode, sys::fs::OF_None);
if (errorCode) {
errs() << "Error opening core file: " << outputCorePath << ": " << errorCode.message() << '\n';
result.status = InvalidOutputFileAccess;
return result;
}
raw_pwrite_stream &coreBinaryStream = instructions
? static_cast<raw_pwrite_stream &>(memoryStream) : *fileStream;
raw_pwrite_stream &coreBinaryStream = *fileStream;
std::unique_ptr<raw_fd_ostream> coreJsonStream;
if (!instructions && pimEmitJson.getValue()) {
if (pimEmitJson.getValue()) {
std::string outputCoreJsonPath = outputDirPath + "/core_" + std::to_string(job.physicalCoreId) + ".json";
errorCode = std::error_code();
coreJsonStream = std::make_unique<raw_fd_ostream>(outputCoreJsonPath, errorCode);
@@ -1332,12 +1313,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
PimCodeGen coreCodeGen(jobMemory, instructionWriter, coreJsonStream.get());
auto finalizeInstructions = [&]() {
bool succeeded = mlir::succeeded(instructionWriter.finalize());
if (fileStream) {
fileStream->close();
if (fileStream->has_error()) { fileStream->clear_error(); succeeded = false; }
} else {
result.instructions.assign(binary.begin(), binary.end());
}
fileStream->close();
if (fileStream->has_error()) { fileStream->clear_error(); succeeded = false; }
return succeeded;
};
@@ -1390,14 +1367,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
};
std::vector<CoreEmissionResult> jobResults(jobs.size());
auto emissionStart = std::chrono::steady_clock::now();
if (timing)
timing->preparationSeconds = std::chrono::duration<double>(emissionStart - start).count();
mlir::parallelFor(
moduleOp.getContext(), 0, jobs.size(), [&](size_t index) { jobResults[index] = emitJob(jobs[index]); });
if (timing)
timing->emissionSeconds = std::chrono::duration<double>(
std::chrono::steady_clock::now() - emissionStart).count();
pim::CappedDiagnosticReporter diagnostics;
Operation* summaryAnchor = nullptr;
@@ -1427,33 +1398,13 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
physicalCoreId < activePhysicalCores.size(); ++physicalCoreId) {
if (activePhysicalCores[physicalCoreId])
continue;
if (!instructions)
if (auto err = emitEmptyCoreArtifacts(outputDirPath, physicalCoreId))
return err;
if (auto err = emitEmptyCoreArtifacts(outputDirPath, physicalCoreId))
return err;
xbarsPerArrayGroup["core" + std::to_string(physicalCoreId)] =
json::Array {};
memory.recordCoreReport(physicalCoreId, MemoryReportRow {});
}
if (instructions) {
instructions->instructions.resize(maxPhysicalCoreId + 1);
instructions->arrayGroups.resize(maxPhysicalCoreId + 1);
for (size_t core = 0; core <= maxPhysicalCoreId; ++core) {
instructions->arrayGroups[core].clear();
llvm::SmallVector<char> empty;
raw_svector_ostream stream(empty);
pim_binary::writeHeader(stream);
instructions->instructions[core].assign(empty.begin(), empty.end());
}
for (size_t i = 0; i < jobs.size(); ++i) {
size_t core = jobs[i].physicalCoreId;
instructions->instructions[core] = std::move(jobResults[i].instructions);
for (const auto &weight : jobResults[i].usedWeights)
instructions->arrayGroups[core].push_back(weight.shape[1] / crossbarSize.getValue());
}
return CompilerSuccess;
}
llvm::SmallVector<WeightFileRequest, 8> weightRequests;
weightRequests.reserve(jobs.size());
for (size_t jobIndex = 0; jobIndex < jobs.size(); ++jobIndex) {
+1 -8
View File
@@ -25,7 +25,6 @@
namespace onnx_mlir {
struct CompiledCoreProgram;
struct PimInstructionProgram;
class PimInstructionWriter;
struct MemEntry {
@@ -227,13 +226,7 @@ public:
};
struct PimCodegenTiming {
double preparationSeconds = 0;
double emissionSeconds = 0;
};
OnnxMlirCompilerErrorCodes compileToPimCode(mlir::ModuleOp& moduleOpRef, std::string& outputDirName,
PimInstructionProgram *instructions = nullptr, PimCodegenTiming *timing = nullptr);
OnnxMlirCompilerErrorCodes compileToPimCode(mlir::ModuleOp& moduleOpRef, std::string& outputDirName);
} // namespace onnx_mlir
+1 -1
View File
@@ -90,7 +90,7 @@ llvm::cl::opt<bool> pimReportConvLowering("pim-report-conv-lowering",
llvm::cl::cat(OnnxMlirOptions));
llvm::cl::opt<bool> pimPeriodicDiagnostics("pim-periodic-diagnostics",
llvm::cl::desc("Emit the optional pre-codegen recurrence timing report"),
llvm::cl::desc("Emit optional Spatial recurrence and execution-work reports"),
llvm::cl::init(false), llvm::cl::cat(OnnxMlirOptions));
llvm::cl::opt<bool> pimEmitJson("pim-emit-json",
+1 -1
View File
@@ -16,7 +16,7 @@
#include "src/Accelerators/PIM/Compiler/PimCompilerOptions.hpp"
#include "PimPipelineSelection.hpp"
#include "PimInstructionTiming.hpp"
#include "PimPeriodicAnalysis.hpp"
#include "PimSpatialCost.hpp"
#include "src/Accelerators/PIM/Compiler/PimCompilerUtils.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
+32
View File
@@ -0,0 +1,32 @@
#pragma once
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/SmallVector.h"
#include <cassert>
#include <cstdint>
namespace onnx_mlir {
// Emptiness is normalized after consumption. Capacity and consumed prefixes
// cannot affect virtual timing, readiness, or the remaining FIFO sequence.
class PimDependencyTokens {
llvm::SmallVector<uint64_t, 1> values;
size_t head = 0;
public:
void initialize(size_t distance) { values.assign(distance, 0); head = 0; }
bool empty() const { return values.empty(); }
void push(uint64_t time) { values.push_back(time); }
uint64_t consume() {
assert(!empty());
uint64_t time = values[head++];
if (head == values.size()) { values.clear(); head = 0; }
else if (head >= values.size() - head) {
values.erase(values.begin(), values.begin() + head);
head = 0;
}
return time;
}
llvm::ArrayRef<uint64_t> remaining() const { return llvm::ArrayRef(values).drop_front(head); }
};
} // namespace onnx_mlir
+9 -6
View File
@@ -6,6 +6,7 @@ namespace onnx_mlir {
// PIM transport and instruction timing. Spatial consumes only the base costs.
struct PimInstructionTiming : spatial::SchedulingTarget {
Cost matrixCellBits = 0;
bool peerPermissionThenSourceRead = false;
Cost signalIssueNs = 0;
Cost controlMessageBytes = 0;
@@ -22,15 +23,17 @@ struct PimInstructionTiming : spatial::SchedulingTarget {
&& localMemoryWidthBytes && transferWidthBytes;
}
Cost getLinkServiceNs(Cost bytes, size_t source, size_t destination) const {
Cost flits = bytes / transferWidthBytes + (bytes % transferWidthBytes != 0);
return checkedMultiply(flits,
template <typename Arithmetic = SchedulingCostArithmetic>
Cost getLinkServiceNs(Cost bytes, size_t source, size_t destination, Arithmetic math = {}) const {
Cost flits = math.divideUp(bytes, transferWidthBytes);
return math.multiply(flits,
getInterProcessorLatencyNs(source, destination));
}
Cost getSharedMemoryLinkServiceNs(Cost bytes, size_t core, bool response) const {
Cost flits = bytes / transferWidthBytes + (bytes % transferWidthBytes != 0);
return checkedMultiply(flits, response ? sharedMemoryResponseLatencyNs[core]
template <typename Arithmetic = SchedulingCostArithmetic>
Cost getSharedMemoryLinkServiceNs(Cost bytes, size_t core, bool response, Arithmetic math = {}) const {
Cost flits = math.divideUp(bytes, transferWidthBytes);
return math.multiply(flits, response ? sharedMemoryResponseLatencyNs[core]
: sharedMemoryRequestLatencyNs[core]);
}
File diff suppressed because it is too large Load Diff
+96 -2
View File
@@ -1,14 +1,65 @@
#pragma once
#include "llvm/Support/JSON.h"
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp"
#include <cstdint>
#include <string>
#include <vector>
namespace onnx_mlir {
namespace spatial { struct SchedulingTarget; }
struct PimInstructionTiming;
void configurePimScheduleCosts(spatial::SchedulingTarget &target, PimInstructionTiming timing);
enum class PimQueueTermination {
NotRun, IntervalConverged, RecurrenceProved, EventBudget, GenerationLimit,
HistoryLimit, NonMonotonic, NoReadyEvents
};
struct PimQueueEventLocation {
int64_t core = 0;
uint32_t pc = 0;
std::string kind;
};
struct PimQueueProgress {
size_t processedEvents = 0, completedGenerations = 0;
uint64_t maximumQueueWaitNs = 0;
std::optional<PimQueueEventLocation> maximumQueueRequest, generationLimit;
std::vector<std::pair<int64_t, double>> perCoreQueueWaitNs;
std::vector<uint64_t> completedIntervalsNs;
};
struct PimQueueIntervalWindow {
std::optional<double> intervalNs;
std::string reason;
PimQueueTermination termination = PimQueueTermination::NotRun;
PimQueueProgress progress;
};
struct PimQueuePolicyResult {
PimQueueIntervalWindow legacy;
spatial::ScheduleRecurrenceEvidence recurrence;
PimQueueProgress progress;
PimQueueTermination termination = PimQueueTermination::NotRun;
size_t checkpoints = 0, historyBytes = 0, stateWords = 0, proofProcessedEvents = 0;
bool reverseTies = false;
std::optional<double> intervalNs() const;
};
struct PimQueueEvaluation {
std::array<PimQueuePolicyResult, 2> policies;
uint64_t memoryServiceNs = 0;
double evaluationSeconds = 0;
bool recurrenceRequired = false, details = false, measureTime = false;
spatial::WholeScheduleCost cost() const;
};
llvm::json::Object serializePimQueueEvaluation(const PimQueueEvaluation &evaluation);
struct PimPeriodicEstimate {
spatial::WholeScheduleCost cost;
llvm::json::Object report;
std::optional<PimQueueEvaluation> queue;
};
// Binary instructions, indexed by physical core. No executable or measured data.
struct PimInstructionProgram {
@@ -16,15 +67,58 @@ struct PimInstructionProgram {
std::vector<std::vector<int64_t>> arrayGroups;
};
// Hardware work only. Bytes are already in the target timing precision.
// Repeat keeps ordered request patterns compact until queue construction.
struct PimExecutionEvent {
enum Kind { Work, Load, Store, Send, Receive, Signal, Wait, Repeat };
enum WorkClass { Matrix, Vector, LocalCopy, Fill, VectorCopy, ScalarIssue };
Kind kind = Work;
uint64_t amount = 0;
int64_t peer = 0, reg = 0, count = 0;
std::vector<PimExecutionEvent> body;
bool constant = false;
WorkClass workClass = Fill;
PimExecutionEvent(Kind kind, uint64_t amount, int64_t peer = 0,
int64_t reg = 0, int64_t count = 0, std::vector<PimExecutionEvent> body = {})
: kind(kind), amount(amount), peer(peer), reg(reg), count(count), body(std::move(body)) {}
};
using PimExecutionProgram = std::vector<std::vector<PimExecutionEvent>>;
struct PimPeriodicOptions {
bool networkQueues = true;
bool instructionIssue = true;
bool scalarIssue = true;
bool ignoreLocalCopies = false;
bool workBreakdown = false;
bool measureTime = false;
// Optional ordered work/IO extraction for timing validation; no execution.
PimExecutionProgram *executionTrace = nullptr;
// Prove recurrence of the modeled state. Instruction diagnostics retain
// their legacy interval-window result separately.
bool verifyPeriodicState = false;
// Optional diagnostic cap. The complete-state proof keeps its original budget.
std::optional<size_t> intervalWindowEventBudget;
bool memoryController = true;
bool detailed = false;
// Apply the oracle's fp32-to-int8 timing projection without changing the program.
bool oraclePrecision = false;
};
llvm::json::Object analyzePimExecution(const PimExecutionProgram &program,
const PimInstructionTiming &target, PimPeriodicOptions options = {});
PimPeriodicEstimate evaluatePimExecution(const PimExecutionProgram &program,
const PimInstructionTiming &target, PimPeriodicOptions options = {});
uint64_t getPimLocalTransferWorkNs(const PimInstructionTiming &target,
uint64_t bytes, bool read, bool write, bool instructionIssue = true);
uint64_t getPimVectorWorkNs(const PimInstructionTiming &target,
uint64_t elements, uint64_t inputBytes, uint64_t outputBytes,
bool binary, bool instructionIssue = true);
uint64_t getPimMatrixOutputBytes(const PimInstructionTiming &target,
uint64_t cellBits, uint64_t weightBits, uint64_t groups, uint64_t outputBytes);
uint64_t getPimMatrixWorkNs(const PimInstructionTiming &target,
uint64_t inputBits, uint64_t outputBytes, bool instructionIssue = true);
llvm::json::Object analyzePimInstructions(const PimInstructionProgram &program,
const PimInstructionTiming &target, int64_t cellBits,
PimPeriodicOptions options = {});
+236 -128
View File
@@ -13,7 +13,7 @@
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/JSON.h"
#include "llvm/Support/raw_os_ostream.h"
#include <algorithm>
#include "src/Accelerators/PIM/Common/Support/Diagnostics.hpp"
#include <chrono>
using namespace mlir;
@@ -57,6 +57,14 @@ struct CandidateTiming final : PassInstrumentation {
}
};
void writeSelectionReport(llvm::json::Object report) {
auto file = openReportFileWithExtension("pipeline_selection", "json");
if (file) {
llvm::raw_os_ostream out(file);
out << llvm::formatv("{0:2}\n", llvm::json::Value(std::move(report)));
}
}
struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, OperationPass<ModuleOp>> {
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(PimPipelineSelectionPass)
PimPipelineSelectionPass(const SchedulingTarget &target, const SpatialTargetResources &resources,
@@ -68,156 +76,256 @@ struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, Op
SpatialDataflowExportStage exportStage;
StringRef getArgument() const override { return "pim-pipeline-selection"; }
StringRef getDescription() const override { return "Select a throughput pipeline using static instruction queue analysis"; }
StringRef getDescription() const override { return "Select a throughput pipeline using static Spatial queue analysis"; }
LogicalResult lower(ModuleOp module, llvm::json::Object *summary = nullptr) {
PassManager pm(module.getContext());
addPimLoweringPasses(pm, resources, EmitPimCodegen, /*reportProgress=*/false);
if (enableTiming && summary)
pm.addInstrumentation(std::make_unique<CandidateTiming>(*summary));
return pm.run(module);
}
LogicalResult evaluate(ModuleOp checkpoint, const MergeScheduleResult &schedule,
OwningOpRef<ModuleOp> &candidate,
std::string &scheduleReport, llvm::json::Object &summary) {
LogicalResult realizeCandidate(ModuleOp checkpoint, const MergeScheduleResult &schedule,
OwningOpRef<ModuleOp> &candidate, std::string &scheduleReport,
llvm::json::Object &summary) {
auto begin = Clock::now();
IRMapping mapping;
candidate = cast<ModuleOp>(checkpoint->clone(mapping));
auto remapped = remapSpatialSchedule(schedule, mapping);
if (failed(realizeSpatialSchedule(*candidate, remapped, target, stages,
SpatialDataflowExportStage::None, nullptr, &scheduleReport))
|| failed(lower(*candidate, &summary))) return failure();
if (enableTiming) summary["codegen_input_ir"] = countIR(candidate->getOperation());
summary["lowering_seconds"] = elapsed(begin);
SpatialDataflowExportStage::None, nullptr, &scheduleReport))) return failure();
summary["realization_seconds"] = elapsed(begin);
return success();
}
WholeScheduleCost spatialCost(ModuleOp candidate, llvm::json::Object &summary) {
WholeScheduleCost cost;
auto begin = Clock::now();
llvm::json::Object estimate;
if (target.estimateRealizedScheduleCost)
cost = target.estimateRealizedScheduleCost(candidate, estimate);
else cost.reason = target.wholeScheduleCostUnavailable.empty()
? "target has no realized-Spatial cost query" : target.wholeScheduleCostUnavailable;
cost.selectionMargin = kPipelineSelectionMargin;
estimate["safe_for_selection"] = cost.eligible();
estimate["diagnostic_only"] = false;
summary["spatial_cost"] = std::move(estimate);
summary["spatial_cost_seconds"] = elapsed(begin);
return cost;
}
LogicalResult evaluateSpatialCandidates(ModuleOp checkpoint,
const PreparedSpatialSchedules &prepared, std::vector<WholeScheduleCost> &costs,
llvm::json::Object &report) {
llvm::json::Array summaries;
double realization = 0, extraction = 0, expansion = 0, queue = 0;
std::string errors;
for (size_t i = 0; i < costs.size(); ++i) {
llvm::json::Object summary{{"index", static_cast<int64_t>(i)}};
OwningOpRef<ModuleOp> candidate;
std::string scheduleReport;
{
ScopedCompilerOutputSuppression silence;
if (failed(realizeCandidate(checkpoint, prepared.candidates[i], candidate, scheduleReport, summary)))
costs[i].reason = "Spatial schedule realization failed verification";
else costs[i] = spatialCost(*candidate, summary);
}
realization += summary.getNumber("realization_seconds").value_or(0);
extraction += costs[i].timing.extractionSeconds;
expansion += costs[i].timing.expansionSeconds;
queue += costs[i].timing.evaluationSeconds;
bool supported = costs[i].loweringSupport == StaticLoweringSupport::Supported;
auto convergence = costs[i].convergenceProved();
auto support = costs[i].loweringSupported();
bool converged = convergence == true;
bool safe = costs[i].eligible();
summary["cost_available"] = costs[i].available();
summary["convergence_proved"] = convergence
? llvm::json::Value(*convergence) : llvm::json::Value(nullptr);
summary["lowering_supported"] = support
? llvm::json::Value(*support) : llvm::json::Value(nullptr);
summary["eligible"] = safe;
if (costs[i].primary) summary["interval_ns"] = *costs[i].primary;
if (costs[i].alternative) summary["alternative_tie_interval_ns"] = *costs[i].alternative;
if (!safe) {
std::string property;
if (!costs[i].available()) property += "unavailable timing; ";
if (!supported) property += "unsupported or unknown static lowering; ";
if (!converged) property += "unproved modeled-state convergence; ";
if (property.empty()) property = "unavailable selection margin";
summary["reason"] = property + ": " + costs[i].reason;
errors += "\ncandidate " + std::to_string(i) + ": " + property + ": " + costs[i].reason;
}
summaries.push_back(std::move(summary));
}
report["candidates"] = std::move(summaries);
report["candidate_realization_seconds"] = realization;
report["event_extraction_seconds"] = extraction;
report["event_expansion_seconds"] = expansion;
report["queue_seconds"] = queue;
if (!errors.empty()) {
report["status"] = "incomplete-costs";
report["reason"] = errors;
checkpoint.emitError("Spatial pipeline cannot compare the complete candidate set:") << errors;
return failure();
}
return success();
}
void runOnOperation() override {
ModuleOp module = getOperation();
ModuleOp checkpoint = getOperation();
auto start = Clock::now();
std::string unsupported;
if (!target.prepareScheduleCost)
unsupported = target.wholeScheduleCostUnavailable.empty()
? "target has no whole-schedule cost query" : target.wholeScheduleCostUnavailable;
auto prepared = prepareSpatialSchedules(module, target, stages, unsupported.empty());
auto prepared = prepareSpatialSchedules(checkpoint, target, stages, /*collectCandidates=*/true);
if (failed(prepared)) { signalPassFailure(); return; }
bool singleCandidate = unsupported.empty() && prepared->candidates.size() == 1;
double generationSeconds = elapsed(start);
std::vector<WholeScheduleCost> scores(prepared->candidates.size());
llvm::json::Array summaries;
OwningOpRef<ModuleOp> bestModule;
std::string bestScheduleReport;
size_t bestIndex = 0;
double loweringSeconds = 0, instructionSeconds = 0, queueSeconds = 0;
size_t lowered = 0, analyzed = 0, unavailable = 0, duplicates = 0;
for (size_t i = 0; i < prepared->candidates.size(); ++i) {
llvm::json::Object summary{{"index", static_cast<int64_t>(i)}};
if (!unsupported.empty()) {
scores[i].reason = unsupported;
} else if (singleCandidate) {
scores[i].reason = "one distinct generated schedule; no cost comparison required";
} else {
ScopedCompilerOutputSuppression silence;
OwningOpRef<ModuleOp> candidate;
std::string scheduleReport;
if (failed(evaluate(module, prepared->candidates[i], candidate, scheduleReport, summary))) {
if (!i) { signalPassFailure(); return; }
scores[i].reason = "candidate lowering failed verification";
} else {
++lowered;
loweringSeconds += *summary.getNumber("lowering_seconds");
auto query = target.prepareScheduleCost(*candidate, summary);
if (failed(query)) {
if (!i) { signalPassFailure(); return; }
scores[i].reason = "candidate cost preparation failed verification";
++unavailable;
summary["eligible"] = false;
summary["reason"] = scores[i].reason;
summaries.push_back(std::move(summary));
continue;
}
instructionSeconds += summary.getNumber("instruction_generation_seconds").value_or(0);
scores[i].identity = query->identity;
auto duplicate = std::find_if(scores.begin(), scores.begin() + i,
[&](const WholeScheduleCost &other) { return other.identity == scores[i].identity; });
if (duplicate != scores.begin() + i) {
++duplicates;
scores[i].reason = "duplicate emitted instruction program";
summary["duplicate_of"] = static_cast<int64_t>(duplicate - scores.begin());
} else {
auto begin = Clock::now();
scores[i] = query->evaluate(summary);
scores[i].identity = query->identity;
double seconds = elapsed(begin);
queueSeconds += seconds;
++analyzed;
summary["queue_seconds"] = seconds;
}
std::string partialReason;
size_t partialSelection = selectPipelineCandidate(
llvm::ArrayRef<WholeScheduleCost>(scores.data(), i + 1), partialReason);
if (!bestModule || partialSelection == i) {
bestIndex = i;
bestModule = std::move(candidate);
bestScheduleReport = std::move(scheduleReport);
}
}
}
if (!singleCandidate && !scores[i].eligible()
&& scores[i].reason != "duplicate emitted instruction program") ++unavailable;
summary["eligible"] = scores[i].eligible();
summary["identity"] = scores[i].identity;
if (scores[i].primary) summary["interval_ns"] = *scores[i].primary;
if (scores[i].alternative) summary["alternative_tie_interval_ns"] = *scores[i].alternative;
if (scores[i].relativeUncertainty)
summary["relative_prediction_uncertainty"] = *scores[i].relativeUncertainty;
if (!scores[i].reason.empty()) summary["reason"] = scores[i].reason;
summaries.push_back(std::move(summary));
if (prepared->candidates.empty()) {
checkpoint.emitError("Spatial pipeline requires at least one candidate schedule");
signalPassFailure(); return;
}
bool single = prepared->candidates.size() == 1;
llvm::json::Object report{{"strategy", "spatial-queue-aware"}, {"static_analysis", true},
{"candidate_generation_seconds", elapsed(start)},
{"generated_candidates", static_cast<int64_t>(prepared->candidates.size())},
{"single_candidate_analysis_skipped", single}, {"selection_margin", kPipelineSelectionMargin},
{"prediction_error_bound_available", false}, {"temporary_disk_bytes", 0}};
std::vector<WholeScheduleCost> costs(prepared->candidates.size());
if (!single && failed(evaluateSpatialCandidates(checkpoint, *prepared, costs, report))) {
writeSelectionReport(std::move(report)); signalPassFailure(); return;
}
if (single) report["candidates"] = llvm::json::Array{llvm::json::Object{{"index", 0}}};
OwningOpRef<ModuleOp> winner;
std::string scheduleReport;
llvm::json::Array attempts;
double realizationSeconds = 0, backendSeconds = 0;
size_t lowered = 0;
auto attempt = [&](size_t index) {
llvm::json::Object summary{{"candidate", static_cast<int64_t>(index)}};
std::string capturedReport;
OwningOpRef<ModuleOp> candidate;
LogicalResult realized = failure();
{
ScopedCompilerOutputSuppression silence;
realized = realizeCandidate(checkpoint, prepared->candidates[index], candidate, capturedReport, summary);
}
PimCandidateResult result = PimCandidateResult::Failure;
if (failed(realized)) {
summary["reason"] = "selected Spatial schedule failed deterministic realization";
summary["status"] = "realization-failed";
} else {
++lowered;
result = lowerPimPipelineCandidate(*candidate, resources, summary);
}
realizationSeconds += summary.getNumber("realization_seconds").value_or(0);
backendSeconds += summary.getNumber("backend_seconds").value_or(0);
if (result == PimCandidateResult::Success) {
summary["status"] = "lowered";
winner = std::move(candidate);
scheduleReport = std::move(capturedReport);
} else if (result == PimCandidateResult::ResourceLimit)
llvm::errs() << "Spatial pipeline candidate " << index
<< " rejected by a backend resource limit; rejection recorded in pipeline_selection.json\n";
attempts.push_back(std::move(summary));
return result;
};
std::string reason;
size_t selected = selectPipelineCandidate(scores, reason);
if (singleCandidate) reason = scores.front().reason;
else if (!unsupported.empty()) reason += ": " + unsupported;
else if (!scores.front().eligible() && !scores.front().reason.empty())
reason += ": " + scores.front().reason;
auto finalizeStart = Clock::now();
std::optional<size_t> selected;
if (single) {
if (attempt(0) == PimCandidateResult::Success) selected = 0;
reason = selected ? "one distinct schedule; no cost comparison required"
: "the single schedule failed backend lowering; no alternatives are available";
} else selected = lowerSelectedPimCandidate(costs, attempt, reason);
report["backend_attempts"] = std::move(attempts);
report["lowered_candidates"] = static_cast<int64_t>(lowered);
report["costed_candidates"] = single ? 0 : static_cast<int64_t>(costs.size());
size_t attemptCount = report.getArray("backend_attempts")->size();
report["backend_retries"] = static_cast<int64_t>(attemptCount ? attemptCount - 1 : 0);
report["winner_realization_seconds"] = realizationSeconds;
report["backend_lowering_seconds"] = backendSeconds;
if (!selected) {
report["status"] = "backend-failure";
report["reason"] = reason;
std::string failures;
for (auto &entry : *report.getArray("backend_attempts")) {
auto *failed = entry.getAsObject();
failures += "\ncandidate " + std::to_string(*failed->getInteger("candidate"))
+ ": " + failed->getString("reason").value_or("backend failure").str();
}
checkpoint.emitError("Spatial pipeline backend selection failed; ") << reason << failures;
writeSelectionReport(std::move(report)); signalPassFailure(); return;
}
auto finalize = Clock::now();
bool spatialArtifacts = exportStage != SpatialDataflowExportStage::None
|| pimPeriodicDiagnostics || isPimIrDumpEnabled();
// Explicit Spatial diagnostics require the pre-lowering representation.
if (!bestModule || spatialArtifacts) {
if (failed(realizeSpatialSchedule(module, prepared->candidates[selected], target,
stages, exportStage, pimPeriodicDiagnostics ? &prepared->graph : nullptr))) { signalPassFailure(); return; }
|| pimPeriodicDiagnostics || isPimIrDumpEnabled();
if (spatialArtifacts) {
if (failed(realizeSpatialSchedule(checkpoint, prepared->candidates[*selected], target,
stages, exportStage, pimPeriodicDiagnostics ? &prepared->graph : nullptr))) {
signalPassFailure(); return;
}
} else {
auto file = openDialectDumpFileWithExtension("spatial3_scheduled_no_comm", "/reports", "txt");
if (file) file << bestScheduleReport;
if (file) file << scheduleReport;
}
if (bestModule) {
assert(selected == bestIndex && "retained module must be the selected candidate");
module->setAttrs((*bestModule)->getAttrs());
module.getBodyRegion().takeBody(bestModule->getBodyRegion());
} else if (failed(lower(module))) { signalPassFailure(); return; }
llvm::errs() << "Queue-aware pipeline: candidate " << selected << " of " << scores.size()
checkpoint->setAttrs((*winner)->getAttrs());
checkpoint.getBodyRegion().takeBody(winner->getBodyRegion());
report["selection"] = static_cast<int64_t>(*selected);
report["status"] = "selected";
report["reason"] = reason;
report["finalization_seconds"] = elapsed(finalize);
report["total_seconds"] = elapsed(start);
llvm::errs() << "Spatial queue-aware pipeline: candidate " << *selected << " of " << costs.size()
<< "; " << reason << "\n";
llvm::json::Object report{{"selection", static_cast<int64_t>(selected)}, {"reason", reason},
{"static_analysis", true}, {"strategy", "queue-aware"}, {"candidates", std::move(summaries)},
{"single_candidate_analysis_skipped", singleCandidate},
{"candidate_generation_seconds", generationSeconds}, {"candidate_lowering_seconds", loweringSeconds},
{"instruction_generation_seconds", instructionSeconds},
{"queue_seconds", queueSeconds}, {"finalization_seconds", elapsed(finalizeStart)}, {"total_seconds", elapsed(start)},
{"generated_candidates", static_cast<int64_t>(scores.size())},
{"lowered_candidates", static_cast<int64_t>(lowered)},
{"reused_lowered_candidate", static_cast<bool>(bestModule)},
{"reconstructed_spatial_artifacts", spatialArtifacts && static_cast<bool>(bestModule)},
{"analyzed_candidates", static_cast<int64_t>(analyzed)},
{"unavailable_estimates", static_cast<int64_t>(unavailable)},
{"duplicate_programs", static_cast<int64_t>(duplicates)}, {"temporary_disk_bytes", 0}};
auto file = openReportFileWithExtension("pipeline_selection", "json");
if (file) { llvm::raw_os_ostream out(file); out << llvm::formatv("{0:2}\n", llvm::json::Value(std::move(report))); }
writeSelectionReport(std::move(report));
}
};
} // namespace
PimCandidateResult onnx_mlir::lowerPimPipelineCandidate(ModuleOp module,
const SpatialTargetResources &resources, llvm::json::Object &summary) {
bool classified = true, anyError = false;
std::string diagnostic;
size_t count = 0;
ScopedDiagnosticHandler handler(module.getContext(), [&](Diagnostic &message) {
if (message.getSeverity() == DiagnosticSeverity::Error) {
anyError = true;
classified &= pim::isCandidateResourceError(message);
if (++count <= 8) diagnostic += message.str().substr(0, 2048) + "\n";
}
return success();
});
auto start = Clock::now();
PassManager pm(module.getContext());
addPimLoweringPasses(pm, resources, EmitPimCodegen, /*reportProgress=*/false);
if (enableTiming) pm.addInstrumentation(std::make_unique<CandidateTiming>(summary));
LogicalResult result = pm.run(module);
summary["backend_seconds"] = elapsed(start);
if (succeeded(result)) return PimCandidateResult::Success;
summary["reason"] = diagnostic.empty() ? "backend failed without a classified diagnostic" : diagnostic;
bool resource = anyError && classified;
summary["status"] = resource ? "resource-limit" : "systemic-or-unclassified-error";
return resource ? PimCandidateResult::ResourceLimit : PimCandidateResult::Failure;
}
std::optional<size_t> onnx_mlir::lowerSelectedPimCandidate(llvm::ArrayRef<WholeScheduleCost> costs,
const std::function<PimCandidateResult(size_t)> &lower, std::string &reason) {
if (costs.empty()) {
reason = "Spatial pipeline requires at least one candidate schedule";
return std::nullopt;
}
for (const auto &cost : costs)
if (!cost.eligible()) {
reason = "cannot compare the complete candidate set: unavailable cost or unproved eligibility";
return std::nullopt;
}
std::vector<WholeScheduleCost> remaining(costs.begin(), costs.end());
std::vector<size_t> indices;
for (size_t i = 0; i < costs.size(); ++i) indices.push_back(i);
while (!indices.empty()) {
size_t selected = selectPipelineCandidate(remaining, reason);
auto result = lower(indices[selected]);
if (result == PimCandidateResult::Success) return indices[selected];
if (result != PimCandidateResult::ResourceLimit) {
reason = "systemic or unclassified backend failure; recovery is not justified";
return std::nullopt;
}
remaining.erase(remaining.begin() + selected);
indices.erase(indices.begin() + selected);
}
reason = "all candidate schedules failed backend resource constraints";
return std::nullopt;
}
std::unique_ptr<Pass> onnx_mlir::createPimPipelineSelectionPass(
const SchedulingTarget &target, const SpatialTargetResources &resources,
size_t stages, SpatialDataflowExportStage exportStage) {
+14 -1
View File
@@ -1,8 +1,21 @@
#pragma once
#include "mlir/Pass/Pass.h"
#include "llvm/ADT/ArrayRef.h"
#include <functional>
#include <optional>
#include <string>
namespace mlir { class ModuleOp; }
namespace llvm::json { class Object; }
namespace onnx_mlir {
namespace spatial { struct SchedulingTarget; struct SpatialTargetResources; enum class SpatialDataflowExportStage; }
namespace spatial { struct SchedulingTarget; struct SpatialTargetResources; struct WholeScheduleCost; enum class SpatialDataflowExportStage; }
enum class PimCandidateResult { Success, ResourceLimit, Failure };
PimCandidateResult lowerPimPipelineCandidate(mlir::ModuleOp module,
const spatial::SpatialTargetResources &resources, llvm::json::Object &summary);
// Callback failures distinguish owner-classified resource limits from other errors.
std::optional<size_t> lowerSelectedPimCandidate(llvm::ArrayRef<spatial::WholeScheduleCost> costs,
const std::function<PimCandidateResult(size_t)> &lower, std::string &reason);
std::unique_ptr<mlir::Pass> createPimPipelineSelectionPass(
const spatial::SchedulingTarget &target, const spatial::SpatialTargetResources &resources,
size_t stages, spatial::SpatialDataflowExportStage exportStage);
+125
View File
@@ -0,0 +1,125 @@
#include "PimPeriodicAnalysis.hpp"
using namespace onnx_mlir;
namespace {
const char *windowStatus(const PimQueuePolicyResult &policy) {
if (policy.legacy.intervalNs) return "estimated";
return policy.legacy.termination == PimQueueTermination::NotRun && policy.recurrence.proved()
? "not-required" : "unavailable";
}
const char *terminationName(PimQueueTermination termination) {
switch (termination) {
case PimQueueTermination::NotRun: return "not-run";
case PimQueueTermination::IntervalConverged: return "interval-converged";
case PimQueueTermination::RecurrenceProved: return "recurrence-proved";
case PimQueueTermination::EventBudget: return "event-budget";
case PimQueueTermination::GenerationLimit: return "generation-limit";
case PimQueueTermination::HistoryLimit: return "history-limit";
case PimQueueTermination::NonMonotonic: return "non-monotonic";
case PimQueueTermination::NoReadyEvents: return "no-ready-events";
}
llvm_unreachable("invalid queue termination");
}
void serializeProgress(llvm::json::Object &report, const PimQueueProgress &progress,
bool estimated, bool details, bool intervals) {
if (progress.processedEvents) report["processed_events"] = progress.processedEvents;
if (estimated) {
report["completed_generations"] = progress.completedGenerations;
report["maximum_queue_wait_ns_including_bootstrap"] = progress.maximumQueueWaitNs;
if (progress.maximumQueueRequest) {
report["maximum_queue_core"] = progress.maximumQueueRequest->core;
report["maximum_queue_pc"] = progress.maximumQueueRequest->pc;
}
if (details) {
llvm::json::Array cores;
for (auto [core, wait] : progress.perCoreQueueWaitNs)
cores.push_back(llvm::json::Object{{"core", core}, {"queue_wait_ns", wait}});
report["per_core"] = std::move(cores);
}
}
if (progress.generationLimit) {
report["generation_limit_core"] = progress.generationLimit->core;
report["generation_limit_pc"] = progress.generationLimit->pc;
report["generation_limit_kind"] = progress.generationLimit->kind;
}
if (intervals) {
llvm::json::Array values;
for (uint64_t value : progress.completedIntervalsNs) values.push_back(value);
report["completed_interval_ns"] = std::move(values);
}
}
llvm::json::Object serializeWindow(const PimQueuePolicyResult &policy, bool details) {
const auto &window = policy.legacy;
llvm::json::Object result{{"status", windowStatus(policy)}};
if (window.intervalNs) {
result["interval_ns"] = *window.intervalNs;
result["tie_policy"] = policy.reverseTies ? "reverse event-index order for simultaneous readiness"
: "event-index order for simultaneous readiness";
}
if (!window.reason.empty()) result["reason"] = window.reason;
serializeProgress(result, window.progress, window.intervalNs.has_value(), details,
window.termination == PimQueueTermination::EventBudget
|| window.termination == PimQueueTermination::NoReadyEvents);
return result;
}
llvm::json::Object serializeRecurrence(const PimQueuePolicyResult &policy) {
const auto &evidence = policy.recurrence;
llvm::json::Object result{{"status", evidence.proved() ? "proved" : "unproved"},
{"termination", terminationName(policy.termination)}};
if (policy.checkpoints) {
result["checkpoints"] = policy.checkpoints;
result["history_bytes"] = policy.historyBytes;
}
if (evidence.proved()) {
result["period_generations"] = evidence.periodGenerations;
result["interval_ns"] = *evidence.intervalNs;
result["state_words"] = policy.stateWords;
result["complete_scheduler_state"] = true;
result["proof_processed_events"] = policy.proofProcessedEvents;
}
if (!evidence.reason.empty()) result["reason"] = evidence.reason;
return result;
}
void addAlternative(llvm::json::Object &report, const PimQueuePolicyResult &policy, bool legacy) {
auto interval = legacy ? policy.legacy.intervalNs : policy.intervalNs();
report["alternative_tie_status"] = legacy ? windowStatus(policy) : interval ? "estimated" : "unavailable";
if (interval) report["alternative_tie_interval_ns"] = *interval;
if ((legacy || !policy.recurrence.proved()) && !policy.legacy.reason.empty())
report["alternative_tie_reason"] = policy.legacy.reason;
}
} // namespace
llvm::json::Object onnx_mlir::serializePimQueueEvaluation(const PimQueueEvaluation &evaluation) {
const auto &primary = evaluation.policies[0];
const auto &alternative = evaluation.policies[1];
llvm::json::Object report;
if (evaluation.recurrenceRequired && primary.recurrence.proved()) {
report = llvm::json::Object{{"status", "estimated"}, {"interval_ns", *primary.intervalNs()},
{"tie_policy", primary.reverseTies ? "reverse event-index order for simultaneous readiness"
: "event-index order for simultaneous readiness"}};
serializeProgress(report, primary.progress, true, evaluation.details, false);
} else report = serializeWindow(primary, evaluation.details);
if (evaluation.recurrenceRequired) {
auto legacy = serializeWindow(primary, evaluation.details);
legacy["termination"] = terminationName(primary.legacy.termination);
legacy["alternative_termination"] = terminationName(alternative.legacy.termination);
addAlternative(legacy, alternative, true);
report["interval_source"] = primary.recurrence.proved() ? "complete-state recurrence" : "legacy interval window";
report["alternative_interval_source"] = alternative.recurrence.proved() ? "complete-state recurrence" : "legacy interval window";
report["periodic_state"] = serializeRecurrence(primary);
report["alternative_periodic_state"] = serializeRecurrence(alternative);
report["legacy_interval_window"] = std::move(legacy);
}
addAlternative(report, alternative, false);
if (auto interval = primary.intervalNs())
report["memory_service_fraction"] = evaluation.memoryServiceNs / *interval;
if (evaluation.measureTime) report["analysis_ms"] = evaluation.evaluationSeconds * 1000;
report["arbitration"] = "FIFO by predicted controller arrival";
return report;
}
@@ -357,6 +357,7 @@ static mlir::FailureOr<PeriodicDependencyEstimate> analyzePlan(PeriodicPlan &pla
return analyzePeriodicDependencies(plan.events.size(), plan.edges, plan.error);
}
} // namespace
llvm::json::Object getPeriodicScheduleReport(const ComputeGraph &graph,
+623
View File
@@ -0,0 +1,623 @@
#include "PimSpatialCost.hpp"
#include "PimInstructionTiming.hpp"
#include "PimTimingArithmetic.hpp"
#include "PimBinaryFormat.hpp"
#include "PimCompilerUtils.hpp"
#include "PimCompilerOptions.hpp"
#include "PimScheduleDiagnostic.hpp"
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp"
#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp"
#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/DeferredProjectionAnalysis.hpp"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/IR/Matchers.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/Support/MathExtras.h"
#include <chrono>
#include <array>
#include <map>
#include <limits>
using namespace mlir;
using namespace onnx_mlir;
using namespace onnx_mlir::spatial;
using pim_timing::add;
using pim_timing::mul;
using pim_timing::ceil;
namespace {
using Event = PimExecutionEvent;
using Sequence = std::vector<Event>;
enum class Residence { Local, Host, Constant, Zero, CapturedWeight };
PimPeriodicEstimate unavailableCost(llvm::StringRef reason) {
PimPeriodicEstimate result;
result.cost.reason = reason.str();
result.report = llvm::json::Object{{"status", "unavailable"}, {"cost_available", false},
{"convergence_proved", nullptr}, {"lowering_supported", nullptr},
{"safe_for_selection", false}, {"reason", reason.str()}};
return result;
}
class Extraction {
public:
const PimInstructionTiming &target;
bool diagnostics;
PimExecutionProgram program;
std::string error, loweringReason;
StaticLoweringSupport loweringSupport = StaticLoweringSupport::Supported;
std::map<std::string, int64_t> gaps;
int64_t visited = 0, constantLoads = 0;
explicit Extraction(const PimInstructionTiming &target, bool diagnostics)
: target(target), diagnostics(diagnostics), program(target.processorCount) {}
bool build(ModuleOp module);
private:
llvm::SmallDenseSet<int64_t, 16> assignedCores;
DenseMap<Value, Residence> residence;
DenseMap<Value, CopyAccessPlan> publicationAccess;
DenseMap<Value, SmallVector<unsigned, 2>> directOutputs;
llvm::SmallDenseSet<Value, 4> compactStorage;
llvm::SmallPtrSet<Operation *, 32> indexed;
StaticIndexEnvironment environment;
Residence storage(Value value);
uint64_t elements(Type type) {
auto shape = dyn_cast<ShapedType>(type);
if (!shape || !shape.hasStaticShape()) {
error = "abstract work requires a static shaped payload";
return 0;
}
if (!shape.getElementType().isIntOrFloat() || shape.getElementTypeBitWidth() != 32) {
error = "abstract precision projection requires 32-bit elements";
return 0;
}
uint64_t result = 1;
for (int64_t dimension : shape.getShape()) result = mul(result, dimension);
return result;
}
uint64_t bytes(Type type) {
uint64_t count = elements(type);
auto shaped = dyn_cast<ShapedType>(type);
if (!shaped || !shaped.getElementType().isIntOrFloat()) {
error = "abstract transfer requires a scalar integer or floating element";
return 0;
}
uint64_t nativeBytes = ceil(mul(count, shaped.getElementTypeBitWidth()), 8);
if (nativeBytes > std::numeric_limits<int32_t>::max()) {
loweringSupport = StaticLoweringSupport::Unknown;
loweringReason = "native transfer size requires backend normalization and size verification";
}
return ceil(mul(count, target.computeBitwidth), 8);
}
void work(Sequence &sequence, uint64_t ns, Event::WorkClass resource = Event::LocalCopy) {
if (!ns) return;
if (!sequence.empty() && sequence.back().kind == Event::Work
&& sequence.back().workClass == resource)
sequence.back().amount = add(sequence.back().amount, ns);
else {
sequence.push_back({Event::Work, ns});
sequence.back().workClass = resource;
}
}
uint64_t localWork(uint64_t bytes, bool read, bool write) const {
return getPimLocalTransferWorkNs(target, bytes, read, write);
}
void materialize(Value value, Sequence &sequence);
bool accessSlice(tensor::ExtractSliceOp slice, Sequence &sequence, bool host);
bool block(Block &body, Sequence &sequence);
bool operation(Operation &op, Sequence &sequence);
bool loop(scf::ForOp op, Sequence &sequence);
bool core(Operation *op, int64_t physicalCore, unsigned lane);
};
Residence Extraction::storage(Value value) {
if (auto it = residence.find(value); it != residence.end()) return it->second;
Attribute constant;
if (matchPattern(value, m_Constant(&constant)))
return isZeroBitPatternSplat(constant) ? Residence::Zero : Residence::Constant;
if (auto slice = value.getDefiningOp<tensor::ExtractSliceOp>()) return storage(slice.getSource());
if (auto collapse = value.getDefiningOp<tensor::CollapseShapeOp>()) return storage(collapse.getSrc());
if (auto expand = value.getDefiningOp<tensor::ExpandShapeOp>()) return storage(expand.getSrc());
if (auto cast = value.getDefiningOp<tensor::CastOp>()) return storage(cast.getSource());
if (auto branch = value.getDefiningOp<scf::IfOp>()) {
auto condition = evaluateDeferredIndex(branch.getCondition(), environment);
if (succeeded(condition)) {
Region &region = *condition ? branch.getThenRegion() : branch.getElseRegion();
return storage(region.front().getTerminator()->getOperand(cast<OpResult>(value).getResultNumber()));
}
}
if (auto arg = dyn_cast<BlockArgument>(value)) {
if (auto loop = dyn_cast_or_null<scf::ForOp>(arg.getOwner()->getParentOp()))
if (arg.getArgNumber()) return Residence::Local;
return Residence::Host;
}
if (isa_and_nonnull<SpatVMMOp, SpatVAddOp, SpatVSubOp, SpatVMulOp,
SpatVMaxOp, SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp,
SpatChannelReceiveOp, SpatHostWaitLoadOp, SpatScheduledCompute,
SpatScheduledComputeBatch, SpatConcatOp, tensor::EmptyOp,
tensor::InsertSliceOp, tensor::PadOp, linalg::TransposeOp, scf::ForOp>(value.getDefiningOp()))
return Residence::Local;
error = "abstract storage residence is unavailable for "
+ value.getDefiningOp()->getName().getStringRef().str();
return Residence::Local; // The caller discards the estimate when error is set.
}
void Extraction::materialize(Value value, Sequence &sequence) {
auto place = storage(value);
if (place == Residence::Host || place == Residence::Constant || place == Residence::CapturedWeight) {
sequence.push_back({Event::Load, bytes(value.getType())});
// Captured weights remain block arguments; only direct global zero loads fold to fills.
if (place == Residence::Constant || place == Residence::CapturedWeight) {
++constantLoads;
sequence.back().constant = true;
}
} else if (place == Residence::Zero) {
work(sequence, localWork(bytes(value.getType()), false, true), Event::Fill);
}
}
bool Extraction::accessSlice(tensor::ExtractSliceOp slice, Sequence &sequence, bool host) {
auto source = computeRowMajorStridesChecked(slice.getSourceType().getShape());
auto destination = computeRowMajorStridesChecked(slice.getStaticSizes());
if (failed(source) || failed(destination)) { error = "abstract slice geometry unavailable"; return false; }
SmallVector<int64_t> strides;
for (auto [stride, scale] : llvm::zip(*source, slice.getStaticStrides())) {
int64_t result;
if (scale <= 0 || llvm::MulOverflow(stride, scale, result)) {
error = "abstract slice stride unavailable"; return false;
}
strides.push_back(result);
}
auto copy = CopyAccessPlan::build(slice.getStaticSizes(), strides, *destination);
if (failed(copy)) { error = "abstract slice access plan unavailable"; return false; }
uint64_t size = ceil(mul(copy->getRunElements(), target.computeBitwidth), 8);
if (host) {
sequence.push_back({Event::Repeat, static_cast<uint64_t>(copy->getRunCount()),
0, 0, 0, {{Event::Load, size}}});
residence[slice.getResult()] = Residence::Local;
compactStorage.insert(slice.getResult());
} else if (compactStorage.contains(slice.getSource())
&& slice.getSourceType().getRank() == slice.getType().getRank()
&& llvm::all_of(slice.getResult().getUsers(), [](Operation *user) {
return isa<SpatHostStoreSyncOp>(user);
})) {
// Publication-only slices retain the compact source's subview geometry.
publicationAccess.insert_or_assign(slice.getResult(), *copy);
} else if (copy->getRunCount() > 1) {
if (diagnostics) ++gaps["local slice materialization and forwarding"];
work(sequence, mul(copy->getRunCount(), localWork(size, true, true)));
}
if (compactStorage.contains(slice.getSource()) && copy->getRunCount() == 1)
compactStorage.insert(slice.getResult());
return true;
}
bool Extraction::loop(scf::ForOp op, Sequence &sequence) {
auto lower = evaluateDeferredIndex(op.getLowerBound(), environment);
auto upper = evaluateDeferredIndex(op.getUpperBound(), environment);
auto step = evaluateDeferredIndex(op.getStep(), environment);
if (failed(lower) || failed(upper) || failed(step) || *step <= 0) {
error = "abstract loop has unavailable static bounds"; return false;
}
for (Value initial : op.getInitArgs()) materialize(initial, sequence);
int64_t distance = 0;
if (*upper > *lower && llvm::SubOverflow(*upper, *lower, distance)) {
error = "unrepresentable abstract loop trip count"; return false;
}
uint64_t count = ceil(distance, *step);
bool indexSensitive = indexed.contains(op);
if (indexSensitive && count > 100000) {
error = "abstract indexed loop exceeds analysis budget"; return false;
}
if (!count) return true;
if (indexSensitive) {
for (uint64_t i = 0; i < count; ++i) {
int64_t induction;
if (llvm::AddOverflow(*lower, static_cast<int64_t>(mul(i, *step)), induction)) {
error = "unrepresentable abstract induction value"; return false;
}
environment.bindings[op.getInductionVar()] = induction;
if (!block(*op.getBody(), sequence)) return false;
}
} else {
environment.bindings[op.getInductionVar()] = *lower;
Event repeat{Event::Repeat, count};
if (!block(*op.getBody(), repeat.body)) return false;
sequence.push_back(std::move(repeat));
}
environment.bindings.erase(op.getInductionVar());
return true;
}
bool Extraction::operation(Operation &op, Sequence &sequence) {
if (auto loopOp = dyn_cast<scf::ForOp>(op)) return loop(loopOp, sequence);
if (auto branch = dyn_cast<scf::IfOp>(op)) {
auto condition = evaluateDeferredIndex(branch.getCondition(), environment);
if (failed(condition)) { error = "abstract branch condition is unavailable"; return false; }
Region &region = *condition ? branch.getThenRegion() : branch.getElseRegion();
return region.empty() || block(region.front(), sequence);
}
auto index = [&](Value value) -> int64_t {
auto result = evaluateDeferredIndex(value, environment);
if (failed(result)) { error = "abstract communication index is unavailable"; return -1; }
return *result;
};
if (auto sync = dyn_cast<SpatSyncOp>(op)) {
sequence.push_back({Event::Signal, 0, index(sync.getTargetCoreId()), index(sync.getEventRegister())});
} else if (auto wait = dyn_cast<SpatWaitOp>(op)) {
int64_t count = index(wait.getWaitValue());
if (count) sequence.push_back({Event::Wait, 0, 0, index(wait.getEventRegister()), count});
} else if (auto send = dyn_cast<SpatChannelSendOp>(op)) {
materialize(send.getInput(), sequence);
sequence.push_back({Event::Send, bytes(send.getInput().getType()), index(send.getTargetCoreId())});
} else if (auto recv = dyn_cast<SpatChannelReceiveOp>(op)) {
sequence.push_back({Event::Receive, bytes(recv.getOutput().getType()), index(recv.getSourceCoreId())});
} else if (auto store = dyn_cast<SpatHostStoreSyncOp>(op)) {
materialize(store.getInput(), sequence);
if (auto plan = publicationAccess.find(store.getInput()); plan != publicationAccess.end())
sequence.push_back({Event::Repeat, static_cast<uint64_t>(plan->second.getRunCount()),
0, 0, 0, {{Event::Store, ceil(mul(plan->second.getRunElements(), target.computeBitwidth), 8)}}});
else sequence.push_back({Event::Store, bytes(store.getInput().getType())});
sequence.push_back({Event::Signal, 0, index(store.getTargetCoreId()), index(store.getEventRegister())});
} else if (auto load = dyn_cast<SpatHostWaitLoadOp>(op)) {
int64_t count = index(load.getWaitValue());
if (count) sequence.push_back({Event::Wait, 0, 0, index(load.getEventRegister()), count});
sequence.push_back({Event::Load, bytes(load.getOutput().getType())});
} else if (auto vmm = dyn_cast<SpatVMMOp>(op)) {
materialize(vmm.getInput(), sequence);
auto weight = dyn_cast<RankedTensorType>(vmm.getWeight().getType());
if (!weight || weight.getRank() != 2 || !weight.hasStaticShape() || !target.matrixColumns
|| weight.getDimSize(1) <= 0
|| static_cast<uint64_t>(weight.getDimSize(1)) % target.matrixColumns
|| weight.getDimSize(0) <= 0
|| static_cast<uint64_t>(weight.getDimSize(0)) > target.matrixRows
|| weight.getEncoding() || !weight.getElementType().isIntOrFloat()
|| weight.getElementTypeBitWidth() != 32) {
error = "unavailable physical matrix weight geometry"; return false;
}
if (diagnostics && static_cast<uint64_t>(weight.getDimSize(0)) < target.matrixRows)
++gaps["matrix input padding and copy forwarding"];
uint64_t output = getPimMatrixOutputBytes(target, target.matrixCellBits,
pim_binary::kMatrixWeightPrecisionBits, weight.getDimSize(1) / target.matrixColumns,
ceil(target.computeBitwidth, 8));
work(sequence, getPimMatrixWorkNs(target, target.computeBitwidth, output), Event::Matrix);
} else if (isa<SpatVAddOp, SpatVSubOp, SpatVMulOp, SpatVMaxOp,
SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp>(op)) {
for (Value operand : op.getOperands()) materialize(operand, sequence);
uint64_t width = ceil(target.computeBitwidth, 8);
work(sequence, getPimVectorWorkNs(target, elements(op.getOperand(0).getType()),
width, width, op.getNumOperands() == 2), Event::Vector);
} else if (auto extract = dyn_cast<tensor::ExtractSliceOp>(op)) {
auto place = storage(extract.getSource());
if (place == Residence::Host || place == Residence::Local)
if (!accessSlice(extract, sequence, place == Residence::Host)) return false;
} else if (auto insert = dyn_cast<tensor::InsertSliceOp>(op)) {
materialize(insert.getSource(), sequence);
uint64_t size = bytes(insert.getSource().getType());
work(sequence, localWork(size, true, true));
if (compactStorage.contains(insert.getDest())) compactStorage.insert(insert.getResult());
if (diagnostics) ++gaps["output copy forwarding"];
} else if (auto transpose = dyn_cast<linalg::TransposeOp>(op)) {
materialize(transpose.getInput(), sequence);
auto type = cast<ShapedType>(transpose.getInput().getType());
uint64_t volume = elements(type);
if (!volume) {
if (error.empty()) error = "zero-element transpose has no abstract timing contract";
return false;
}
if (failed(computeRowMajorStridesChecked(type.getShape()))) {
error = "abstract transpose geometry unavailable"; return false;
}
auto run = getTransposeRun(type.getShape(), transpose.getPermutation());
uint64_t width = ceil(target.computeBitwidth, 8);
uint64_t runBytes = mul(run.elements, width);
uint64_t cost = run.sourceStride == 1
? localWork(runBytes, true, true)
: getPimVectorWorkNs(target, run.elements, width, width, false);
work(sequence, mul(volume / run.elements, cost),
run.sourceStride == 1 ? Event::LocalCopy : Event::VectorCopy);
if (diagnostics) ++gaps["transpose alias and constant folding"];
} else if (auto pad = dyn_cast<tensor::PadOp>(op)) {
materialize(pad.getSource(), sequence);
work(sequence, localWork(bytes(pad.getResult().getType()), false, true), Event::Fill);
work(sequence, localWork(bytes(pad.getSource().getType()), true, true));
if (diagnostics) ++gaps["padding fill and copy normalization"];
} else if (auto concat = dyn_cast<SpatConcatOp>(op)) {
for (Value input : concat.getInputs()) {
materialize(input, sequence);
uint64_t size = bytes(input.getType());
work(sequence, localWork(size, true, true));
}
compactStorage.insert(concat.getOutput());
if (diagnostics) ++gaps["concat alias and copy elimination"];
} else if (auto empty = dyn_cast<tensor::EmptyOp>(op)) {
compactStorage.insert(empty.getResult());
} else if (isa<arith::ConstantOp, tensor::ExtractOp,
tensor::CollapseShapeOp, tensor::ExpandShapeOp, tensor::CastOp,
scf::YieldOp, SpatYieldOp>(op) || isPureIndexComputationOp(&op)) {
// Compile-time shape/index work has no device execution cost.
} else if (auto parallel = dyn_cast<SpatInParallelOp>(op)) {
auto batch = parallel->getParentOfType<SpatScheduledComputeBatch>();
auto first = batch ? batch.getOutputArgument(0) : std::nullopt;
if (!first) { error = "unavailable abstract batch output mapping"; return false; }
for (Operation &nested : parallel.getRegion().front()) {
auto insert = dyn_cast<tensor::ParallelInsertSliceOp>(nested);
auto destination = insert ? dyn_cast<BlockArgument>(insert.getDest()) : BlockArgument{};
if (!destination || destination.getArgNumber() < first->getArgNumber()) {
error = "unsupported abstract batch output"; return false;
}
unsigned result = destination.getArgNumber() - first->getArgNumber();
if (result >= batch.getNumResults()) { error = "invalid abstract batch output"; return false; }
if (succeeded(getDirectReturnOperandIndex(cast<OpResult>(batch.getResult(result))))) {
materialize(insert.getSource(), sequence);
sequence.push_back({Event::Store, bytes(insert.getSource().getType())});
} else if (!batch.getResult(result).use_empty()) {
error = "unavailable indirect batch output collection"; return false;
}
}
} else {
error = "unsupported abstract operation: " + op.getName().getStringRef().str();
return false;
}
return error.empty();
}
bool Extraction::block(Block &body, Sequence &sequence) {
for (Operation &op : body) {
if (++visited > 1000000) { error = "abstract extraction budget exceeded"; return false; }
if (!operation(op, sequence)) return false;
for (Value result : op.getResults())
if (auto outputs = directOutputs.find(result); outputs != directOutputs.end())
// Lowering inserts each store immediately after the shared producer.
for (unsigned destination : llvm::reverse(outputs->second)) {
(void)destination; // Distinct host destinations require distinct stores.
sequence.push_back({Event::Store, bytes(result.getType())});
}
}
return true;
}
bool Extraction::core(Operation *op, int64_t physicalCore, unsigned lane) {
if (physicalCore < 0 || physicalCore >= static_cast<int64_t>(program.size())) {
loweringSupport = StaticLoweringSupport::Unsupported;
error = "physical core ID is outside the target: " + std::to_string(physicalCore); return false;
}
if (!assignedCores.insert(physicalCore).second) {
loweringSupport = StaticLoweringSupport::Unsupported;
error = "duplicate physical core assignment: " + std::to_string(physicalCore); return false;
}
residence.clear(); publicationAccess.clear(); directOutputs.clear(); compactStorage.clear();
environment.bindings.clear();
ValueRange inputs, weights;
auto single = dyn_cast<SpatScheduledCompute>(op);
auto batch = dyn_cast<SpatScheduledComputeBatch>(op);
if (single) { inputs = single.getInputs(); weights = single.getWeights(); }
else { inputs = batch.getInputs(); weights = batch.getWeights(); environment.bindings[*batch.getLaneArgument()] = lane; }
Sequence &sequence = program[physicalCore];
if (single) {
auto yield = cast<SpatYieldOp>(single.getBody().front().getTerminator());
for (auto [result, value] : llvm::zip(single.getResults(), yield.getOperands()))
if (auto destination = getDirectReturnOperandIndex(cast<OpResult>(result)); succeeded(destination)) {
if (!value.getDefiningOp() || value.getDefiningOp()->getBlock() != &single.getBody().front()) {
error = "unavailable direct output insertion position"; return false;
}
directOutputs[value].push_back(*destination);
} else if (!result.use_empty()) {
error = "unavailable indirect compute output collection"; return false;
}
}
for (unsigned i = 0; i < weights.size(); ++i)
residence[single ? *single.getWeightArgument(i) : *batch.getWeightArgument(i)] = Residence::CapturedWeight;
for (unsigned i = 0; i < inputs.size(); ++i) {
Value argument = single ? *single.getInputArgument(i) : *batch.getInputArgument(i);
auto plan = planComputeInputMaterialization(op, i);
if (failed(plan)) { error = "input materialization plan unavailable"; return false; }
residence[argument] = storage(inputs[i]);
// FuncOpArgToGlobalMemoryPattern removes this input and exposes the host
// tensor in the body before entry-copy materialization runs.
if (*plan == ComputeInputMaterialization::ForwardFunctionInput) {
if (diagnostics) ++gaps["function-input whole-load versus slice materialization"];
continue;
}
if (*plan == ComputeInputMaterialization::CopyAtEntry) {
materialize(inputs[i], sequence);
residence[argument] = Residence::Local;
compactStorage.insert(argument);
if (diagnostics) ++gaps["input host slicing and receive sinking"];
}
}
return block(op->getRegion(0).front(), sequence);
}
bool Extraction::build(ModuleOp module) {
bool okay = true;
module.walk<WalkOrder::PostOrder>([&](Operation *op) {
if (indexed.contains(op) || isa<SpatChannelSendOp, SpatChannelReceiveOp,
SpatSyncOp, SpatWaitOp, SpatHostStoreSyncOp, SpatHostWaitLoadOp, scf::IfOp>(op))
if (Operation *parent = op->getParentOp()) indexed.insert(parent);
});
module.walk([&](Operation *op) {
if (!okay) return WalkResult::interrupt();
if (auto single = dyn_cast<SpatScheduledCompute>(op)) {
auto coreId = op->getAttrOfType<IntegerAttr>("coreId");
if (!coreId) { error = "missing physical core"; okay = false; }
else okay = core(op, coreId.getInt(), 0);
return WalkResult::skip();
}
if (auto batch = dyn_cast<SpatScheduledComputeBatch>(op)) {
auto ids = op->getAttrOfType<DenseI32ArrayAttr>("coreIds");
if (!ids || ids.size() != batch.getLaneCount()) { error = "missing physical batch cores"; okay = false; }
else for (auto [lane, id] : llvm::enumerate(ids.asArrayRef())) if (!(okay = core(op, id, lane))) break;
return WalkResult::skip();
}
return WalkResult::advance();
});
return okay && error.empty();
}
} // namespace
PimPeriodicEstimate onnx_mlir::evaluatePimSpatialSchedule(ModuleOp module,
const PimInstructionTiming &target, PimPeriodicOptions options) try {
if (!target.hasPeriodicTransferTiming() || !target.computeBitwidth)
return unavailableCost("unsupported abstract timing contract");
auto start = std::chrono::steady_clock::now();
Extraction extraction(target, options.workBreakdown);
bool available = extraction.build(module);
double extractionMs = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - start).count();
PimPeriodicEstimate estimate;
if (available) estimate = evaluatePimExecution(extraction.program, target, options);
else {
estimate.cost.reason = extraction.error;
estimate.report = llvm::json::Object{{"status", "unavailable"}, {"reason", extraction.error}};
if (options.verifyPeriodicState)
for (auto &recurrence : estimate.cost.recurrence)
recurrence.status = ScheduleRecurrenceEvidence::Unproved;
}
auto &report = estimate.report;
auto &cost = estimate.cost;
cost.timing.extractionSeconds = extractionMs / 1000;
if (available || extraction.loweringSupport == StaticLoweringSupport::Unsupported)
cost.loweringSupport = extraction.loweringSupport;
if (available && options.workBreakdown) {
PimPeriodicOptions sensitivity;
sensitivity.ignoreLocalCopies = true;
auto withoutCopies = analyzePimExecution(extraction.program, target, sensitivity);
if (auto *queue = withoutCopies.getObject("memory_controller"))
report["without_local_copy_work"] = std::move(*queue);
}
auto convergence = cost.convergenceProved();
auto support = cost.loweringSupported();
report["cost_available"] = cost.primary.has_value() && cost.alternative.has_value();
report["convergence_proved"] = convergence
? llvm::json::Value(*convergence) : llvm::json::Value(nullptr);
report["lowering_supported"] = support
? llvm::json::Value(*support) : llvm::json::Value(nullptr);
std::string failureReason = std::move(cost.reason);
cost.reason = "Spatial prediction accuracy has no formal bound";
if (!cost.available()) cost.reason = "Spatial queue cost is unavailable";
else if (cost.loweringSupport != StaticLoweringSupport::Supported)
cost.reason = "Spatial lowering support is unproved";
else if (convergence != true)
cost.reason = "complete-state recurrence is unproved under one or both arbitration policies";
if (!extraction.loweringReason.empty()) cost.reason = extraction.loweringReason;
if (!failureReason.empty()) cost.reason = std::move(failureReason);
report["lowering_support_scope"] = "static operation and transfer contract; final backend verification is required";
if (!extraction.loweringReason.empty()) report["lowering_reason"] = extraction.loweringReason;
report["safe_for_selection"] = false;
report["diagnostic_only"] = true;
report["error_bound_available"] = false;
report["extraction_ms"] = extractionMs;
report["visited_operations"] = extraction.visited;
if (!options.workBreakdown) return estimate;
report["constant_load_sites"] = extraction.constantLoads;
uint64_t workNs = 0, records = 0, constantRequests = 0, constantBytes = 0;
constexpr const char *workClassNames[] = {
"matrix", "vector", "local_copy", "fill", "vector_copy", "scalar_issue"};
std::array<uint64_t, std::size(workClassNames)> workByClass{};
static_assert(Event::ScalarIssue + 1 == std::size(workClassNames));
std::array<uint64_t, 7> counts{};
std::map<uint64_t, uint64_t> readSizes;
std::function<void(const Sequence &, uint64_t)> summarize =
[&](const Sequence &sequence, uint64_t multiplier) {
for (const Event &event : sequence) {
++records;
if (event.kind == Event::Repeat) summarize(event.body, mul(multiplier, event.amount));
else {
counts[event.kind] = add(counts[event.kind], multiplier);
if (event.kind == Event::Work) {
workNs = add(workNs, mul(multiplier, event.amount));
switch (event.workClass) {
case Event::Matrix:
case Event::Vector:
case Event::LocalCopy:
case Event::Fill:
case Event::VectorCopy:
case Event::ScalarIssue:
workByClass[event.workClass] = add(workByClass[event.workClass], mul(multiplier, event.amount));
break;
default: throw pim_timing::Unrepresentable{"unsupported execution work class"};
}
} else if (event.kind == Event::Load) {
readSizes[event.amount] = add(readSizes[event.amount], multiplier);
if (event.constant) {
constantRequests = add(constantRequests, multiplier);
constantBytes = add(constantBytes, mul(multiplier, event.amount));
}
}
}
}
};
for (const Sequence &sequence : extraction.program) summarize(sequence, 1);
report["compute_and_local_work_ns"] = workNs;
llvm::json::Array classes;
for (uint64_t value : workByClass) classes.push_back(value);
llvm::json::Array columns;
for (const char *name : workClassNames) columns.push_back(name);
report["work_class_columns"] = std::move(columns);
report["work_class_ns"] = std::move(classes);
llvm::json::Object eventCounts;
const char *names[] = {"work_chunks", "load", "store", "send", "receive", "sync", "wait"};
for (size_t i = 0; i < counts.size(); ++i) eventCounts[names[i]] = counts[i];
report["event_counts"] = std::move(eventCounts);
report["compact_records"] = records;
report["constant_read_requests"] = constantRequests;
report["constant_read_bytes"] = constantBytes;
llvm::json::Array sizes;
for (auto [bytes, count] : readSizes)
sizes.push_back(llvm::json::Object{{"bytes", bytes}, {"requests", count}});
report["read_sizes"] = std::move(sizes);
llvm::json::Object gaps;
for (const auto &[name, count] : extraction.gaps) gaps[name] = count;
report["unresolved_decisions"] = std::move(gaps);
report["omitted_effects"] = llvm::json::Array{"scalar register setup", "indirect output collection",
"host slice splitting and folding", "copy forwarding and local reuse legality"};
report["analysis_ms"] = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - start).count();
return estimate;
} catch (const pim_timing::Unrepresentable &failure) {
return unavailableCost(failure.reason);
}
llvm::json::Object onnx_mlir::analyzePimSpatialSchedule(ModuleOp module,
const PimInstructionTiming &target, PimPeriodicOptions options) {
return evaluatePimSpatialSchedule(module, target, options).report;
}
void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target,
PimInstructionTiming timing) {
auto contract = getPimPeriodicTimingContract();
if (contract) timing.matrixCellBits = *contract;
auto hardware = std::make_shared<const PimInstructionTiming>(std::move(timing));
target.reportPeriodicSchedule = [hardware](const spatial::ComputeGraph &graph,
const spatial::ScheduledComputeMaterializationResult &materialization,
const spatial::DeferredTransferPlan &transfers,
const spatial::DeferredBoundaryPlan &boundaries) {
dumpPeriodicScheduleEstimate(graph, materialization, transfers, boundaries, *hardware);
};
if (!contract) target.wholeScheduleCostUnavailable = llvm::toString(contract.takeError());
if (!hardware->hasPeriodicTransferTiming())
target.wholeScheduleCostUnavailable = "target has no supported queue timing contract";
if (pimDisableSynchronization)
target.wholeScheduleCostUnavailable = "synchronization ablation has no periodic queue timing contract";
if (!target.wholeScheduleCostUnavailable.empty()) return;
target.estimateRealizedScheduleCost = [hardware](mlir::ModuleOp module,
llvm::json::Object &summary) {
PimPeriodicOptions options;
options.workBreakdown = pimPeriodicDiagnostics;
options.measureTime = true;
options.verifyPeriodicState = true;
auto estimate = evaluatePimSpatialSchedule(module, *hardware, options);
auto cost = std::move(estimate.cost);
summary = std::move(estimate.report);
if (estimate.queue)
summary["interval_source"] = cost.convergenceProved() == true
? "complete-state recurrence under both arbitration policies"
: "recurrence is unproved under one or both arbitration policies";
summary["cost_available"] = cost.available();
return cost;
};
}
+16
View File
@@ -0,0 +1,16 @@
#pragma once
#include "PimPeriodicAnalysis.hpp"
#include "mlir/IR/BuiltinOps.h"
namespace onnx_mlir {
void configurePimScheduleCosts(spatial::SchedulingTarget &target, PimInstructionTiming timing);
// Read-only, target-specific work extraction. No target dialect is constructed.
PimPeriodicEstimate evaluatePimSpatialSchedule(mlir::ModuleOp module,
const PimInstructionTiming &target, PimPeriodicOptions options = {});
llvm::json::Object analyzePimSpatialSchedule(mlir::ModuleOp module,
const PimInstructionTiming &target, PimPeriodicOptions options = {});
} // namespace onnx_mlir
+37
View File
@@ -0,0 +1,37 @@
#pragma once
#include <cstdint>
#include <limits>
namespace onnx_mlir::pim_timing {
struct Unrepresentable { const char *reason; };
// Optional timing queries must fail without wrapping or terminating compilation.
struct Arithmetic {
static constexpr uint64_t limit = std::numeric_limits<int64_t>::max();
unsigned long long add(uint64_t a, uint64_t b) const {
if (a > limit || b > limit - a) throw Unrepresentable{"timing addition overflow"};
return a + b;
}
unsigned long long multiply(uint64_t a, uint64_t b) const {
if (a > limit || b > limit || (a && b > limit / a))
throw Unrepresentable{"timing multiplication overflow"};
return a * b;
}
unsigned long long subtract(uint64_t a, uint64_t b) const {
if (a > limit || b > a) throw Unrepresentable{"timing subtraction underflow"};
return a - b;
}
unsigned long long divideUp(uint64_t a, uint64_t b) const {
if (!b || a > limit || b > limit) throw Unrepresentable{"unrepresentable timing division"};
return a / b + (a % b != 0);
}
};
inline constexpr Arithmetic arithmetic{};
inline uint64_t add(uint64_t a, uint64_t b) { return arithmetic.add(a, b); }
inline uint64_t mul(uint64_t a, uint64_t b) { return arithmetic.multiply(a, b); }
inline uint64_t sub(uint64_t a, uint64_t b) { return arithmetic.subtract(a, b); }
inline uint64_t ceil(uint64_t a, uint64_t b) { return arithmetic.divideUp(a, b); }
} // namespace onnx_mlir::pim_timing
@@ -15,6 +15,7 @@
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
#include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp"
#include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp"
#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp"
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
using namespace mlir;
@@ -34,15 +35,6 @@ static bool isUsedOnlyByExtractSlices(Value value) {
&& llvm::all_of(value.getUsers(), [](Operation* user) { return isa<tensor::ExtractSliceOp>(user); });
}
static FailureOr<unsigned> getDirectReturnOperandIndex(OpResult result) {
if (!result.hasOneUse())
return failure();
auto returnOp = dyn_cast<func::ReturnOp>(*result.getUsers().begin());
if (!returnOp)
return failure();
return result.getUses().begin()->getOperandNumber();
}
static FailureOr<SmallVector<FragmentAssemblyCopy, 8>>
collectFragmentAssemblyCopiesFromBlueprint(spatial::SpatBlueprintOp blueprint,
@@ -85,7 +77,9 @@ collectFragmentAssemblyCopiesFromBlueprint(spatial::SpatBlueprintOp blueprint,
flatStrides)))
return failure();
SmallVector<int64_t> hostStrides = computeRowMajorStrides(resultType.getShape());
auto checkedStrides = computeRowMajorStridesChecked(resultType.getShape());
if (failed(checkedStrides)) return failure();
SmallVector<int64_t> hostStrides = std::move(*checkedStrides);
for (int64_t fragmentIndex = 0; fragmentIndex < static_cast<int64_t>(operandIndices.size()); ++fragmentIndex) {
Value source = fixedSource ? fixedSource : mapper.lookupOrDefault(fragmentOperands[operandIndices[fragmentIndex]]);
auto sourceType = dyn_cast<RankedTensorType>(source.getType());
@@ -183,7 +177,9 @@ collectTopLevelFragmentAssemblyCopies(OpResult result, RankedTensorType packedRe
flatSizes,
flatStrides)))
return failure();
SmallVector<int64_t> hostStrides = computeRowMajorStrides(hostResultType.getShape());
auto checkedStrides = computeRowMajorStridesChecked(hostResultType.getShape());
if (failed(checkedStrides)) return failure();
SmallVector<int64_t> hostStrides = std::move(*checkedStrides);
for (int64_t fragmentIndex = 0; fragmentIndex < static_cast<int64_t>(operandIndices.size()); ++fragmentIndex) {
if (operandIndices[fragmentIndex] != static_cast<int64_t>(use.getOperandNumber()))
continue;
@@ -239,14 +235,16 @@ static Value createScaledIndexValue(IRRewriter& rewriter, Location loc, Value ba
return arith::MulIOp::create(rewriter, loc, base, scaleValue).getResult();
}
static Value createHostTargetOffset(IRRewriter& rewriter,
static FailureOr<Value> createHostTargetOffset(IRRewriter& rewriter,
Location loc,
ShapedType destinationType,
ArrayRef<OpFoldResult> mixedOffsets,
ArrayRef<int64_t> additionalOffsets,
IRMapping& mapper) {
int64_t elementBytes = static_cast<int64_t>(getElementTypeSizeInBytes(destinationType.getElementType()));
SmallVector<int64_t> strides = computeRowMajorStrides(destinationType.getShape());
auto checkedStrides = computeRowMajorStridesChecked(destinationType.getShape());
if (failed(checkedStrides)) return failure();
SmallVector<int64_t> strides = std::move(*checkedStrides);
Value totalOffset;
for (auto [dim, offset] : llvm::enumerate(mixedOffsets)) {
@@ -277,7 +275,7 @@ static Value createHostTargetOffset(IRRewriter& rewriter,
return totalOffset;
}
static Value createHostTargetOffset(IRRewriter& rewriter,
static FailureOr<Value> createHostTargetOffset(IRRewriter& rewriter,
tensor::ParallelInsertSliceOp insertSlice,
ShapedType destinationType,
IRMapping& mapper) {
@@ -380,6 +378,9 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
mapper.map(*oldWeightArg, coreBatchOp.getWeightArgument(weightIndex));
}
for (unsigned inputIndex = 0; inputIndex < computeBatchOp.getInputs().size(); ++inputIndex) {
auto inputPlan = planComputeInputMaterialization(computeBatchOp, inputIndex);
if (failed(inputPlan) || *inputPlan != ComputeInputMaterialization::CopyAtEntry)
return computeBatchOp.emitOpError("expected body-entry compute_batch input materialization");
auto oldArg = computeBatchOp.getInputArgument(inputIndex);
if (!oldArg)
return computeBatchOp.emitOpError("expected compute_batch input block arguments before lowering");
@@ -491,19 +492,20 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
if (failed(fragmentAssemblyRuns))
return failure();
SmallVector<int64_t> zeroOffsets(hostTargetType.getRank(), 0);
Value baseHostOffset = createHostTargetOffset(rewriter,
auto baseHostOffset = createHostTargetOffset(rewriter,
blueprint.getLoc(),
hostTargetType,
insertSlice.getMixedOffsets(),
zeroOffsets,
mapper);
if (failed(baseHostOffset)) return failure();
FailureOr<Value> updatedHostTarget = emitFragmentAssemblyCopyRuns(rewriter,
blueprint.getLoc(),
*fragmentAssemblyRuns,
hostTarget,
coreBatchOp.getOperation(),
std::nullopt,
baseHostOffset);
*baseHostOffset);
if (failed(updatedHostTarget))
return failure();
hostOutputTensors[resultIndex] = *updatedHostTarget;
@@ -511,7 +513,8 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
}
}
Value hostTargetOffset = createHostTargetOffset(rewriter, insertSlice, hostTargetType, mapper);
auto hostTargetOffset = createHostTargetOffset(rewriter, insertSlice, hostTargetType, mapper);
if (failed(hostTargetOffset)) return failure();
Value zeroOffset = getOrCreateIndexConstant(rewriter, coreBatchOp.getOperation(), 0);
auto sizeAttr = getTensorSizeInBytesAttr(rewriter, coreBatchOp.getOperation(), mappedSource);
if (failed(sizeAttr))
@@ -519,7 +522,7 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
pim::PimMemCopyDevToHostOp::create(rewriter,
insertSlice.getLoc(),
hostTarget.getType(),
hostTargetOffset,
*hostTargetOffset,
zeroOffset,
hostTarget,
mappedSource,
+7 -2
View File
@@ -123,7 +123,7 @@ static bool isLoopCarriedOutput(mlir::Value operand, Operation* operation) {
.getOperand(argument.getArgNumber() - 1) == operation->getResult(0);
}
mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Operation* operation) {
mlir::Value getReusableOutputTensor(Operation* operation) {
assert("Only support operations with a single result" && operation->getNumResults() == 1);
mlir::Value result = operation->getResult(0);
auto resultType = result.getType();
@@ -141,8 +141,13 @@ mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Op
if (bestOperand != validOperands.end())
return *bestOperand;
return {};
}
auto resultShapedType = cast<ShapedType>(resultType);
mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Operation* operation) {
if (mlir::Value operand = getReusableOutputTensor(operation))
return operand;
auto resultShapedType = cast<ShapedType>(operation->getResult(0).getType());
rewriter.setInsertionPoint(operation);
return tensor::EmptyOp::create(
rewriter, operation->getLoc(), resultShapedType.getShape(), resultShapedType.getElementType());
@@ -47,6 +47,10 @@ mlir::Operation* getEarliestUserWithinBlock(mlir::Value value);
mlir::SmallVector<mlir::Value> getOpOperandsSortedByUses(mlir::Operation* operation);
// Destination selection only. Bufferization still proves physical aliasing.
// Valid for the current SSA graph only; recompute after use or order changes.
mlir::Value getReusableOutputTensor(mlir::Operation* operation);
mlir::Value getBestOutputTensorFromOperandsOrAllocate(mlir::RewriterBase& rewriter, mlir::Operation* operation);
mlir::LogicalResult validateFragmentAssemblyMetadata(onnx_mlir::spatial::SpatBlueprintOp blueprint,
@@ -1,4 +1,6 @@
#include <cassert>
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
@@ -7,6 +9,41 @@ using namespace mlir;
namespace onnx_mlir {
FailureOr<unsigned> getDirectReturnOperandIndex(OpResult result) {
if (!result.hasOneUse())
return failure();
auto returnOp = dyn_cast<func::ReturnOp>(*result.getUsers().begin());
if (!returnOp)
return failure();
return result.getUses().begin()->getOperandNumber();
}
FailureOr<ComputeInputMaterialization>
planComputeInputMaterialization(Operation* owner, unsigned inputIndex) {
Value input;
bool batched = false;
if (auto compute = dyn_cast<spatial::SpatScheduledCompute>(owner)) {
if (inputIndex >= compute.getInputs().size() || !compute.getInputArgument(inputIndex))
return failure();
if (compute.getInputArgument(inputIndex)->use_empty())
return ComputeInputMaterialization::Unused;
input = compute.getInputs()[inputIndex];
} else if (auto batch = dyn_cast<spatial::SpatScheduledComputeBatch>(owner)) {
if (inputIndex >= batch.getInputs().size() || !batch.getInputArgument(inputIndex))
return failure();
input = batch.getInputs()[inputIndex];
batched = true;
} else return failure();
if (auto argument = dyn_cast<BlockArgument>(input); argument
&& isa<func::FuncOp>(argument.getOwner()->getParentOp()))
return ComputeInputMaterialization::ForwardFunctionInput;
if (!batched && input.getDefiningOp<arith::ConstantOp>())
return ComputeInputMaterialization::ForwardConstant;
return ComputeInputMaterialization::CopyAtEntry;
}
std::optional<unsigned> getDirectComputeLikeInputIndex(Operation* owner, unsigned operandNumber) {
auto getInputIndex = [operandNumber](Operation* op, unsigned inputCount) -> std::optional<unsigned> {
if (inputCount == 0)
@@ -7,6 +7,14 @@
namespace onnx_mlir {
// Input placement only; bufferization and folding determine physical requests.
enum class ComputeInputMaterialization { Unused, ForwardConstant, ForwardFunctionInput, CopyAtEntry };
mlir::FailureOr<unsigned> getDirectReturnOperandIndex(mlir::OpResult result);
mlir::FailureOr<ComputeInputMaterialization>
planComputeInputMaterialization(mlir::Operation* owner, unsigned inputIndex);
std::optional<unsigned> getDirectComputeLikeInputIndex(mlir::Operation* owner, unsigned operandNumber);
void replaceAndEraseDirectComputeLikeInput(mlir::PatternRewriter& rewriter,
@@ -16,6 +16,7 @@
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
#include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp"
#include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp"
#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp"
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
#include "src/Dialect/ONNX/ONNXOps.hpp"
@@ -80,7 +81,9 @@ static FailureOr<Value> lowerFragmentAssemblyBlueprint(IRRewriter& rewriter,
return blueprint.emitOpError("contiguous row-major fragment storage does not match its logical result"), failure();
return *reshaped;
}
SmallVector<int64_t> hostStrides = computeRowMajorStrides(resultType.getShape());
auto checkedStrides = computeRowMajorStridesChecked(resultType.getShape());
if (failed(checkedStrides)) return failure();
SmallVector<int64_t> hostStrides = std::move(*checkedStrides);
SmallVector<FragmentAssemblyCopy, 8> copies;
for (int64_t fragmentIndex = 0; fragmentIndex < static_cast<int64_t>(operandIndices.size()); ++fragmentIndex) {
int64_t operandIndex = operandIndices[fragmentIndex];
@@ -430,10 +433,16 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeOp(spatial::SpatScheduledCom
auto blockArg = computeOp.getInputArgument(inputIndex);
if (!blockArg)
return computeOp.emitOpError("expected compute input block arguments during input materialization");
if (blockArg->use_empty())
auto inputPlan = planComputeInputMaterialization(computeOp, inputIndex);
if (failed(inputPlan))
return computeOp.emitOpError("failed to plan compute input materialization");
if (*inputPlan == ComputeInputMaterialization::Unused)
continue;
if (*inputPlan == ComputeInputMaterialization::ForwardFunctionInput)
return computeOp.emitOpError("function inputs must be forwarded before core input materialization");
if (auto constantOp = input.getDefiningOp<arith::ConstantOp>()) {
if (*inputPlan == ComputeInputMaterialization::ForwardConstant) {
auto constantOp = input.getDefiningOp<arith::ConstantOp>();
blockArg->replaceAllUsesWith(getOrCreateConstantLike(constantFolder, constantOp));
continue;
}
@@ -19,10 +19,12 @@ static void copyRaptorDebugAttrs(Operation* source, Operation* target) {
}
}
static Value createDestinationByteOffset(PatternRewriter& rewriter,
static FailureOr<Value> createDestinationByteOffset(PatternRewriter& rewriter,
tensor::InsertSliceOp insert) {
auto destinationType = cast<RankedTensorType>(insert.getDestType());
SmallVector<int64_t> strides = computeRowMajorStrides(destinationType.getShape());
auto checkedStrides = computeRowMajorStridesChecked(destinationType.getShape());
if (failed(checkedStrides)) return failure();
SmallVector<int64_t> strides = std::move(*checkedStrides);
int64_t elementBytes = getElementTypeSizeInBytes(destinationType.getElementType());
Value total = arith::ConstantIndexOp::create(rewriter, insert.getLoc(), 0);
for (auto [dimension, offset] : llvm::enumerate(insert.getMixedOffsets())) {
@@ -114,9 +116,10 @@ static LogicalResult lowerReceive(
}
rewriter.setInsertionPoint(destinationInsert);
Value targetOffset = createDestinationByteOffset(rewriter, destinationInsert);
auto targetOffset = createDestinationByteOffset(rewriter, destinationInsert);
if (failed(targetOffset)) return failure();
auto copy = pim::PimMemCopyOp::create(
rewriter, op.getLoc(), destinationInsert.getDestType(), targetOffset, zero,
rewriter, op.getLoc(), destinationInsert.getDestType(), *targetOffset, zero,
destinationInsert.getDest(), *received, *sizeAttr);
rewriter.replaceOp(destinationInsert, copy.getOutput());
rewriter.eraseOp(op);
@@ -179,29 +179,18 @@ struct FuncOpArgToGlobalMemoryPattern final : OpRewritePattern<mlir::func::FuncO
for (auto& argUses : llvm::make_early_inc_range(arg.getUses())) {
auto argUser = argUses.getOwner();
if (auto spatCompute = dyn_cast<spatial::SpatScheduledCompute>(argUser)) {
auto inputIndex = getDirectComputeLikeInputIndex(spatCompute, argUses.getOperandNumber());
if (!inputIndex)
if (isa<spatial::SpatScheduledCompute, spatial::SpatScheduledComputeBatch>(argUser)) {
auto inputIndex = getDirectComputeLikeInputIndex(argUser, argUses.getOperandNumber());
if (!inputIndex) return failure();
auto plan = planComputeInputMaterialization(argUser, *inputIndex);
if (failed(plan) || (*plan != ComputeInputMaterialization::ForwardFunctionInput
&& *plan != ComputeInputMaterialization::Unused))
return failure();
auto BBArgIndex = *inputIndex;
rewriter.setInsertionPoint(&spatCompute.getBody().front().front());
rewriter.setInsertionPoint(&argUser->getRegion(0).front().front());
auto getGlobalOp = memref::GetGlobalOp::create(rewriter, loc, memRefType, argName);
auto toTensor = bufferization::ToTensorOp::create(
rewriter, loc, argRankedTensorType, getGlobalOp, rewriter.getUnitAttr(), rewriter.getUnitAttr());
replaceAndEraseDirectComputeLikeInput(rewriter, spatCompute.getOperation(), BBArgIndex, toTensor);
}
else if (auto spatComputeBatch = dyn_cast<spatial::SpatScheduledComputeBatch>(argUser)) {
auto inputIndex = getDirectComputeLikeInputIndex(spatComputeBatch, argUses.getOperandNumber());
if (!inputIndex)
return failure();
auto BBArgIndex = *inputIndex;
rewriter.setInsertionPoint(&spatComputeBatch.getBody().front().front());
auto getGlobalOp = memref::GetGlobalOp::create(rewriter, loc, memRefType, argName);
auto toTensor = bufferization::ToTensorOp::create(
rewriter, loc, argRankedTensorType, getGlobalOp, rewriter.getUnitAttr(), rewriter.getUnitAttr());
replaceAndEraseDirectComputeLikeInput(rewriter, spatComputeBatch.getOperation(), BBArgIndex, toTensor);
replaceAndEraseDirectComputeLikeInput(rewriter, argUser, *inputIndex, toTensor);
}
else {
rewriter.setInsertionPoint(argUser);
@@ -5,6 +5,7 @@
#include "llvm/Support/MathExtras.h"
#include "ContiguityPatterns.hpp"
#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp"
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
#include "src/Accelerators/PIM/Common/IR/LoopUtils.hpp"
#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
@@ -84,17 +85,6 @@ static FailureOr<int64_t> checkedPositiveMul(int64_t lhs, int64_t rhs) {
return result;
}
static FailureOr<int64_t> checkedPositiveProduct(ArrayRef<int64_t> values) {
int64_t result = 1;
for (int64_t value : values) {
auto product = checkedPositiveMul(result, value);
if (failed(product))
return failure();
result = *product;
}
return result;
}
static FailureOr<SmallVector<int64_t>> getStaticMemRefStrides(MemRefType type) {
SmallVector<int64_t> strides;
int64_t offset = 0;
@@ -348,12 +338,9 @@ inferLogicalCopyShape(MemRefType targetType, MemRefType sourceType, int64_t size
}
static FailureOr<int64_t> getContiguousSuffixRank(
Value value, ArrayRef<int64_t> copyShape, Type elementType = {}) {
Value value, ArrayRef<int64_t> copyShape) {
auto type = dyn_cast<MemRefType>(value.getType());
if (type && elementType && isPackedByteBuffer(type))
return copyShape.size();
if (!type || !type.hasStaticShape() || !hasByteSizedElementType(type.getElementType())
|| (elementType && type.getElementType() != elementType)
|| type.getRank() != static_cast<int64_t>(copyShape.size()))
return failure();
if (llvm::any_of(copyShape, [](int64_t dim) { return dim <= 0; }))
@@ -363,42 +350,19 @@ static FailureOr<int64_t> getContiguousSuffixRank(
if (failed(strides))
return failure();
int64_t expectedStride = 1;
int64_t contiguousSuffixRank = 0;
for (int64_t dim = type.getRank() - 1; dim >= 0; --dim) {
if (copyShape[dim] != 1 && (*strides)[dim] != expectedStride)
break;
++contiguousSuffixRank;
auto nextStride = checkedPositiveMul(expectedStride, copyShape[dim]);
if (failed(nextStride))
return failure();
expectedStride = *nextStride;
}
return contiguousSuffixRank;
return onnx_mlir::getContiguousSuffixRank(copyShape, *strides);
}
static FailureOr<SmallVector<int64_t>> getOuterByteStrides(
Value value, const LogicalCopyShape &copyShape, size_t outerRank) {
static FailureOr<SmallVector<int64_t>> getCopyElementStrides(
Value value, const LogicalCopyShape &copyShape) {
auto type = cast<MemRefType>(value.getType());
SmallVector<int64_t> strides;
if (isPackedByteBuffer(type))
strides = computeRowMajorStrides(copyShape.dimensions);
else {
auto proven = getProvenMemRefStrides(value);
if (failed(proven))
return failure();
strides = std::move(*proven);
}
int64_t elementByteWidth = static_cast<int64_t>(
getElementTypeSizeInBytes(copyShape.elementType));
SmallVector<int64_t> result;
for (int64_t stride : ArrayRef<int64_t>(strides).take_front(outerRank)) {
auto byteStride = checkedPositiveMul(stride, elementByteWidth);
if (failed(byteStride))
return failure();
result.push_back(*byteStride);
}
return result;
return computeRowMajorStridesChecked(copyShape.dimensions);
if (!hasByteSizedElementType(type.getElementType())
|| type.getElementType() != copyShape.elementType
|| type.getRank() != static_cast<int64_t>(copyShape.dimensions.size()))
return failure();
return getProvenMemRefStrides(value);
}
static FailureOr<CopyEndpointPlan> analyzeCopyEndpoint(Value value, Value initialByteOffset, MemRefType logicalType) {
@@ -498,20 +462,19 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO
if (failed(logicalCopyShape))
return failure();
auto targetSuffixRank = getContiguousSuffixRank(
target, logicalCopyShape->dimensions, logicalCopyShape->elementType);
auto sourceSuffixRank = getContiguousSuffixRank(
source, logicalCopyShape->dimensions, logicalCopyShape->elementType);
if (failed(targetSuffixRank) || failed(sourceSuffixRank))
auto targetStrides = getCopyElementStrides(target, *logicalCopyShape);
auto sourceStrides = getCopyElementStrides(source, *logicalCopyShape);
if (failed(targetStrides) || failed(sourceStrides))
return failure();
auto access = CopyAccessPlan::build(
logicalCopyShape->dimensions, *sourceStrides, *targetStrides);
if (failed(access))
return failure();
CopyRewritePlan plan;
plan.target = *targetPlan;
plan.source = *sourcePlan;
int64_t contiguousSuffixRank = std::min(*targetSuffixRank, *sourceSuffixRank);
if (contiguousSuffixRank
== static_cast<int64_t>(logicalCopyShape->dimensions.size())) {
if (access->getOuterShape().empty()) {
plan.kind = CopyRewritePlan::Kind::Direct;
plan.directBytes = size;
return plan;
@@ -522,30 +485,22 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO
plan.kind = CopyRewritePlan::Kind::Loop;
plan.loop.targetBaseOffset = plan.target.offset;
plan.loop.sourceBaseOffset = plan.source.offset;
plan.loop.outerShape.assign(
logicalCopyShape->dimensions.begin(),
logicalCopyShape->dimensions.end() - contiguousSuffixRank);
SmallVector<int64_t> chunkShape(
logicalCopyShape->dimensions.end() - contiguousSuffixRank,
logicalCopyShape->dimensions.end());
auto outerElements = checkedPositiveProduct(plan.loop.outerShape);
auto chunkElements = checkedPositiveProduct(chunkShape);
auto chunkBytes = failed(chunkElements)
? FailureOr<int64_t>(failure())
: checkedPositiveMul(*chunkElements, elementByteWidth);
if (failed(outerElements) || failed(chunkBytes))
plan.loop.outerShape.assign(access->getOuterShape().begin(), access->getOuterShape().end());
auto chunkBytes = checkedPositiveMul(access->getRunElements(), elementByteWidth);
if (failed(chunkBytes))
return failure();
plan.loop.outerElements = *outerElements;
plan.loop.outerElements = access->getRunCount();
plan.loop.chunkBytes = *chunkBytes;
auto targetStrides = getOuterByteStrides(
target, *logicalCopyShape, plan.loop.outerShape.size());
auto sourceStrides = getOuterByteStrides(
source, *logicalCopyShape, plan.loop.outerShape.size());
if (failed(targetStrides) || failed(sourceStrides))
return failure();
plan.loop.targetOuterByteStrides = std::move(*targetStrides);
plan.loop.sourceOuterByteStrides = std::move(*sourceStrides);
if (plan.loop.chunkBytes <= 0)
auto scaleStrides = [&](ArrayRef<int64_t> strides, SmallVectorImpl<int64_t> &result) {
for (int64_t stride : strides) {
auto byteStride = checkedPositiveMul(stride, elementByteWidth);
if (failed(byteStride)) return failure();
result.push_back(*byteStride);
}
return success();
};
if (failed(scaleStrides(access->getTargetOuterStrides(), plan.loop.targetOuterByteStrides))
|| failed(scaleStrides(access->getSourceOuterStrides(), plan.loop.sourceOuterByteStrides)))
return failure();
return plan;
}
@@ -94,8 +94,12 @@ foldDenseSubview(DenseElementsAttr denseAttr, ArrayRef<int64_t> staticOffsets, A
if (numResultElements < 0)
return failure();
auto sourceStrides = computeRowMajorStrides(sourceType.getShape());
auto resultStrides = computeRowMajorStrides(resultTensorType.getShape());
auto checkedSourceStrides = computeRowMajorStridesChecked(sourceType.getShape());
if (failed(checkedSourceStrides)) return failure();
auto sourceStrides = std::move(*checkedSourceStrides);
auto checkedResultStrides = computeRowMajorStridesChecked(resultTensorType.getShape());
if (failed(checkedResultStrides)) return failure();
auto resultStrides = std::move(*checkedResultStrides);
SmallVector<Attribute> sourceValues(denseAttr.getValues<Attribute>());
SmallVector<Attribute> resultValues(numResultElements);
for (int64_t i = 0; i < numResultElements; ++i) {
@@ -218,7 +218,9 @@ static FailureOr<DenseElementsAttr> foldConstantAlloc(memref::AllocOp allocOp, M
return failure();
SmallVector<Attribute> resultValues(numElements, fillValue);
auto resultStrides = computeRowMajorStrides(resultTensorType.getShape());
auto checkedResultStrides = computeRowMajorStridesChecked(resultTensorType.getShape());
if (failed(checkedResultStrides)) return failure();
auto resultStrides = std::move(*checkedResultStrides);
llvm::sort(copies, [](const ConstantSubviewCopy& lhs, const ConstantSubviewCopy& rhs) {
return lhs.copyOp->isBeforeInBlock(rhs.copyOp);
@@ -232,7 +234,11 @@ static FailureOr<DenseElementsAttr> foldConstantAlloc(memref::AllocOp allocOp, M
|| sourceType.getRank() != static_cast<int64_t>(copy.strides.size()))
return failure();
auto sourceStrides = computeRowMajorStrides(sourceType.getShape());
auto checkedSourceStrides = computeRowMajorStridesChecked(sourceType.getShape());
if (failed(checkedSourceStrides)) return failure();
auto sourceStrides = std::move(*checkedSourceStrides);
SmallVector<Attribute> sourceValues(copy.source.getValues<Attribute>());
for (auto [linearIndex, value] : llvm::enumerate(sourceValues)) {
SmallVector<int64_t> sourceIndices =
@@ -7,6 +7,7 @@
#include <tuple>
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
#include "src/Accelerators/PIM/Common/Support/Diagnostics.hpp"
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
#include "src/Accelerators/PIM/Dialect/Pim/Passes/Transforms/LocalMemoryPlanning/LocalMemoryPlanning.hpp"
#include "src/Accelerators/PIM/Passes/PIMPasses.h"
@@ -149,7 +150,7 @@ FailureOr<CoreMemoryPlan> buildCoreMemoryPlan(Operation* coreLikeOp) {
plan.intervals = std::move(*intervals);
auto placements = planLocalMemoryPlacements(plan.intervals, kPimLocalMemoryAddressLimit);
if (failed(placements)) {
coreLikeOp->emitError("Pim local-memory plan exceeds the signed int32 address range");
pim::emitCandidateResourceError(coreLikeOp, "Pim local-memory plan exceeds the signed int32 address range");
return failure();
}
plan.placements = std::move(*placements);
@@ -11,6 +11,7 @@ struct PreparedSpatialSchedules {
std::vector<MergeScheduleResult> candidates;
};
// Success always includes at least one candidate, including an empty compute graph.
mlir::FailureOr<PreparedSpatialSchedules> prepareSpatialSchedules(
mlir::ModuleOp module, const SchedulingTarget &target,
size_t pipelineStages, bool collectCandidates);
@@ -15,6 +15,7 @@
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CheckedArithmetic.h"
#include <algorithm>
#include <functional>
@@ -30,6 +31,7 @@
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/DeferredProjectionAnalysis.hpp"
#include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp"
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp"
#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp"
#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
#include "src/Support/TypeUtilities.hpp"
@@ -55,6 +57,7 @@ struct SchedulerCostModel {
const SchedulingTarget& target;
ComputeCostKind kind = ComputeCostKind::Execution;
mutable bool projectionFallbackReported = false;
static Cost ceilDiv(Cost numerator, Cost denominator) {
assert(denominator > 0 && "denominator must be positive");
@@ -534,18 +537,10 @@ static bool isSharedMemoryInput(Value input, const SchedulingTarget &target) {
Attribute value;
if (!matchPattern(input, m_Constant(&value)) || !isa<ElementsAttr>(value))
return false;
auto dense = dyn_cast<DenseElementsAttr>(value);
if (!target.zeroSplatUsesSharedMemory && dense && dense.isSplat()) {
auto element = dense.getSplatValue<Attribute>();
if (auto floating = dyn_cast<FloatAttr>(element))
return !floating.getValue().isZero() || floating.getValue().isNegative();
if (auto integer = dyn_cast<IntegerAttr>(element))
return !integer.getValue().isZero();
}
return true;
return target.zeroSplatUsesSharedMemory || !isZeroBitPatternSplat(value);
}
static std::pair<Cost, Cost> getProjectionMemoryRequests(Type type, Value projection,
static FailureOr<std::pair<Cost, Cost>> getProjectionMemoryRequests(Type type, Value projection,
const SchedulerCostModel &costModel) {
Cost bytes = costModel.getByteSize(type, costModel.getComputeBitwidth(type));
Cost requests = 1;
@@ -553,40 +548,53 @@ static std::pair<Cost, Cost> getProjectionMemoryRequests(Type type, Value projec
extract && extract.getSourceType().hasStaticShape()
&& !ShapedType::isDynamicShape(extract.getStaticSizes())
&& !ShapedType::isDynamicShape(extract.getStaticStrides())) {
auto strides = computeRowMajorStrides(extract.getSourceType().getShape());
Cost run = 1;
for (size_t dim = strides.size(); dim-- > 0;) {
Cost size = extract.getStaticSizes()[dim];
if (size == 1) continue;
if (checkedMultiply(static_cast<Cost>(strides[dim]),
static_cast<Cost>(extract.getStaticStrides()[dim])) != run)
break;
run = checkedMultiply(run, size);
auto unavailable = [&]() -> FailureOr<std::pair<Cost, Cost>> {
if (!costModel.projectionFallbackReported) {
extract.emitWarning("projection memory cost unavailable: using one request per element because static copy geometry exceeds supported offset or stride limits");
costModel.projectionFallbackReported = true;
}
// A physical copy cannot need more than one request per element. Round
// each element separately so sub-byte elements cannot undercharge service.
return std::make_pair(SchedulerCostModel::ceilDiv(costModel.getComputeBitwidth(type), Cost{8}),
static_cast<Cost>(cast<ShapedType>(type).getNumElements()));
};
auto strides = computeRowMajorStridesChecked(extract.getSourceType().getShape());
auto targetStrides = computeRowMajorStridesChecked(extract.getStaticSizes());
if (failed(strides) || failed(targetStrides)) return unavailable();
for (size_t dim = 0; dim < strides->size(); ++dim) {
auto stride = llvm::checkedMul((*strides)[dim], extract.getStaticStrides()[dim]);
if (!stride) return unavailable();
(*strides)[dim] = *stride;
}
Cost elements = cast<ShapedType>(type).getNumElements();
if (elements) {
auto access = CopyAccessPlan::build(extract.getStaticSizes(), *strides, *targetStrides);
if (failed(access))
return unavailable();
Cost run = access->getRunElements();
requests = checkedMultiply(requests, elements / run);
bytes = SchedulerCostModel::ceilDiv(
checkedMultiply(run, costModel.getComputeBitwidth(type)), Cost{8});
}
}
return {bytes, requests};
return std::make_pair(bytes, requests);
}
static SharedMemoryDemand getInputMemoryDemand(const InputFragment &fragment,
static FailureOr<SharedMemoryDemand> getInputMemoryDemand(const InputFragment &fragment,
const SchedulerCostModel &costModel) {
auto [bytes, runs] = getProjectionMemoryRequests(fragment.type, fragment.projection, costModel);
return costModel.target.getSharedMemoryDemand(bytes,
checkedMultiply(runs, fragment.count), false);
auto access = getProjectionMemoryRequests(fragment.type, fragment.projection, costModel);
if (failed(access)) return failure();
return costModel.target.getSharedMemoryDemand(access->first,
checkedMultiply(access->second, fragment.count), false);
}
static SharedMemoryDemand getCapturedConstantMemoryDemand(
static FailureOr<SharedMemoryDemand> getCapturedConstantMemoryDemand(
Operation *owner, const SchedulerCostModel &costModel) {
// shortcut: later aliasing/load folding can change baseline requests; exact
// accounting requires a placed, bufferized program.
SharedMemoryDemand demand;
owner->walk([&](Operation *user) {
if (user == owner) return;
auto result = owner->walk([&](Operation *user) -> WalkResult {
if (user == owner) return WalkResult::advance();
for (Value value : user->getOperands()) {
if (isa<BlockArgument>(value) || !isSharedMemoryInput(value, costModel.target)) continue;
if (auto extract = dyn_cast<tensor::ExtractOp>(user)) {
@@ -618,9 +626,13 @@ static SharedMemoryDemand getCapturedConstantMemoryDemand(
}
auto extract = dyn_cast<tensor::ExtractSliceOp>(user);
Type type = extract ? extract.getResult().getType() : value.getType();
demand += getInputMemoryDemand({type, count, extract ? extract.getResult() : value}, costModel);
auto memory = getInputMemoryDemand({type, count, extract ? extract.getResult() : value}, costModel);
if (failed(memory)) return WalkResult::interrupt();
demand += *memory;
}
return WalkResult::advance();
});
if (result.wasInterrupted()) return failure();
return demand;
}
@@ -840,10 +852,11 @@ static LogicalResult collectPublicationTransfer(Value input, size_t targetIndex,
if (auto fragments = getProjectedInputFragments(input, consumer)) {
for (const InputFragment &fragment : *fragments) {
Cost bytes = costModel.getByteSize(fragment.type, costModel.getComputeBitwidth(fragment.type));
auto [storeBytes, storeRuns] = getProjectionMemoryRequests(
auto access = getProjectionMemoryRequests(
fragment.type, fragment.projection, costModel);
if (failed(access)) return failure();
graph.publicationTransfers.push_back({sourceIndex, targetIndex, value,
0, fragment.projection, bytes, fragment.count, {}, storeBytes, storeRuns});
0, fragment.projection, bytes, fragment.count, {}, access->first, access->second});
}
} else {
Cost bytes = costModel.getByteSize(input.getType(), costModel.getComputeBitwidth(input.getType()));
@@ -1292,8 +1305,11 @@ FailureOr<ComputeGraph> buildComputeGraph(Operation* entryOp,
DenseMap<Operation *, SharedMemoryDemand> capturedMemory;
for (const auto& [targetIndex, node] : llvm::enumerate(graph.nodes)) {
auto [captured, inserted] = capturedMemory.try_emplace(node.instance.op);
if (inserted)
captured->second = getCapturedConstantMemoryDemand(node.instance.op, costModel);
if (inserted) {
auto memory = getCapturedConstantMemoryDemand(node.instance.op, costModel);
if (failed(memory)) return failure();
captured->second = *memory;
}
auto memory = captured->second;
memory.readBytes = checkedMultiply(memory.readBytes, static_cast<Cost>(node.instance.laneCount));
memory.readRequests = checkedMultiply(memory.readRequests, static_cast<Cost>(node.instance.laneCount));
@@ -1304,8 +1320,11 @@ FailureOr<ComputeGraph> buildComputeGraph(Operation* entryOp,
size_t publicationsBefore = graph.publicationTransfers.size();
if (isSharedMemoryInput(input, target)) {
if (auto fragments = getProjectedInputFragments(input, node.instance)) {
for (const InputFragment &fragment : *fragments)
graph.baselineMemory += getInputMemoryDemand(fragment, costModel);
for (const InputFragment &fragment : *fragments) {
auto memory = getInputMemoryDemand(fragment, costModel);
if (failed(memory)) return failure();
graph.baselineMemory += *memory;
}
} else {
graph.baselineMemory += target.getSharedMemoryDemand(
costModel.getByteSize(input.getType(), costModel.getComputeBitwidth(input.getType())),
@@ -6,11 +6,33 @@
namespace onnx_mlir::spatial {
bool WholeScheduleCost::eligible() const {
bool ScheduleRecurrenceEvidence::proved() const {
return status == Proved && periodGenerations && intervalNs
&& std::isfinite(*intervalNs) && *intervalNs > 0;
}
std::optional<bool> WholeScheduleCost::convergenceProved() const {
if (recurrence[0].status == ScheduleRecurrenceEvidence::NotChecked
&& recurrence[1].status == ScheduleRecurrenceEvidence::NotChecked)
return std::nullopt;
return recurrence[0].proved() && recurrence[1].proved();
}
std::optional<bool> WholeScheduleCost::loweringSupported() const {
if (loweringSupport == StaticLoweringSupport::Unknown) return std::nullopt;
return loweringSupport == StaticLoweringSupport::Supported;
}
bool WholeScheduleCost::available() const {
return primary && alternative && std::isfinite(*primary) && std::isfinite(*alternative)
&& *primary > 0 && *alternative > 0 && relativeUncertainty
&& std::isfinite(*relativeUncertainty) && *relativeUncertainty >= 0
&& *relativeUncertainty < 1;
&& *primary > 0 && *alternative > 0;
}
bool WholeScheduleCost::eligible() const {
return available() && convergenceProved() == true
&& loweringSupport == StaticLoweringSupport::Supported && selectionMargin
&& std::isfinite(*selectionMargin) && *selectionMargin >= 0
&& *selectionMargin < 1;
}
size_t selectPipelineCandidate(llvm::ArrayRef<WholeScheduleCost> scores, std::string &reason) {
@@ -21,17 +43,17 @@ size_t selectPipelineCandidate(llvm::ArrayRef<WholeScheduleCost> scores, std::st
size_t best = 0;
auto upper = [&](size_t i) {
return std::max(*scores[i].primary, *scores[i].alternative)
/ (1 - *scores[i].relativeUncertainty);
/ (1 - *scores[i].selectionMargin);
};
for (size_t i = 1; i < scores.size(); ++i)
if (scores[i].eligible() && upper(i) < upper(best)) best = i;
if (!best) { reason = "original has the lowest conservative estimate"; return 0; }
double originalLower = std::min(*scores.front().primary, *scores.front().alternative);
if (upper(best) >= originalLower / (1 + *scores.front().relativeUncertainty)) {
reason = "candidate and original overlap static prediction uncertainty";
if (upper(best) >= originalLower / (1 + *scores.front().selectionMargin)) {
reason = "candidate and original overlap the selection policy margin";
return 0;
}
reason = "candidate improves on original under both arbitration tie policies and prediction uncertainty";
reason = "candidate improves on original under both arbitration tie policies and the selection margin";
return best;
}
@@ -8,6 +8,7 @@
namespace onnx_mlir::spatial {
struct MergeScheduleResult;
inline constexpr double kPipelineSelectionMargin = 0.021;
size_t selectPipelineCandidate(llvm::ArrayRef<WholeScheduleCost> scores,
std::string &reason);
@@ -1,6 +1,7 @@
#pragma once
#include <cassert>
#include <array>
#include <cstddef>
#include <functional>
#include <optional>
@@ -39,20 +40,36 @@ struct ScheduledComputeMaterializationResult;
struct DeferredTransferPlan;
struct DeferredBoundaryPlan;
// Unknown lacks a proof. Unsupported identifies a known static backend violation.
// Supported covers checked static forms, not final allocation or code generation.
enum class StaticLoweringSupport { Unknown, Supported, Unsupported };
struct ScheduleRecurrenceEvidence {
enum Status { NotChecked, Unproved, Proved };
Status status = NotChecked;
size_t periodGenerations = 0;
std::optional<double> intervalNs;
std::string reason;
bool proved() const;
};
struct WholeScheduleCost {
// Steady-state intervals in nanoseconds. The alternative tests sensitivity.
std::optional<double> primary, alternative;
std::string reason;
std::string identity;
std::optional<double> relativeUncertainty;
// Selection policy only. This is not a prediction error bound.
std::optional<double> selectionMargin;
std::array<ScheduleRecurrenceEvidence, 2> recurrence;
StaticLoweringSupport loweringSupport = StaticLoweringSupport::Unknown;
struct AnalysisTiming {
double extractionSeconds = 0, expansionSeconds = 0, evaluationSeconds = 0;
} timing;
std::optional<bool> convergenceProved() const;
std::optional<bool> loweringSupported() const;
bool available() const;
bool eligible() const;
};
struct ScheduleCostQuery {
std::string identity;
std::function<WholeScheduleCost(llvm::json::Object &)> evaluate;
};
struct SchedulingTarget {
size_t processorCount = 0;
size_t residentWeightCapacity = 0;
@@ -76,38 +93,38 @@ struct SchedulingTarget {
Cost sharedMemoryWriteLatencyNs = 0;
bool zeroSplatUsesSharedMemory = true;
// Input is a verified, target-lowered candidate. Preparation and evaluation
// are read-only. An absent query keeps the original schedule.
std::function<mlir::FailureOr<ScheduleCostQuery>(mlir::ModuleOp,
llvm::json::Object &)> prepareScheduleCost;
// Optional read-only estimate over realized Spatial. Finite costs do not
// establish convergence or lowering support. Callers must validate these
// properties and apply the Spatial selection policy.
std::function<WholeScheduleCost(mlir::ModuleOp,
llvm::json::Object &)> estimateRealizedScheduleCost;
std::string wholeScheduleCostUnavailable;
std::function<void(const ComputeGraph &,
const ScheduledComputeMaterializationResult &, const DeferredTransferPlan &,
const DeferredBoundaryPlan &)> reportPeriodicSchedule;
Cost getLocalMemoryServiceNs(Cost bytes, bool write) const {
Cost accesses = bytes / localMemoryWidthBytes
+ (bytes % localMemoryWidthBytes != 0);
return checkedMultiply(accesses, checkedMultiply(localMemoryPeriodNs,
write ? localMemoryWriteLatencyCycles : localMemoryReadLatencyCycles));
template <typename Arithmetic = SchedulingCostArithmetic>
Cost getLocalMemoryServiceNs(Cost bytes, bool write, Arithmetic math = {}) const {
return math.multiply(math.divideUp(bytes, localMemoryWidthBytes),
math.multiply(localMemoryPeriodNs,
write ? localMemoryWriteLatencyCycles : localMemoryReadLatencyCycles));
}
template <typename Arithmetic = SchedulingCostArithmetic>
SharedMemoryDemand getSharedMemoryDemand(
Cost bytesPerRequest, Cost requests, bool write) const {
Cost bytesPerRequest, Cost requests, bool write, Arithmetic math = {}) const {
SharedMemoryDemand demand;
if (write) {
demand.writeBytes = checkedMultiply(bytesPerRequest, requests);
demand.writeBytes = math.multiply(bytesPerRequest, requests);
demand.writeRequests = requests;
} else {
demand.readBytes = checkedMultiply(bytesPerRequest, requests);
demand.readBytes = math.multiply(bytesPerRequest, requests);
demand.readRequests = requests;
}
if (sharedMemoryWidthBytes) {
Cost accesses = bytesPerRequest / sharedMemoryWidthBytes
+ (bytesPerRequest % sharedMemoryWidthBytes != 0);
demand.serviceNs = checkedMultiply(requests, checkedMultiply(accesses,
if (sharedMemoryWidthBytes)
demand.serviceNs = math.multiply(requests, math.multiply(
math.divideUp(bytesPerRequest, sharedMemoryWidthBytes),
write ? sharedMemoryWriteLatencyNs : sharedMemoryReadLatencyNs));
}
return demand;
}
@@ -127,33 +144,25 @@ struct SchedulingTarget {
Cost matrixInputBufferLatencyCycles = 0;
bool matrixPipeline = true;
Cost getMatrixComputeLatencyNs(Cost inputBitwidth) const {
auto divideRoundedUp = [](Cost value, Cost width) {
return value / width + (value % width != 0);
};
Cost inputTimes = divideRoundedUp(inputBitwidth, matrixInputResolutionBits);
Cost inputPasses = divideRoundedUp(matrixRows, matrixInputParallelism);
Cost outputPasses = divideRoundedUp(matrixColumns, matrixOutputParallelism);
Cost readCycles = divideRoundedUp(matrixReadLatencyNs, matrixPeriodNs);
Cost frontStage = matrixInputBufferLatencyCycles + matrixInputLatencyCycles
+ readCycles + matrixSampleLatencyCycles;
template <typename Arithmetic = SchedulingCostArithmetic>
Cost getMatrixComputeLatencyNs(Cost inputBitwidth, Arithmetic math = {}) const {
Cost inputTimes = math.divideUp(inputBitwidth, matrixInputResolutionBits);
Cost inputPasses = math.divideUp(matrixRows, matrixInputParallelism);
Cost outputPasses = math.divideUp(matrixColumns, matrixOutputParallelism);
Cost readCycles = math.divideUp(matrixReadLatencyNs, matrixPeriodNs);
Cost frontStage = math.add(math.add(matrixInputBufferLatencyCycles,
matrixInputLatencyCycles), math.add(readCycles, matrixSampleLatencyCycles));
Cost backPipe = std::max(matrixOutputLatencyCycles,
checkedAdd(matrixShiftLatencyCycles,
matrixBufferLatencyCycles));
Cost backStage = checkedAdd(
checkedAdd(matrixOutputLatencyCycles, matrixShiftLatencyCycles),
matrixBufferLatencyCycles);
backStage = checkedAdd(backStage, checkedMultiply(outputPasses - 1, backPipe));
Cost totalTimes = checkedMultiply(inputTimes, inputPasses);
math.add(matrixShiftLatencyCycles, matrixBufferLatencyCycles));
Cost backStage = math.add(math.add(matrixOutputLatencyCycles,
matrixShiftLatencyCycles), matrixBufferLatencyCycles);
backStage = math.add(backStage, math.multiply(math.subtract(outputPasses, 1), backPipe));
Cost totalTimes = math.multiply(inputTimes, inputPasses);
if (!matrixPipeline)
return checkedMultiply(
checkedMultiply(checkedAdd(frontStage, backStage), totalTimes),
matrixPeriodNs);
return math.multiply(math.multiply(math.add(frontStage, backStage), totalTimes), matrixPeriodNs);
Cost stagePipe = std::max(frontStage, backStage);
return checkedMultiply(
checkedAdd(checkedAdd(frontStage, backStage),
checkedMultiply(totalTimes - 1, stagePipe)),
matrixPeriodNs);
return math.multiply(math.add(math.add(frontStage, backStage),
math.multiply(math.subtract(totalTimes, 1), stagePipe)), matrixPeriodNs);
}
Cost getInterProcessorLatencyNs(size_t source,
@@ -36,6 +36,13 @@ inline T checkedMultiply(T lhs, T rhs) {
return lhs * rhs;
}
struct SchedulingCostArithmetic {
Cost add(Cost a, Cost b) const { return checkedAdd(a, b); }
Cost multiply(Cost a, Cost b) const { return checkedMultiply(a, b); }
Cost subtract(Cost a, Cost b) const { assert(a >= b); return a - b; }
Cost divideUp(Cost a, Cost b) const { assert(b); return a / b + (a % b != 0); }
};
template <typename T>
inline T addOrMax(T lhs, T rhs) {
static_assert(std::is_unsigned_v<T>, "addOrMax only supports unsigned types");