better throughput scheduling with steady state cost model, now limited to spatial
This commit is contained in:
@@ -3,6 +3,7 @@ add_pim_library(OMPimCommon
|
||||
IR/AddressAnalysis.cpp
|
||||
IR/BatchCoreUtils.cpp
|
||||
IR/ConstantUtils.cpp
|
||||
IR/CopyAccessPlan.cpp
|
||||
IR/CoreBlockUtils.cpp
|
||||
IR/EntryPointUtils.cpp
|
||||
IR/IndexingUtils.cpp
|
||||
|
||||
@@ -171,7 +171,9 @@ static llvm::FailureOr<int64_t> resolveConstantGlobalLoad(mlir::memref::LoadOp l
|
||||
if (indices.size() != static_cast<size_t>(globalType.getRank()))
|
||||
return mlir::failure();
|
||||
|
||||
auto strides = computeRowMajorStrides(globalType.getShape());
|
||||
auto checkedStrides = computeRowMajorStridesChecked(globalType.getShape());
|
||||
if (failed(checkedStrides)) return mlir::failure();
|
||||
auto strides = std::move(*checkedStrides);
|
||||
int64_t linearIndex = linearizeIndex(indices, strides);
|
||||
if (linearIndex < 0 || linearIndex >= globalType.getNumElements())
|
||||
return mlir::failure();
|
||||
@@ -308,7 +310,9 @@ llvm::FailureOr<CompiledIndexExpr> compileConstantGlobalLoad(mlir::memref::LoadO
|
||||
CompiledIndexExprNode expr;
|
||||
expr.kind = CompiledIndexExprNode::Kind::ConstantGlobalLoad;
|
||||
expr.globalOp = globalOp;
|
||||
expr.globalStrides = computeRowMajorStrides(globalType.getShape());
|
||||
auto checkedStrides = computeRowMajorStridesChecked(globalType.getShape());
|
||||
if (failed(checkedStrides)) return mlir::failure();
|
||||
expr.globalStrides = std::move(*checkedStrides);
|
||||
expr.operands.reserve(loadOp.getIndices().size());
|
||||
for (mlir::Value index : loadOp.getIndices()) {
|
||||
auto compiledIndex = compileIndexValueImpl(index);
|
||||
|
||||
@@ -10,6 +10,18 @@ using namespace mlir;
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
bool isZeroBitPatternSplat(Attribute value) {
|
||||
auto dense = dyn_cast_or_null<DenseElementsAttr>(value);
|
||||
if (!dense || !dense.isSplat())
|
||||
return false;
|
||||
Attribute element = dense.getSplatValue<Attribute>();
|
||||
if (auto floating = dyn_cast<FloatAttr>(element))
|
||||
return floating.getValue().isZero() && !floating.getValue().isNegative();
|
||||
if (auto integer = dyn_cast<IntegerAttr>(element))
|
||||
return integer.getValue().isZero();
|
||||
return false;
|
||||
}
|
||||
|
||||
ConstantPool::ConstantPool(Operation *constantAnchor, OpBuilder &builder)
|
||||
: anchor(constantAnchor), block(getConstantInsertionBlock(constantAnchor)),
|
||||
builder(builder) {
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
// Negative floating zero has a distinct representation and must not match.
|
||||
bool isZeroBitPatternSplat(mlir::Attribute value);
|
||||
|
||||
class ConstantPool {
|
||||
public:
|
||||
ConstantPool(mlir::Operation *constantAnchor, mlir::OpBuilder &builder);
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
#include "CopyAccessPlan.hpp"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/Support/CheckedArithmetic.h"
|
||||
#include <algorithm>
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
mlir::FailureOr<int64_t> getContiguousSuffixRank(
|
||||
llvm::ArrayRef<int64_t> shape, llvm::ArrayRef<int64_t> elementStrides) {
|
||||
if (shape.size() != elementStrides.size()
|
||||
|| llvm::any_of(shape, [](int64_t size) { return size <= 0; }))
|
||||
return mlir::failure();
|
||||
int64_t expectedStride = 1, rank = 0;
|
||||
for (size_t dim = shape.size(); dim-- > 0;) {
|
||||
if (shape[dim] != 1 && elementStrides[dim] != expectedStride)
|
||||
break;
|
||||
++rank;
|
||||
auto next = llvm::checkedMul(expectedStride, shape[dim]);
|
||||
if (!next) return mlir::failure();
|
||||
expectedStride = *next;
|
||||
}
|
||||
return rank;
|
||||
}
|
||||
|
||||
mlir::FailureOr<CopyAccessPlan> CopyAccessPlan::build(
|
||||
llvm::ArrayRef<int64_t> shape, llvm::ArrayRef<int64_t> sourceElementStrides,
|
||||
llvm::ArrayRef<int64_t> targetElementStrides) {
|
||||
auto sourceSuffix = getContiguousSuffixRank(shape, sourceElementStrides);
|
||||
auto targetSuffix = getContiguousSuffixRank(shape, targetElementStrides);
|
||||
if (mlir::failed(sourceSuffix) || mlir::failed(targetSuffix))
|
||||
return mlir::failure();
|
||||
size_t outerRank = shape.size() - std::min(*sourceSuffix, *targetSuffix);
|
||||
CopyAccessPlan plan;
|
||||
plan.outerShape.assign(shape.begin(), shape.begin() + outerRank);
|
||||
plan.sourceOuterStrides.assign(sourceElementStrides.begin(), sourceElementStrides.begin() + outerRank);
|
||||
plan.targetOuterStrides.assign(targetElementStrides.begin(), targetElementStrides.begin() + outerRank);
|
||||
int64_t elements = 1, maximumSource = 0, maximumTarget = 0;
|
||||
for (size_t dim = 0; dim < shape.size(); ++dim) {
|
||||
if (dim < outerRank && (sourceElementStrides[dim] < 0 || targetElementStrides[dim] < 0))
|
||||
return mlir::failure();
|
||||
int64_t &product = dim < outerRank ? plan.runCount : plan.runElements;
|
||||
auto next = llvm::checkedMul(product, shape[dim]);
|
||||
auto total = llvm::checkedMul(elements, shape[dim]);
|
||||
if (!next || !total) return mlir::failure();
|
||||
product = *next;
|
||||
elements = *total;
|
||||
// Unit dimensions never access their stride, including negative strides.
|
||||
if (shape[dim] != 1) {
|
||||
auto sourceEnd = llvm::checkedMulAdd(shape[dim] - 1, sourceElementStrides[dim], maximumSource);
|
||||
auto targetEnd = llvm::checkedMulAdd(shape[dim] - 1, targetElementStrides[dim], maximumTarget);
|
||||
if (!sourceEnd || !targetEnd) return mlir::failure();
|
||||
maximumSource = *sourceEnd;
|
||||
maximumTarget = *targetEnd;
|
||||
}
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
mlir::FailureOr<std::pair<int64_t, int64_t>> CopyAccessPlan::getRunOffsets(int64_t run) const {
|
||||
if (run < 0 || run >= runCount) return mlir::failure();
|
||||
int64_t source = 0, target = 0;
|
||||
for (size_t dim = outerShape.size(); dim-- > 0;) {
|
||||
int64_t coordinate = run % outerShape[dim];
|
||||
run /= outerShape[dim];
|
||||
auto nextSource = llvm::checkedMulAdd(coordinate, sourceOuterStrides[dim], source);
|
||||
auto nextTarget = llvm::checkedMulAdd(coordinate, targetOuterStrides[dim], target);
|
||||
if (!nextSource || !nextTarget) return mlir::failure();
|
||||
source = *nextSource;
|
||||
target = *nextTarget;
|
||||
}
|
||||
return std::pair{source, target};
|
||||
}
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#include "mlir/Support/LogicalResult.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
// Geometry only. Callers must prove storage layout, aliasing, and copy legality.
|
||||
mlir::FailureOr<int64_t> getContiguousSuffixRank(
|
||||
llvm::ArrayRef<int64_t> shape, llvm::ArrayRef<int64_t> elementStrides);
|
||||
|
||||
class CopyAccessPlan {
|
||||
public:
|
||||
// All accessed relative element offsets must fit int64_t, including run tails.
|
||||
// Byte scaling and addition of base addresses remain caller responsibilities.
|
||||
static mlir::FailureOr<CopyAccessPlan> build(llvm::ArrayRef<int64_t> shape,
|
||||
llvm::ArrayRef<int64_t> sourceElementStrides,
|
||||
llvm::ArrayRef<int64_t> targetElementStrides);
|
||||
|
||||
int64_t getRunCount() const { return runCount; }
|
||||
int64_t getRunElements() const { return runElements; }
|
||||
llvm::ArrayRef<int64_t> getOuterShape() const { return outerShape; }
|
||||
llvm::ArrayRef<int64_t> getSourceOuterStrides() const { return sourceOuterStrides; }
|
||||
llvm::ArrayRef<int64_t> getTargetOuterStrides() const { return targetOuterStrides; }
|
||||
// Source and target element offsets, relative to their separately proven bases.
|
||||
mlir::FailureOr<std::pair<int64_t, int64_t>> getRunOffsets(int64_t run) const;
|
||||
|
||||
private:
|
||||
int64_t runCount = 1;
|
||||
int64_t runElements = 1;
|
||||
llvm::SmallVector<int64_t> outerShape, sourceOuterStrides, targetOuterStrides;
|
||||
};
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -1,6 +1,7 @@
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/CheckedArithmetic.h"
|
||||
|
||||
#include <functional>
|
||||
|
||||
@@ -9,9 +10,20 @@
|
||||
namespace onnx_mlir {
|
||||
|
||||
llvm::SmallVector<int64_t> computeRowMajorStrides(llvm::ArrayRef<int64_t> shape) {
|
||||
auto strides = computeRowMajorStridesChecked(shape);
|
||||
if (mlir::failed(strides))
|
||||
llvm::report_fatal_error("row-major strides are not representable in int64_t");
|
||||
return std::move(*strides);
|
||||
}
|
||||
|
||||
mlir::FailureOr<llvm::SmallVector<int64_t>>
|
||||
computeRowMajorStridesChecked(llvm::ArrayRef<int64_t> shape) {
|
||||
llvm::SmallVector<int64_t> strides(shape.size(), 1);
|
||||
for (int64_t dim = static_cast<int64_t>(shape.size()) - 2; dim >= 0; --dim)
|
||||
strides[dim] = strides[dim + 1] * shape[dim + 1];
|
||||
for (int64_t dim = static_cast<int64_t>(shape.size()) - 2; dim >= 0; --dim) {
|
||||
auto stride = llvm::checkedMul(strides[dim + 1], shape[dim + 1]);
|
||||
if (!stride) return mlir::failure();
|
||||
strides[dim] = *stride;
|
||||
}
|
||||
return strides;
|
||||
}
|
||||
|
||||
|
||||
@@ -19,7 +19,11 @@ namespace onnx_mlir {
|
||||
using HSliceId = size_t;
|
||||
using CoreId = size_t;
|
||||
|
||||
// Invariant form fails explicitly on overflow, including assertion-free builds.
|
||||
// Optional analyses and rewrites must use the checked form.
|
||||
llvm::SmallVector<int64_t> computeRowMajorStrides(llvm::ArrayRef<int64_t> shape);
|
||||
mlir::FailureOr<llvm::SmallVector<int64_t>>
|
||||
computeRowMajorStridesChecked(llvm::ArrayRef<int64_t> shape);
|
||||
|
||||
struct TransposeRun {
|
||||
size_t start;
|
||||
|
||||
@@ -191,7 +191,9 @@ resolveWeightView(mlir::Operation* coreLikeOp, mlir::Value weight, const StaticV
|
||||
ResolvedWeightView view;
|
||||
view.globalOp = globalOp;
|
||||
view.shape.assign(denseAttr.getType().getShape().begin(), denseAttr.getType().getShape().end());
|
||||
view.strides = computeRowMajorStrides(view.shape);
|
||||
auto checkedStrides = computeRowMajorStridesChecked(view.shape);
|
||||
if (failed(checkedStrides)) return mlir::failure();
|
||||
view.strides = std::move(*checkedStrides);
|
||||
|
||||
CompiledIndexExpr offsetExpr = makeConstantExpr(0);
|
||||
for (mlir::Operation* viewOp : llvm::reverse(viewOps)) {
|
||||
|
||||
@@ -4,6 +4,21 @@
|
||||
|
||||
namespace onnx_mlir::pim {
|
||||
|
||||
static constexpr llvm::StringLiteral candidateResourceFailure = "pim.candidate-resource-limit";
|
||||
|
||||
mlir::InFlightDiagnostic emitCandidateResourceError(mlir::Operation *op, llvm::StringRef message) {
|
||||
auto diagnostic = op->emitError(message);
|
||||
diagnostic.getUnderlyingDiagnostic()->getMetadata().emplace_back(llvm::StringRef(candidateResourceFailure));
|
||||
return diagnostic;
|
||||
}
|
||||
|
||||
bool isCandidateResourceError(mlir::Diagnostic &diagnostic) {
|
||||
return llvm::any_of(diagnostic.getMetadata(), [](const mlir::DiagnosticArgument &argument) {
|
||||
return argument.getKind() == mlir::DiagnosticArgument::DiagnosticArgumentKind::String
|
||||
&& argument.getAsString() == candidateResourceFailure;
|
||||
});
|
||||
}
|
||||
|
||||
mlir::InFlightDiagnostic emitUnsupportedStaticShapeDiagnostic(mlir::Operation* op, llvm::StringRef valueDescription) {
|
||||
return op->emitOpError() << "requires statically shaped " << valueDescription;
|
||||
}
|
||||
|
||||
@@ -12,6 +12,10 @@
|
||||
|
||||
namespace onnx_mlir::pim {
|
||||
|
||||
/// Only a backend owner can classify a failed schedule as resource-infeasible.
|
||||
mlir::InFlightDiagnostic emitCandidateResourceError(mlir::Operation *op, llvm::StringRef message);
|
||||
bool isCandidateResourceError(mlir::Diagnostic &diagnostic);
|
||||
|
||||
struct CappedDiagnosticReporter {
|
||||
explicit CappedDiagnosticReporter(int64_t maxReportedFailures = 8)
|
||||
: maxReportedFailures(maxReportedFailures) {}
|
||||
|
||||
@@ -17,6 +17,8 @@ add_pim_library(OMPimCompilerUtils
|
||||
PimCompilerUtils.cpp
|
||||
PimScheduleDiagnostic.cpp
|
||||
PimPeriodicAnalysis.cpp
|
||||
PimQueueReporting.cpp
|
||||
PimSpatialCost.cpp
|
||||
PimPipelineSelection.cpp
|
||||
PimArtifactWriter.cpp
|
||||
PimCodeGen.cpp
|
||||
|
||||
@@ -14,6 +14,7 @@ namespace onnx_mlir::pim_binary {
|
||||
|
||||
inline constexpr char kMagic[4] = {'P', 'I', 'M', 'B'};
|
||||
inline constexpr uint32_t kVersion = 1;
|
||||
inline constexpr int32_t kMatrixWeightPrecisionBits = 8;
|
||||
inline constexpr uint64_t kCountOffset = 8;
|
||||
inline constexpr size_t kHeaderSize = 12;
|
||||
inline constexpr size_t kRecordSize = 20;
|
||||
|
||||
@@ -22,7 +22,6 @@
|
||||
#include <absl/types/compare.h>
|
||||
#include <algorithm>
|
||||
#include <cassert>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <limits>
|
||||
@@ -32,6 +31,7 @@
|
||||
#include <utility>
|
||||
|
||||
#include "Common/IR/CompactAsmUtils.hpp"
|
||||
#include "Common/IR/ConstantUtils.hpp"
|
||||
#include "Common/PimCommon.hpp"
|
||||
#include "Common/Support/CheckedArithmetic.hpp"
|
||||
#include "Common/Support/Diagnostics.hpp"
|
||||
@@ -42,7 +42,6 @@
|
||||
#include "src/Accelerators/PIM/Common/Support/FileSystemUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Compiler/PimArtifactWriter.hpp"
|
||||
#include "src/Accelerators/PIM/Compiler/PimBinaryFormat.hpp"
|
||||
#include "src/Accelerators/PIM/Compiler/PimPeriodicAnalysis.hpp"
|
||||
#include "src/Accelerators/PIM/Compiler/PimCodeGen.hpp"
|
||||
#include "src/Accelerators/PIM/Compiler/PimCompilerOptions.hpp"
|
||||
#include "src/Accelerators/PIM/Compiler/PimCoreProgram.hpp"
|
||||
@@ -116,15 +115,7 @@ static bool isZeroSplatGlobal(mlir::Value value) {
|
||||
auto globalOp = moduleOp ? lookupGlobalForGetGlobal(moduleOp, getGlobalOp) : memref::GlobalOp();
|
||||
if (!globalOp || !globalOp.getConstant() || !globalOp.getInitialValue())
|
||||
return false;
|
||||
auto denseAttr = dyn_cast<DenseElementsAttr>(*globalOp.getInitialValue());
|
||||
if (!denseAttr || !denseAttr.isSplat())
|
||||
return false;
|
||||
Attribute valueAttr = denseAttr.getSplatValue<Attribute>();
|
||||
if (auto floatAttr = dyn_cast<FloatAttr>(valueAttr))
|
||||
return floatAttr.getValue().isZero() && !floatAttr.getValue().isNegative();
|
||||
if (auto integerAttr = dyn_cast<IntegerAttr>(valueAttr))
|
||||
return integerAttr.getValue().isZero();
|
||||
return false;
|
||||
return isZeroBitPatternSplat(*globalOp.getInitialValue());
|
||||
}
|
||||
|
||||
// Pim instruction immediates are serialized as signed int32_t fields today
|
||||
@@ -613,7 +604,7 @@ void PimCodeGen::emitMvmOp(size_t groupId, size_t rdAddr, size_t rdOffset, size_
|
||||
instruction.opcode = pim_binary::Opcode::mvmul;
|
||||
instruction.rd = 0;
|
||||
instruction.r1 = 1;
|
||||
instruction.r2OrImm = 8;
|
||||
instruction.r2OrImm = pim_binary::kMatrixWeightPrecisionBits;
|
||||
instruction.generic1 = 0;
|
||||
instruction.generic2 = pim::checkedI32OrCrash(groupId, "mvm group id");
|
||||
emitInstruction(instruction);
|
||||
@@ -895,7 +886,6 @@ struct CoreEmissionResult {
|
||||
|
||||
OnnxMlirCompilerErrorCodes status = CompilerSuccess;
|
||||
MemoryReportRow reportRow;
|
||||
std::string instructions;
|
||||
llvm::SmallVector<ResolvedWeightView, 8> usedWeights;
|
||||
llvm::SmallVector<DiagnosticRecord, kMaxStoredCodegenDiagnostics> diagnostics;
|
||||
size_t diagnosticCount = 0;
|
||||
@@ -1150,10 +1140,8 @@ static OnnxMlirCompilerErrorCodes emitEmptyCoreArtifacts(StringRef outputDirPath
|
||||
return CompilerSuccess;
|
||||
}
|
||||
|
||||
OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::string& outputDirPath,
|
||||
PimInstructionProgram *instructions, PimCodegenTiming *timing) {
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
if (!instructions && !outputDirPath.empty()) {
|
||||
OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::string& outputDirPath) {
|
||||
if (!outputDirPath.empty()) {
|
||||
if (auto error = sys::fs::create_directory(outputDirPath)) {
|
||||
errs() << "Error creating output directory: " << outputDirPath << ": " << error.message() << '\n';
|
||||
return InvalidOutputFileAccess;
|
||||
@@ -1169,9 +1157,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
|
||||
memory.hostMem.allocateHost(moduleOp, funcOp);
|
||||
memory.reportHost();
|
||||
|
||||
if (!instructions)
|
||||
if (auto err = writeMemoryBinary(moduleOp, funcOp, memory, outputDirPath))
|
||||
return err;
|
||||
if (auto err = writeMemoryBinary(moduleOp, funcOp, memory, outputDirPath))
|
||||
return err;
|
||||
|
||||
json::Object xbarsPerArrayGroup;
|
||||
size_t maxCoreId = 0;
|
||||
@@ -1301,22 +1288,16 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
|
||||
|
||||
std::error_code errorCode;
|
||||
auto outputCorePath = outputDirPath + "/core_" + std::to_string(job.physicalCoreId) + ".pim";
|
||||
llvm::SmallVector<char, 0> binary;
|
||||
raw_svector_ostream memoryStream(binary);
|
||||
std::unique_ptr<raw_fd_ostream> fileStream;
|
||||
if (!instructions) {
|
||||
fileStream = std::make_unique<raw_fd_ostream>(outputCorePath, errorCode, sys::fs::OF_None);
|
||||
if (errorCode) {
|
||||
errs() << "Error opening core file: " << outputCorePath << ": " << errorCode.message() << '\n';
|
||||
result.status = InvalidOutputFileAccess;
|
||||
return result;
|
||||
}
|
||||
auto fileStream = std::make_unique<raw_fd_ostream>(outputCorePath, errorCode, sys::fs::OF_None);
|
||||
if (errorCode) {
|
||||
errs() << "Error opening core file: " << outputCorePath << ": " << errorCode.message() << '\n';
|
||||
result.status = InvalidOutputFileAccess;
|
||||
return result;
|
||||
}
|
||||
raw_pwrite_stream &coreBinaryStream = instructions
|
||||
? static_cast<raw_pwrite_stream &>(memoryStream) : *fileStream;
|
||||
raw_pwrite_stream &coreBinaryStream = *fileStream;
|
||||
|
||||
std::unique_ptr<raw_fd_ostream> coreJsonStream;
|
||||
if (!instructions && pimEmitJson.getValue()) {
|
||||
if (pimEmitJson.getValue()) {
|
||||
std::string outputCoreJsonPath = outputDirPath + "/core_" + std::to_string(job.physicalCoreId) + ".json";
|
||||
errorCode = std::error_code();
|
||||
coreJsonStream = std::make_unique<raw_fd_ostream>(outputCoreJsonPath, errorCode);
|
||||
@@ -1332,12 +1313,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
|
||||
PimCodeGen coreCodeGen(jobMemory, instructionWriter, coreJsonStream.get());
|
||||
auto finalizeInstructions = [&]() {
|
||||
bool succeeded = mlir::succeeded(instructionWriter.finalize());
|
||||
if (fileStream) {
|
||||
fileStream->close();
|
||||
if (fileStream->has_error()) { fileStream->clear_error(); succeeded = false; }
|
||||
} else {
|
||||
result.instructions.assign(binary.begin(), binary.end());
|
||||
}
|
||||
fileStream->close();
|
||||
if (fileStream->has_error()) { fileStream->clear_error(); succeeded = false; }
|
||||
return succeeded;
|
||||
};
|
||||
|
||||
@@ -1390,14 +1367,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
|
||||
};
|
||||
|
||||
std::vector<CoreEmissionResult> jobResults(jobs.size());
|
||||
auto emissionStart = std::chrono::steady_clock::now();
|
||||
if (timing)
|
||||
timing->preparationSeconds = std::chrono::duration<double>(emissionStart - start).count();
|
||||
mlir::parallelFor(
|
||||
moduleOp.getContext(), 0, jobs.size(), [&](size_t index) { jobResults[index] = emitJob(jobs[index]); });
|
||||
if (timing)
|
||||
timing->emissionSeconds = std::chrono::duration<double>(
|
||||
std::chrono::steady_clock::now() - emissionStart).count();
|
||||
|
||||
pim::CappedDiagnosticReporter diagnostics;
|
||||
Operation* summaryAnchor = nullptr;
|
||||
@@ -1427,33 +1398,13 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::
|
||||
physicalCoreId < activePhysicalCores.size(); ++physicalCoreId) {
|
||||
if (activePhysicalCores[physicalCoreId])
|
||||
continue;
|
||||
if (!instructions)
|
||||
if (auto err = emitEmptyCoreArtifacts(outputDirPath, physicalCoreId))
|
||||
return err;
|
||||
if (auto err = emitEmptyCoreArtifacts(outputDirPath, physicalCoreId))
|
||||
return err;
|
||||
xbarsPerArrayGroup["core" + std::to_string(physicalCoreId)] =
|
||||
json::Array {};
|
||||
memory.recordCoreReport(physicalCoreId, MemoryReportRow {});
|
||||
}
|
||||
|
||||
if (instructions) {
|
||||
instructions->instructions.resize(maxPhysicalCoreId + 1);
|
||||
instructions->arrayGroups.resize(maxPhysicalCoreId + 1);
|
||||
for (size_t core = 0; core <= maxPhysicalCoreId; ++core) {
|
||||
instructions->arrayGroups[core].clear();
|
||||
llvm::SmallVector<char> empty;
|
||||
raw_svector_ostream stream(empty);
|
||||
pim_binary::writeHeader(stream);
|
||||
instructions->instructions[core].assign(empty.begin(), empty.end());
|
||||
}
|
||||
for (size_t i = 0; i < jobs.size(); ++i) {
|
||||
size_t core = jobs[i].physicalCoreId;
|
||||
instructions->instructions[core] = std::move(jobResults[i].instructions);
|
||||
for (const auto &weight : jobResults[i].usedWeights)
|
||||
instructions->arrayGroups[core].push_back(weight.shape[1] / crossbarSize.getValue());
|
||||
}
|
||||
return CompilerSuccess;
|
||||
}
|
||||
|
||||
llvm::SmallVector<WeightFileRequest, 8> weightRequests;
|
||||
weightRequests.reserve(jobs.size());
|
||||
for (size_t jobIndex = 0; jobIndex < jobs.size(); ++jobIndex) {
|
||||
|
||||
@@ -25,7 +25,6 @@
|
||||
namespace onnx_mlir {
|
||||
|
||||
struct CompiledCoreProgram;
|
||||
struct PimInstructionProgram;
|
||||
class PimInstructionWriter;
|
||||
|
||||
struct MemEntry {
|
||||
@@ -227,13 +226,7 @@ public:
|
||||
|
||||
};
|
||||
|
||||
struct PimCodegenTiming {
|
||||
double preparationSeconds = 0;
|
||||
double emissionSeconds = 0;
|
||||
};
|
||||
|
||||
OnnxMlirCompilerErrorCodes compileToPimCode(mlir::ModuleOp& moduleOpRef, std::string& outputDirName,
|
||||
PimInstructionProgram *instructions = nullptr, PimCodegenTiming *timing = nullptr);
|
||||
OnnxMlirCompilerErrorCodes compileToPimCode(mlir::ModuleOp& moduleOpRef, std::string& outputDirName);
|
||||
|
||||
} // namespace onnx_mlir
|
||||
|
||||
|
||||
@@ -90,7 +90,7 @@ llvm::cl::opt<bool> pimReportConvLowering("pim-report-conv-lowering",
|
||||
llvm::cl::cat(OnnxMlirOptions));
|
||||
|
||||
llvm::cl::opt<bool> pimPeriodicDiagnostics("pim-periodic-diagnostics",
|
||||
llvm::cl::desc("Emit the optional pre-codegen recurrence timing report"),
|
||||
llvm::cl::desc("Emit optional Spatial recurrence and execution-work reports"),
|
||||
llvm::cl::init(false), llvm::cl::cat(OnnxMlirOptions));
|
||||
|
||||
llvm::cl::opt<bool> pimEmitJson("pim-emit-json",
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
#include "src/Accelerators/PIM/Compiler/PimCompilerOptions.hpp"
|
||||
#include "PimPipelineSelection.hpp"
|
||||
#include "PimInstructionTiming.hpp"
|
||||
#include "PimPeriodicAnalysis.hpp"
|
||||
#include "PimSpatialCost.hpp"
|
||||
#include "src/Accelerators/PIM/Compiler/PimCompilerUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#pragma once
|
||||
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
// Emptiness is normalized after consumption. Capacity and consumed prefixes
|
||||
// cannot affect virtual timing, readiness, or the remaining FIFO sequence.
|
||||
class PimDependencyTokens {
|
||||
llvm::SmallVector<uint64_t, 1> values;
|
||||
size_t head = 0;
|
||||
public:
|
||||
void initialize(size_t distance) { values.assign(distance, 0); head = 0; }
|
||||
bool empty() const { return values.empty(); }
|
||||
void push(uint64_t time) { values.push_back(time); }
|
||||
uint64_t consume() {
|
||||
assert(!empty());
|
||||
uint64_t time = values[head++];
|
||||
if (head == values.size()) { values.clear(); head = 0; }
|
||||
else if (head >= values.size() - head) {
|
||||
values.erase(values.begin(), values.begin() + head);
|
||||
head = 0;
|
||||
}
|
||||
return time;
|
||||
}
|
||||
llvm::ArrayRef<uint64_t> remaining() const { return llvm::ArrayRef(values).drop_front(head); }
|
||||
};
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -6,6 +6,7 @@ namespace onnx_mlir {
|
||||
|
||||
// PIM transport and instruction timing. Spatial consumes only the base costs.
|
||||
struct PimInstructionTiming : spatial::SchedulingTarget {
|
||||
Cost matrixCellBits = 0;
|
||||
bool peerPermissionThenSourceRead = false;
|
||||
Cost signalIssueNs = 0;
|
||||
Cost controlMessageBytes = 0;
|
||||
@@ -22,15 +23,17 @@ struct PimInstructionTiming : spatial::SchedulingTarget {
|
||||
&& localMemoryWidthBytes && transferWidthBytes;
|
||||
}
|
||||
|
||||
Cost getLinkServiceNs(Cost bytes, size_t source, size_t destination) const {
|
||||
Cost flits = bytes / transferWidthBytes + (bytes % transferWidthBytes != 0);
|
||||
return checkedMultiply(flits,
|
||||
template <typename Arithmetic = SchedulingCostArithmetic>
|
||||
Cost getLinkServiceNs(Cost bytes, size_t source, size_t destination, Arithmetic math = {}) const {
|
||||
Cost flits = math.divideUp(bytes, transferWidthBytes);
|
||||
return math.multiply(flits,
|
||||
getInterProcessorLatencyNs(source, destination));
|
||||
}
|
||||
|
||||
Cost getSharedMemoryLinkServiceNs(Cost bytes, size_t core, bool response) const {
|
||||
Cost flits = bytes / transferWidthBytes + (bytes % transferWidthBytes != 0);
|
||||
return checkedMultiply(flits, response ? sharedMemoryResponseLatencyNs[core]
|
||||
template <typename Arithmetic = SchedulingCostArithmetic>
|
||||
Cost getSharedMemoryLinkServiceNs(Cost bytes, size_t core, bool response, Arithmetic math = {}) const {
|
||||
Cost flits = math.divideUp(bytes, transferWidthBytes);
|
||||
return math.multiply(flits, response ? sharedMemoryResponseLatencyNs[core]
|
||||
: sharedMemoryRequestLatencyNs[core]);
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,14 +1,65 @@
|
||||
#pragma once
|
||||
|
||||
#include "llvm/Support/JSON.h"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp"
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace onnx_mlir {
|
||||
namespace spatial { struct SchedulingTarget; }
|
||||
struct PimInstructionTiming;
|
||||
void configurePimScheduleCosts(spatial::SchedulingTarget &target, PimInstructionTiming timing);
|
||||
|
||||
enum class PimQueueTermination {
|
||||
NotRun, IntervalConverged, RecurrenceProved, EventBudget, GenerationLimit,
|
||||
HistoryLimit, NonMonotonic, NoReadyEvents
|
||||
};
|
||||
|
||||
struct PimQueueEventLocation {
|
||||
int64_t core = 0;
|
||||
uint32_t pc = 0;
|
||||
std::string kind;
|
||||
};
|
||||
|
||||
struct PimQueueProgress {
|
||||
size_t processedEvents = 0, completedGenerations = 0;
|
||||
uint64_t maximumQueueWaitNs = 0;
|
||||
std::optional<PimQueueEventLocation> maximumQueueRequest, generationLimit;
|
||||
std::vector<std::pair<int64_t, double>> perCoreQueueWaitNs;
|
||||
std::vector<uint64_t> completedIntervalsNs;
|
||||
};
|
||||
|
||||
struct PimQueueIntervalWindow {
|
||||
std::optional<double> intervalNs;
|
||||
std::string reason;
|
||||
PimQueueTermination termination = PimQueueTermination::NotRun;
|
||||
PimQueueProgress progress;
|
||||
};
|
||||
|
||||
struct PimQueuePolicyResult {
|
||||
PimQueueIntervalWindow legacy;
|
||||
spatial::ScheduleRecurrenceEvidence recurrence;
|
||||
PimQueueProgress progress;
|
||||
PimQueueTermination termination = PimQueueTermination::NotRun;
|
||||
size_t checkpoints = 0, historyBytes = 0, stateWords = 0, proofProcessedEvents = 0;
|
||||
bool reverseTies = false;
|
||||
std::optional<double> intervalNs() const;
|
||||
};
|
||||
|
||||
struct PimQueueEvaluation {
|
||||
std::array<PimQueuePolicyResult, 2> policies;
|
||||
uint64_t memoryServiceNs = 0;
|
||||
double evaluationSeconds = 0;
|
||||
bool recurrenceRequired = false, details = false, measureTime = false;
|
||||
spatial::WholeScheduleCost cost() const;
|
||||
};
|
||||
|
||||
llvm::json::Object serializePimQueueEvaluation(const PimQueueEvaluation &evaluation);
|
||||
|
||||
struct PimPeriodicEstimate {
|
||||
spatial::WholeScheduleCost cost;
|
||||
llvm::json::Object report;
|
||||
std::optional<PimQueueEvaluation> queue;
|
||||
};
|
||||
|
||||
// Binary instructions, indexed by physical core. No executable or measured data.
|
||||
struct PimInstructionProgram {
|
||||
@@ -16,15 +67,58 @@ struct PimInstructionProgram {
|
||||
std::vector<std::vector<int64_t>> arrayGroups;
|
||||
};
|
||||
|
||||
// Hardware work only. Bytes are already in the target timing precision.
|
||||
// Repeat keeps ordered request patterns compact until queue construction.
|
||||
struct PimExecutionEvent {
|
||||
enum Kind { Work, Load, Store, Send, Receive, Signal, Wait, Repeat };
|
||||
enum WorkClass { Matrix, Vector, LocalCopy, Fill, VectorCopy, ScalarIssue };
|
||||
Kind kind = Work;
|
||||
uint64_t amount = 0;
|
||||
int64_t peer = 0, reg = 0, count = 0;
|
||||
std::vector<PimExecutionEvent> body;
|
||||
bool constant = false;
|
||||
WorkClass workClass = Fill;
|
||||
PimExecutionEvent(Kind kind, uint64_t amount, int64_t peer = 0,
|
||||
int64_t reg = 0, int64_t count = 0, std::vector<PimExecutionEvent> body = {})
|
||||
: kind(kind), amount(amount), peer(peer), reg(reg), count(count), body(std::move(body)) {}
|
||||
};
|
||||
using PimExecutionProgram = std::vector<std::vector<PimExecutionEvent>>;
|
||||
|
||||
struct PimPeriodicOptions {
|
||||
bool networkQueues = true;
|
||||
bool instructionIssue = true;
|
||||
bool scalarIssue = true;
|
||||
bool ignoreLocalCopies = false;
|
||||
bool workBreakdown = false;
|
||||
bool measureTime = false;
|
||||
// Optional ordered work/IO extraction for timing validation; no execution.
|
||||
PimExecutionProgram *executionTrace = nullptr;
|
||||
// Prove recurrence of the modeled state. Instruction diagnostics retain
|
||||
// their legacy interval-window result separately.
|
||||
bool verifyPeriodicState = false;
|
||||
// Optional diagnostic cap. The complete-state proof keeps its original budget.
|
||||
std::optional<size_t> intervalWindowEventBudget;
|
||||
bool memoryController = true;
|
||||
bool detailed = false;
|
||||
// Apply the oracle's fp32-to-int8 timing projection without changing the program.
|
||||
bool oraclePrecision = false;
|
||||
};
|
||||
|
||||
llvm::json::Object analyzePimExecution(const PimExecutionProgram &program,
|
||||
const PimInstructionTiming &target, PimPeriodicOptions options = {});
|
||||
PimPeriodicEstimate evaluatePimExecution(const PimExecutionProgram &program,
|
||||
const PimInstructionTiming &target, PimPeriodicOptions options = {});
|
||||
|
||||
uint64_t getPimLocalTransferWorkNs(const PimInstructionTiming &target,
|
||||
uint64_t bytes, bool read, bool write, bool instructionIssue = true);
|
||||
uint64_t getPimVectorWorkNs(const PimInstructionTiming &target,
|
||||
uint64_t elements, uint64_t inputBytes, uint64_t outputBytes,
|
||||
bool binary, bool instructionIssue = true);
|
||||
uint64_t getPimMatrixOutputBytes(const PimInstructionTiming &target,
|
||||
uint64_t cellBits, uint64_t weightBits, uint64_t groups, uint64_t outputBytes);
|
||||
uint64_t getPimMatrixWorkNs(const PimInstructionTiming &target,
|
||||
uint64_t inputBits, uint64_t outputBytes, bool instructionIssue = true);
|
||||
|
||||
llvm::json::Object analyzePimInstructions(const PimInstructionProgram &program,
|
||||
const PimInstructionTiming &target, int64_t cellBits,
|
||||
PimPeriodicOptions options = {});
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
#include "llvm/Support/FormatVariadic.h"
|
||||
#include "llvm/Support/JSON.h"
|
||||
#include "llvm/Support/raw_os_ostream.h"
|
||||
#include <algorithm>
|
||||
#include "src/Accelerators/PIM/Common/Support/Diagnostics.hpp"
|
||||
#include <chrono>
|
||||
|
||||
using namespace mlir;
|
||||
@@ -57,6 +57,14 @@ struct CandidateTiming final : PassInstrumentation {
|
||||
}
|
||||
};
|
||||
|
||||
void writeSelectionReport(llvm::json::Object report) {
|
||||
auto file = openReportFileWithExtension("pipeline_selection", "json");
|
||||
if (file) {
|
||||
llvm::raw_os_ostream out(file);
|
||||
out << llvm::formatv("{0:2}\n", llvm::json::Value(std::move(report)));
|
||||
}
|
||||
}
|
||||
|
||||
struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, OperationPass<ModuleOp>> {
|
||||
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(PimPipelineSelectionPass)
|
||||
PimPipelineSelectionPass(const SchedulingTarget &target, const SpatialTargetResources &resources,
|
||||
@@ -68,156 +76,256 @@ struct PimPipelineSelectionPass final : PassWrapper<PimPipelineSelectionPass, Op
|
||||
SpatialDataflowExportStage exportStage;
|
||||
|
||||
StringRef getArgument() const override { return "pim-pipeline-selection"; }
|
||||
StringRef getDescription() const override { return "Select a throughput pipeline using static instruction queue analysis"; }
|
||||
StringRef getDescription() const override { return "Select a throughput pipeline using static Spatial queue analysis"; }
|
||||
|
||||
LogicalResult lower(ModuleOp module, llvm::json::Object *summary = nullptr) {
|
||||
PassManager pm(module.getContext());
|
||||
addPimLoweringPasses(pm, resources, EmitPimCodegen, /*reportProgress=*/false);
|
||||
if (enableTiming && summary)
|
||||
pm.addInstrumentation(std::make_unique<CandidateTiming>(*summary));
|
||||
return pm.run(module);
|
||||
}
|
||||
|
||||
LogicalResult evaluate(ModuleOp checkpoint, const MergeScheduleResult &schedule,
|
||||
OwningOpRef<ModuleOp> &candidate,
|
||||
std::string &scheduleReport, llvm::json::Object &summary) {
|
||||
LogicalResult realizeCandidate(ModuleOp checkpoint, const MergeScheduleResult &schedule,
|
||||
OwningOpRef<ModuleOp> &candidate, std::string &scheduleReport,
|
||||
llvm::json::Object &summary) {
|
||||
auto begin = Clock::now();
|
||||
IRMapping mapping;
|
||||
candidate = cast<ModuleOp>(checkpoint->clone(mapping));
|
||||
auto remapped = remapSpatialSchedule(schedule, mapping);
|
||||
if (failed(realizeSpatialSchedule(*candidate, remapped, target, stages,
|
||||
SpatialDataflowExportStage::None, nullptr, &scheduleReport))
|
||||
|| failed(lower(*candidate, &summary))) return failure();
|
||||
if (enableTiming) summary["codegen_input_ir"] = countIR(candidate->getOperation());
|
||||
summary["lowering_seconds"] = elapsed(begin);
|
||||
SpatialDataflowExportStage::None, nullptr, &scheduleReport))) return failure();
|
||||
summary["realization_seconds"] = elapsed(begin);
|
||||
return success();
|
||||
}
|
||||
|
||||
WholeScheduleCost spatialCost(ModuleOp candidate, llvm::json::Object &summary) {
|
||||
WholeScheduleCost cost;
|
||||
auto begin = Clock::now();
|
||||
llvm::json::Object estimate;
|
||||
if (target.estimateRealizedScheduleCost)
|
||||
cost = target.estimateRealizedScheduleCost(candidate, estimate);
|
||||
else cost.reason = target.wholeScheduleCostUnavailable.empty()
|
||||
? "target has no realized-Spatial cost query" : target.wholeScheduleCostUnavailable;
|
||||
cost.selectionMargin = kPipelineSelectionMargin;
|
||||
estimate["safe_for_selection"] = cost.eligible();
|
||||
estimate["diagnostic_only"] = false;
|
||||
summary["spatial_cost"] = std::move(estimate);
|
||||
summary["spatial_cost_seconds"] = elapsed(begin);
|
||||
return cost;
|
||||
}
|
||||
|
||||
LogicalResult evaluateSpatialCandidates(ModuleOp checkpoint,
|
||||
const PreparedSpatialSchedules &prepared, std::vector<WholeScheduleCost> &costs,
|
||||
llvm::json::Object &report) {
|
||||
llvm::json::Array summaries;
|
||||
double realization = 0, extraction = 0, expansion = 0, queue = 0;
|
||||
std::string errors;
|
||||
for (size_t i = 0; i < costs.size(); ++i) {
|
||||
llvm::json::Object summary{{"index", static_cast<int64_t>(i)}};
|
||||
OwningOpRef<ModuleOp> candidate;
|
||||
std::string scheduleReport;
|
||||
{
|
||||
ScopedCompilerOutputSuppression silence;
|
||||
if (failed(realizeCandidate(checkpoint, prepared.candidates[i], candidate, scheduleReport, summary)))
|
||||
costs[i].reason = "Spatial schedule realization failed verification";
|
||||
else costs[i] = spatialCost(*candidate, summary);
|
||||
}
|
||||
realization += summary.getNumber("realization_seconds").value_or(0);
|
||||
extraction += costs[i].timing.extractionSeconds;
|
||||
expansion += costs[i].timing.expansionSeconds;
|
||||
queue += costs[i].timing.evaluationSeconds;
|
||||
bool supported = costs[i].loweringSupport == StaticLoweringSupport::Supported;
|
||||
auto convergence = costs[i].convergenceProved();
|
||||
auto support = costs[i].loweringSupported();
|
||||
bool converged = convergence == true;
|
||||
bool safe = costs[i].eligible();
|
||||
summary["cost_available"] = costs[i].available();
|
||||
summary["convergence_proved"] = convergence
|
||||
? llvm::json::Value(*convergence) : llvm::json::Value(nullptr);
|
||||
summary["lowering_supported"] = support
|
||||
? llvm::json::Value(*support) : llvm::json::Value(nullptr);
|
||||
summary["eligible"] = safe;
|
||||
if (costs[i].primary) summary["interval_ns"] = *costs[i].primary;
|
||||
if (costs[i].alternative) summary["alternative_tie_interval_ns"] = *costs[i].alternative;
|
||||
if (!safe) {
|
||||
std::string property;
|
||||
if (!costs[i].available()) property += "unavailable timing; ";
|
||||
if (!supported) property += "unsupported or unknown static lowering; ";
|
||||
if (!converged) property += "unproved modeled-state convergence; ";
|
||||
if (property.empty()) property = "unavailable selection margin";
|
||||
summary["reason"] = property + ": " + costs[i].reason;
|
||||
errors += "\ncandidate " + std::to_string(i) + ": " + property + ": " + costs[i].reason;
|
||||
}
|
||||
summaries.push_back(std::move(summary));
|
||||
}
|
||||
report["candidates"] = std::move(summaries);
|
||||
report["candidate_realization_seconds"] = realization;
|
||||
report["event_extraction_seconds"] = extraction;
|
||||
report["event_expansion_seconds"] = expansion;
|
||||
report["queue_seconds"] = queue;
|
||||
if (!errors.empty()) {
|
||||
report["status"] = "incomplete-costs";
|
||||
report["reason"] = errors;
|
||||
checkpoint.emitError("Spatial pipeline cannot compare the complete candidate set:") << errors;
|
||||
return failure();
|
||||
}
|
||||
return success();
|
||||
}
|
||||
|
||||
void runOnOperation() override {
|
||||
ModuleOp module = getOperation();
|
||||
ModuleOp checkpoint = getOperation();
|
||||
auto start = Clock::now();
|
||||
std::string unsupported;
|
||||
if (!target.prepareScheduleCost)
|
||||
unsupported = target.wholeScheduleCostUnavailable.empty()
|
||||
? "target has no whole-schedule cost query" : target.wholeScheduleCostUnavailable;
|
||||
auto prepared = prepareSpatialSchedules(module, target, stages, unsupported.empty());
|
||||
auto prepared = prepareSpatialSchedules(checkpoint, target, stages, /*collectCandidates=*/true);
|
||||
if (failed(prepared)) { signalPassFailure(); return; }
|
||||
bool singleCandidate = unsupported.empty() && prepared->candidates.size() == 1;
|
||||
double generationSeconds = elapsed(start);
|
||||
std::vector<WholeScheduleCost> scores(prepared->candidates.size());
|
||||
llvm::json::Array summaries;
|
||||
OwningOpRef<ModuleOp> bestModule;
|
||||
std::string bestScheduleReport;
|
||||
size_t bestIndex = 0;
|
||||
double loweringSeconds = 0, instructionSeconds = 0, queueSeconds = 0;
|
||||
size_t lowered = 0, analyzed = 0, unavailable = 0, duplicates = 0;
|
||||
for (size_t i = 0; i < prepared->candidates.size(); ++i) {
|
||||
llvm::json::Object summary{{"index", static_cast<int64_t>(i)}};
|
||||
if (!unsupported.empty()) {
|
||||
scores[i].reason = unsupported;
|
||||
} else if (singleCandidate) {
|
||||
scores[i].reason = "one distinct generated schedule; no cost comparison required";
|
||||
} else {
|
||||
ScopedCompilerOutputSuppression silence;
|
||||
OwningOpRef<ModuleOp> candidate;
|
||||
std::string scheduleReport;
|
||||
if (failed(evaluate(module, prepared->candidates[i], candidate, scheduleReport, summary))) {
|
||||
if (!i) { signalPassFailure(); return; }
|
||||
scores[i].reason = "candidate lowering failed verification";
|
||||
} else {
|
||||
++lowered;
|
||||
loweringSeconds += *summary.getNumber("lowering_seconds");
|
||||
auto query = target.prepareScheduleCost(*candidate, summary);
|
||||
if (failed(query)) {
|
||||
if (!i) { signalPassFailure(); return; }
|
||||
scores[i].reason = "candidate cost preparation failed verification";
|
||||
++unavailable;
|
||||
summary["eligible"] = false;
|
||||
summary["reason"] = scores[i].reason;
|
||||
summaries.push_back(std::move(summary));
|
||||
continue;
|
||||
}
|
||||
instructionSeconds += summary.getNumber("instruction_generation_seconds").value_or(0);
|
||||
scores[i].identity = query->identity;
|
||||
auto duplicate = std::find_if(scores.begin(), scores.begin() + i,
|
||||
[&](const WholeScheduleCost &other) { return other.identity == scores[i].identity; });
|
||||
if (duplicate != scores.begin() + i) {
|
||||
++duplicates;
|
||||
scores[i].reason = "duplicate emitted instruction program";
|
||||
summary["duplicate_of"] = static_cast<int64_t>(duplicate - scores.begin());
|
||||
} else {
|
||||
auto begin = Clock::now();
|
||||
scores[i] = query->evaluate(summary);
|
||||
scores[i].identity = query->identity;
|
||||
double seconds = elapsed(begin);
|
||||
queueSeconds += seconds;
|
||||
++analyzed;
|
||||
summary["queue_seconds"] = seconds;
|
||||
}
|
||||
std::string partialReason;
|
||||
size_t partialSelection = selectPipelineCandidate(
|
||||
llvm::ArrayRef<WholeScheduleCost>(scores.data(), i + 1), partialReason);
|
||||
if (!bestModule || partialSelection == i) {
|
||||
bestIndex = i;
|
||||
bestModule = std::move(candidate);
|
||||
bestScheduleReport = std::move(scheduleReport);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!singleCandidate && !scores[i].eligible()
|
||||
&& scores[i].reason != "duplicate emitted instruction program") ++unavailable;
|
||||
summary["eligible"] = scores[i].eligible();
|
||||
summary["identity"] = scores[i].identity;
|
||||
if (scores[i].primary) summary["interval_ns"] = *scores[i].primary;
|
||||
if (scores[i].alternative) summary["alternative_tie_interval_ns"] = *scores[i].alternative;
|
||||
if (scores[i].relativeUncertainty)
|
||||
summary["relative_prediction_uncertainty"] = *scores[i].relativeUncertainty;
|
||||
if (!scores[i].reason.empty()) summary["reason"] = scores[i].reason;
|
||||
summaries.push_back(std::move(summary));
|
||||
if (prepared->candidates.empty()) {
|
||||
checkpoint.emitError("Spatial pipeline requires at least one candidate schedule");
|
||||
signalPassFailure(); return;
|
||||
}
|
||||
bool single = prepared->candidates.size() == 1;
|
||||
llvm::json::Object report{{"strategy", "spatial-queue-aware"}, {"static_analysis", true},
|
||||
{"candidate_generation_seconds", elapsed(start)},
|
||||
{"generated_candidates", static_cast<int64_t>(prepared->candidates.size())},
|
||||
{"single_candidate_analysis_skipped", single}, {"selection_margin", kPipelineSelectionMargin},
|
||||
{"prediction_error_bound_available", false}, {"temporary_disk_bytes", 0}};
|
||||
std::vector<WholeScheduleCost> costs(prepared->candidates.size());
|
||||
if (!single && failed(evaluateSpatialCandidates(checkpoint, *prepared, costs, report))) {
|
||||
writeSelectionReport(std::move(report)); signalPassFailure(); return;
|
||||
}
|
||||
if (single) report["candidates"] = llvm::json::Array{llvm::json::Object{{"index", 0}}};
|
||||
OwningOpRef<ModuleOp> winner;
|
||||
std::string scheduleReport;
|
||||
llvm::json::Array attempts;
|
||||
double realizationSeconds = 0, backendSeconds = 0;
|
||||
size_t lowered = 0;
|
||||
auto attempt = [&](size_t index) {
|
||||
llvm::json::Object summary{{"candidate", static_cast<int64_t>(index)}};
|
||||
std::string capturedReport;
|
||||
OwningOpRef<ModuleOp> candidate;
|
||||
LogicalResult realized = failure();
|
||||
{
|
||||
ScopedCompilerOutputSuppression silence;
|
||||
realized = realizeCandidate(checkpoint, prepared->candidates[index], candidate, capturedReport, summary);
|
||||
}
|
||||
PimCandidateResult result = PimCandidateResult::Failure;
|
||||
if (failed(realized)) {
|
||||
summary["reason"] = "selected Spatial schedule failed deterministic realization";
|
||||
summary["status"] = "realization-failed";
|
||||
} else {
|
||||
++lowered;
|
||||
result = lowerPimPipelineCandidate(*candidate, resources, summary);
|
||||
}
|
||||
realizationSeconds += summary.getNumber("realization_seconds").value_or(0);
|
||||
backendSeconds += summary.getNumber("backend_seconds").value_or(0);
|
||||
if (result == PimCandidateResult::Success) {
|
||||
summary["status"] = "lowered";
|
||||
winner = std::move(candidate);
|
||||
scheduleReport = std::move(capturedReport);
|
||||
} else if (result == PimCandidateResult::ResourceLimit)
|
||||
llvm::errs() << "Spatial pipeline candidate " << index
|
||||
<< " rejected by a backend resource limit; rejection recorded in pipeline_selection.json\n";
|
||||
attempts.push_back(std::move(summary));
|
||||
return result;
|
||||
};
|
||||
std::string reason;
|
||||
size_t selected = selectPipelineCandidate(scores, reason);
|
||||
if (singleCandidate) reason = scores.front().reason;
|
||||
else if (!unsupported.empty()) reason += ": " + unsupported;
|
||||
else if (!scores.front().eligible() && !scores.front().reason.empty())
|
||||
reason += ": " + scores.front().reason;
|
||||
auto finalizeStart = Clock::now();
|
||||
std::optional<size_t> selected;
|
||||
if (single) {
|
||||
if (attempt(0) == PimCandidateResult::Success) selected = 0;
|
||||
reason = selected ? "one distinct schedule; no cost comparison required"
|
||||
: "the single schedule failed backend lowering; no alternatives are available";
|
||||
} else selected = lowerSelectedPimCandidate(costs, attempt, reason);
|
||||
report["backend_attempts"] = std::move(attempts);
|
||||
report["lowered_candidates"] = static_cast<int64_t>(lowered);
|
||||
report["costed_candidates"] = single ? 0 : static_cast<int64_t>(costs.size());
|
||||
size_t attemptCount = report.getArray("backend_attempts")->size();
|
||||
report["backend_retries"] = static_cast<int64_t>(attemptCount ? attemptCount - 1 : 0);
|
||||
report["winner_realization_seconds"] = realizationSeconds;
|
||||
report["backend_lowering_seconds"] = backendSeconds;
|
||||
if (!selected) {
|
||||
report["status"] = "backend-failure";
|
||||
report["reason"] = reason;
|
||||
std::string failures;
|
||||
for (auto &entry : *report.getArray("backend_attempts")) {
|
||||
auto *failed = entry.getAsObject();
|
||||
failures += "\ncandidate " + std::to_string(*failed->getInteger("candidate"))
|
||||
+ ": " + failed->getString("reason").value_or("backend failure").str();
|
||||
}
|
||||
checkpoint.emitError("Spatial pipeline backend selection failed; ") << reason << failures;
|
||||
writeSelectionReport(std::move(report)); signalPassFailure(); return;
|
||||
}
|
||||
auto finalize = Clock::now();
|
||||
bool spatialArtifacts = exportStage != SpatialDataflowExportStage::None
|
||||
|| pimPeriodicDiagnostics || isPimIrDumpEnabled();
|
||||
// Explicit Spatial diagnostics require the pre-lowering representation.
|
||||
if (!bestModule || spatialArtifacts) {
|
||||
if (failed(realizeSpatialSchedule(module, prepared->candidates[selected], target,
|
||||
stages, exportStage, pimPeriodicDiagnostics ? &prepared->graph : nullptr))) { signalPassFailure(); return; }
|
||||
|| pimPeriodicDiagnostics || isPimIrDumpEnabled();
|
||||
if (spatialArtifacts) {
|
||||
if (failed(realizeSpatialSchedule(checkpoint, prepared->candidates[*selected], target,
|
||||
stages, exportStage, pimPeriodicDiagnostics ? &prepared->graph : nullptr))) {
|
||||
signalPassFailure(); return;
|
||||
}
|
||||
} else {
|
||||
auto file = openDialectDumpFileWithExtension("spatial3_scheduled_no_comm", "/reports", "txt");
|
||||
if (file) file << bestScheduleReport;
|
||||
if (file) file << scheduleReport;
|
||||
}
|
||||
if (bestModule) {
|
||||
assert(selected == bestIndex && "retained module must be the selected candidate");
|
||||
module->setAttrs((*bestModule)->getAttrs());
|
||||
module.getBodyRegion().takeBody(bestModule->getBodyRegion());
|
||||
} else if (failed(lower(module))) { signalPassFailure(); return; }
|
||||
llvm::errs() << "Queue-aware pipeline: candidate " << selected << " of " << scores.size()
|
||||
checkpoint->setAttrs((*winner)->getAttrs());
|
||||
checkpoint.getBodyRegion().takeBody(winner->getBodyRegion());
|
||||
report["selection"] = static_cast<int64_t>(*selected);
|
||||
report["status"] = "selected";
|
||||
report["reason"] = reason;
|
||||
report["finalization_seconds"] = elapsed(finalize);
|
||||
report["total_seconds"] = elapsed(start);
|
||||
llvm::errs() << "Spatial queue-aware pipeline: candidate " << *selected << " of " << costs.size()
|
||||
<< "; " << reason << "\n";
|
||||
llvm::json::Object report{{"selection", static_cast<int64_t>(selected)}, {"reason", reason},
|
||||
{"static_analysis", true}, {"strategy", "queue-aware"}, {"candidates", std::move(summaries)},
|
||||
{"single_candidate_analysis_skipped", singleCandidate},
|
||||
{"candidate_generation_seconds", generationSeconds}, {"candidate_lowering_seconds", loweringSeconds},
|
||||
{"instruction_generation_seconds", instructionSeconds},
|
||||
{"queue_seconds", queueSeconds}, {"finalization_seconds", elapsed(finalizeStart)}, {"total_seconds", elapsed(start)},
|
||||
{"generated_candidates", static_cast<int64_t>(scores.size())},
|
||||
{"lowered_candidates", static_cast<int64_t>(lowered)},
|
||||
{"reused_lowered_candidate", static_cast<bool>(bestModule)},
|
||||
{"reconstructed_spatial_artifacts", spatialArtifacts && static_cast<bool>(bestModule)},
|
||||
{"analyzed_candidates", static_cast<int64_t>(analyzed)},
|
||||
{"unavailable_estimates", static_cast<int64_t>(unavailable)},
|
||||
{"duplicate_programs", static_cast<int64_t>(duplicates)}, {"temporary_disk_bytes", 0}};
|
||||
auto file = openReportFileWithExtension("pipeline_selection", "json");
|
||||
if (file) { llvm::raw_os_ostream out(file); out << llvm::formatv("{0:2}\n", llvm::json::Value(std::move(report))); }
|
||||
writeSelectionReport(std::move(report));
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
PimCandidateResult onnx_mlir::lowerPimPipelineCandidate(ModuleOp module,
|
||||
const SpatialTargetResources &resources, llvm::json::Object &summary) {
|
||||
bool classified = true, anyError = false;
|
||||
std::string diagnostic;
|
||||
size_t count = 0;
|
||||
ScopedDiagnosticHandler handler(module.getContext(), [&](Diagnostic &message) {
|
||||
if (message.getSeverity() == DiagnosticSeverity::Error) {
|
||||
anyError = true;
|
||||
classified &= pim::isCandidateResourceError(message);
|
||||
if (++count <= 8) diagnostic += message.str().substr(0, 2048) + "\n";
|
||||
}
|
||||
return success();
|
||||
});
|
||||
auto start = Clock::now();
|
||||
PassManager pm(module.getContext());
|
||||
addPimLoweringPasses(pm, resources, EmitPimCodegen, /*reportProgress=*/false);
|
||||
if (enableTiming) pm.addInstrumentation(std::make_unique<CandidateTiming>(summary));
|
||||
LogicalResult result = pm.run(module);
|
||||
summary["backend_seconds"] = elapsed(start);
|
||||
if (succeeded(result)) return PimCandidateResult::Success;
|
||||
summary["reason"] = diagnostic.empty() ? "backend failed without a classified diagnostic" : diagnostic;
|
||||
bool resource = anyError && classified;
|
||||
summary["status"] = resource ? "resource-limit" : "systemic-or-unclassified-error";
|
||||
return resource ? PimCandidateResult::ResourceLimit : PimCandidateResult::Failure;
|
||||
}
|
||||
|
||||
std::optional<size_t> onnx_mlir::lowerSelectedPimCandidate(llvm::ArrayRef<WholeScheduleCost> costs,
|
||||
const std::function<PimCandidateResult(size_t)> &lower, std::string &reason) {
|
||||
if (costs.empty()) {
|
||||
reason = "Spatial pipeline requires at least one candidate schedule";
|
||||
return std::nullopt;
|
||||
}
|
||||
for (const auto &cost : costs)
|
||||
if (!cost.eligible()) {
|
||||
reason = "cannot compare the complete candidate set: unavailable cost or unproved eligibility";
|
||||
return std::nullopt;
|
||||
}
|
||||
std::vector<WholeScheduleCost> remaining(costs.begin(), costs.end());
|
||||
std::vector<size_t> indices;
|
||||
for (size_t i = 0; i < costs.size(); ++i) indices.push_back(i);
|
||||
while (!indices.empty()) {
|
||||
size_t selected = selectPipelineCandidate(remaining, reason);
|
||||
auto result = lower(indices[selected]);
|
||||
if (result == PimCandidateResult::Success) return indices[selected];
|
||||
if (result != PimCandidateResult::ResourceLimit) {
|
||||
reason = "systemic or unclassified backend failure; recovery is not justified";
|
||||
return std::nullopt;
|
||||
}
|
||||
remaining.erase(remaining.begin() + selected);
|
||||
indices.erase(indices.begin() + selected);
|
||||
}
|
||||
reason = "all candidate schedules failed backend resource constraints";
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::unique_ptr<Pass> onnx_mlir::createPimPipelineSelectionPass(
|
||||
const SchedulingTarget &target, const SpatialTargetResources &resources,
|
||||
size_t stages, SpatialDataflowExportStage exportStage) {
|
||||
|
||||
@@ -1,8 +1,21 @@
|
||||
#pragma once
|
||||
#include "mlir/Pass/Pass.h"
|
||||
#include "llvm/ADT/ArrayRef.h"
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
namespace mlir { class ModuleOp; }
|
||||
namespace llvm::json { class Object; }
|
||||
|
||||
namespace onnx_mlir {
|
||||
namespace spatial { struct SchedulingTarget; struct SpatialTargetResources; enum class SpatialDataflowExportStage; }
|
||||
namespace spatial { struct SchedulingTarget; struct SpatialTargetResources; struct WholeScheduleCost; enum class SpatialDataflowExportStage; }
|
||||
enum class PimCandidateResult { Success, ResourceLimit, Failure };
|
||||
PimCandidateResult lowerPimPipelineCandidate(mlir::ModuleOp module,
|
||||
const spatial::SpatialTargetResources &resources, llvm::json::Object &summary);
|
||||
// Callback failures distinguish owner-classified resource limits from other errors.
|
||||
std::optional<size_t> lowerSelectedPimCandidate(llvm::ArrayRef<spatial::WholeScheduleCost> costs,
|
||||
const std::function<PimCandidateResult(size_t)> &lower, std::string &reason);
|
||||
std::unique_ptr<mlir::Pass> createPimPipelineSelectionPass(
|
||||
const spatial::SchedulingTarget &target, const spatial::SpatialTargetResources &resources,
|
||||
size_t stages, spatial::SpatialDataflowExportStage exportStage);
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
#include "PimPeriodicAnalysis.hpp"
|
||||
|
||||
using namespace onnx_mlir;
|
||||
|
||||
namespace {
|
||||
const char *windowStatus(const PimQueuePolicyResult &policy) {
|
||||
if (policy.legacy.intervalNs) return "estimated";
|
||||
return policy.legacy.termination == PimQueueTermination::NotRun && policy.recurrence.proved()
|
||||
? "not-required" : "unavailable";
|
||||
}
|
||||
|
||||
const char *terminationName(PimQueueTermination termination) {
|
||||
switch (termination) {
|
||||
case PimQueueTermination::NotRun: return "not-run";
|
||||
case PimQueueTermination::IntervalConverged: return "interval-converged";
|
||||
case PimQueueTermination::RecurrenceProved: return "recurrence-proved";
|
||||
case PimQueueTermination::EventBudget: return "event-budget";
|
||||
case PimQueueTermination::GenerationLimit: return "generation-limit";
|
||||
case PimQueueTermination::HistoryLimit: return "history-limit";
|
||||
case PimQueueTermination::NonMonotonic: return "non-monotonic";
|
||||
case PimQueueTermination::NoReadyEvents: return "no-ready-events";
|
||||
}
|
||||
llvm_unreachable("invalid queue termination");
|
||||
}
|
||||
|
||||
void serializeProgress(llvm::json::Object &report, const PimQueueProgress &progress,
|
||||
bool estimated, bool details, bool intervals) {
|
||||
if (progress.processedEvents) report["processed_events"] = progress.processedEvents;
|
||||
if (estimated) {
|
||||
report["completed_generations"] = progress.completedGenerations;
|
||||
report["maximum_queue_wait_ns_including_bootstrap"] = progress.maximumQueueWaitNs;
|
||||
if (progress.maximumQueueRequest) {
|
||||
report["maximum_queue_core"] = progress.maximumQueueRequest->core;
|
||||
report["maximum_queue_pc"] = progress.maximumQueueRequest->pc;
|
||||
}
|
||||
if (details) {
|
||||
llvm::json::Array cores;
|
||||
for (auto [core, wait] : progress.perCoreQueueWaitNs)
|
||||
cores.push_back(llvm::json::Object{{"core", core}, {"queue_wait_ns", wait}});
|
||||
report["per_core"] = std::move(cores);
|
||||
}
|
||||
}
|
||||
if (progress.generationLimit) {
|
||||
report["generation_limit_core"] = progress.generationLimit->core;
|
||||
report["generation_limit_pc"] = progress.generationLimit->pc;
|
||||
report["generation_limit_kind"] = progress.generationLimit->kind;
|
||||
}
|
||||
if (intervals) {
|
||||
llvm::json::Array values;
|
||||
for (uint64_t value : progress.completedIntervalsNs) values.push_back(value);
|
||||
report["completed_interval_ns"] = std::move(values);
|
||||
}
|
||||
}
|
||||
|
||||
llvm::json::Object serializeWindow(const PimQueuePolicyResult &policy, bool details) {
|
||||
const auto &window = policy.legacy;
|
||||
llvm::json::Object result{{"status", windowStatus(policy)}};
|
||||
if (window.intervalNs) {
|
||||
result["interval_ns"] = *window.intervalNs;
|
||||
result["tie_policy"] = policy.reverseTies ? "reverse event-index order for simultaneous readiness"
|
||||
: "event-index order for simultaneous readiness";
|
||||
}
|
||||
if (!window.reason.empty()) result["reason"] = window.reason;
|
||||
serializeProgress(result, window.progress, window.intervalNs.has_value(), details,
|
||||
window.termination == PimQueueTermination::EventBudget
|
||||
|| window.termination == PimQueueTermination::NoReadyEvents);
|
||||
return result;
|
||||
}
|
||||
|
||||
llvm::json::Object serializeRecurrence(const PimQueuePolicyResult &policy) {
|
||||
const auto &evidence = policy.recurrence;
|
||||
llvm::json::Object result{{"status", evidence.proved() ? "proved" : "unproved"},
|
||||
{"termination", terminationName(policy.termination)}};
|
||||
if (policy.checkpoints) {
|
||||
result["checkpoints"] = policy.checkpoints;
|
||||
result["history_bytes"] = policy.historyBytes;
|
||||
}
|
||||
if (evidence.proved()) {
|
||||
result["period_generations"] = evidence.periodGenerations;
|
||||
result["interval_ns"] = *evidence.intervalNs;
|
||||
result["state_words"] = policy.stateWords;
|
||||
result["complete_scheduler_state"] = true;
|
||||
result["proof_processed_events"] = policy.proofProcessedEvents;
|
||||
}
|
||||
if (!evidence.reason.empty()) result["reason"] = evidence.reason;
|
||||
return result;
|
||||
}
|
||||
|
||||
void addAlternative(llvm::json::Object &report, const PimQueuePolicyResult &policy, bool legacy) {
|
||||
auto interval = legacy ? policy.legacy.intervalNs : policy.intervalNs();
|
||||
report["alternative_tie_status"] = legacy ? windowStatus(policy) : interval ? "estimated" : "unavailable";
|
||||
if (interval) report["alternative_tie_interval_ns"] = *interval;
|
||||
if ((legacy || !policy.recurrence.proved()) && !policy.legacy.reason.empty())
|
||||
report["alternative_tie_reason"] = policy.legacy.reason;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
llvm::json::Object onnx_mlir::serializePimQueueEvaluation(const PimQueueEvaluation &evaluation) {
|
||||
const auto &primary = evaluation.policies[0];
|
||||
const auto &alternative = evaluation.policies[1];
|
||||
llvm::json::Object report;
|
||||
if (evaluation.recurrenceRequired && primary.recurrence.proved()) {
|
||||
report = llvm::json::Object{{"status", "estimated"}, {"interval_ns", *primary.intervalNs()},
|
||||
{"tie_policy", primary.reverseTies ? "reverse event-index order for simultaneous readiness"
|
||||
: "event-index order for simultaneous readiness"}};
|
||||
serializeProgress(report, primary.progress, true, evaluation.details, false);
|
||||
} else report = serializeWindow(primary, evaluation.details);
|
||||
if (evaluation.recurrenceRequired) {
|
||||
auto legacy = serializeWindow(primary, evaluation.details);
|
||||
legacy["termination"] = terminationName(primary.legacy.termination);
|
||||
legacy["alternative_termination"] = terminationName(alternative.legacy.termination);
|
||||
addAlternative(legacy, alternative, true);
|
||||
report["interval_source"] = primary.recurrence.proved() ? "complete-state recurrence" : "legacy interval window";
|
||||
report["alternative_interval_source"] = alternative.recurrence.proved() ? "complete-state recurrence" : "legacy interval window";
|
||||
report["periodic_state"] = serializeRecurrence(primary);
|
||||
report["alternative_periodic_state"] = serializeRecurrence(alternative);
|
||||
report["legacy_interval_window"] = std::move(legacy);
|
||||
}
|
||||
addAlternative(report, alternative, false);
|
||||
if (auto interval = primary.intervalNs())
|
||||
report["memory_service_fraction"] = evaluation.memoryServiceNs / *interval;
|
||||
if (evaluation.measureTime) report["analysis_ms"] = evaluation.evaluationSeconds * 1000;
|
||||
report["arbitration"] = "FIFO by predicted controller arrival";
|
||||
return report;
|
||||
}
|
||||
@@ -357,6 +357,7 @@ static mlir::FailureOr<PeriodicDependencyEstimate> analyzePlan(PeriodicPlan &pla
|
||||
return analyzePeriodicDependencies(plan.events.size(), plan.edges, plan.error);
|
||||
}
|
||||
|
||||
|
||||
} // namespace
|
||||
|
||||
llvm::json::Object getPeriodicScheduleReport(const ComputeGraph &graph,
|
||||
|
||||
@@ -0,0 +1,623 @@
|
||||
#include "PimSpatialCost.hpp"
|
||||
#include "PimInstructionTiming.hpp"
|
||||
#include "PimTimingArithmetic.hpp"
|
||||
#include "PimBinaryFormat.hpp"
|
||||
#include "PimCompilerUtils.hpp"
|
||||
#include "PimCompilerOptions.hpp"
|
||||
#include "PimScheduleDiagnostic.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/DeferredProjectionAnalysis.hpp"
|
||||
#include "mlir/Dialect/Linalg/IR/Linalg.h"
|
||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||
#include "mlir/Dialect/SCF/IR/SCF.h"
|
||||
#include "mlir/IR/Matchers.h"
|
||||
#include "llvm/ADT/SmallPtrSet.h"
|
||||
#include "llvm/ADT/DenseSet.h"
|
||||
#include "llvm/Support/MathExtras.h"
|
||||
#include <chrono>
|
||||
#include <array>
|
||||
#include <map>
|
||||
#include <limits>
|
||||
|
||||
using namespace mlir;
|
||||
using namespace onnx_mlir;
|
||||
using namespace onnx_mlir::spatial;
|
||||
using pim_timing::add;
|
||||
using pim_timing::mul;
|
||||
using pim_timing::ceil;
|
||||
namespace {
|
||||
using Event = PimExecutionEvent;
|
||||
using Sequence = std::vector<Event>;
|
||||
enum class Residence { Local, Host, Constant, Zero, CapturedWeight };
|
||||
|
||||
PimPeriodicEstimate unavailableCost(llvm::StringRef reason) {
|
||||
PimPeriodicEstimate result;
|
||||
result.cost.reason = reason.str();
|
||||
result.report = llvm::json::Object{{"status", "unavailable"}, {"cost_available", false},
|
||||
{"convergence_proved", nullptr}, {"lowering_supported", nullptr},
|
||||
{"safe_for_selection", false}, {"reason", reason.str()}};
|
||||
return result;
|
||||
}
|
||||
|
||||
class Extraction {
|
||||
public:
|
||||
const PimInstructionTiming ⌖
|
||||
bool diagnostics;
|
||||
PimExecutionProgram program;
|
||||
std::string error, loweringReason;
|
||||
StaticLoweringSupport loweringSupport = StaticLoweringSupport::Supported;
|
||||
std::map<std::string, int64_t> gaps;
|
||||
int64_t visited = 0, constantLoads = 0;
|
||||
explicit Extraction(const PimInstructionTiming &target, bool diagnostics)
|
||||
: target(target), diagnostics(diagnostics), program(target.processorCount) {}
|
||||
bool build(ModuleOp module);
|
||||
private:
|
||||
llvm::SmallDenseSet<int64_t, 16> assignedCores;
|
||||
DenseMap<Value, Residence> residence;
|
||||
DenseMap<Value, CopyAccessPlan> publicationAccess;
|
||||
DenseMap<Value, SmallVector<unsigned, 2>> directOutputs;
|
||||
llvm::SmallDenseSet<Value, 4> compactStorage;
|
||||
llvm::SmallPtrSet<Operation *, 32> indexed;
|
||||
StaticIndexEnvironment environment;
|
||||
Residence storage(Value value);
|
||||
uint64_t elements(Type type) {
|
||||
auto shape = dyn_cast<ShapedType>(type);
|
||||
if (!shape || !shape.hasStaticShape()) {
|
||||
error = "abstract work requires a static shaped payload";
|
||||
return 0;
|
||||
}
|
||||
if (!shape.getElementType().isIntOrFloat() || shape.getElementTypeBitWidth() != 32) {
|
||||
error = "abstract precision projection requires 32-bit elements";
|
||||
return 0;
|
||||
}
|
||||
uint64_t result = 1;
|
||||
for (int64_t dimension : shape.getShape()) result = mul(result, dimension);
|
||||
return result;
|
||||
}
|
||||
uint64_t bytes(Type type) {
|
||||
uint64_t count = elements(type);
|
||||
auto shaped = dyn_cast<ShapedType>(type);
|
||||
if (!shaped || !shaped.getElementType().isIntOrFloat()) {
|
||||
error = "abstract transfer requires a scalar integer or floating element";
|
||||
return 0;
|
||||
}
|
||||
uint64_t nativeBytes = ceil(mul(count, shaped.getElementTypeBitWidth()), 8);
|
||||
if (nativeBytes > std::numeric_limits<int32_t>::max()) {
|
||||
loweringSupport = StaticLoweringSupport::Unknown;
|
||||
loweringReason = "native transfer size requires backend normalization and size verification";
|
||||
}
|
||||
return ceil(mul(count, target.computeBitwidth), 8);
|
||||
}
|
||||
void work(Sequence &sequence, uint64_t ns, Event::WorkClass resource = Event::LocalCopy) {
|
||||
if (!ns) return;
|
||||
if (!sequence.empty() && sequence.back().kind == Event::Work
|
||||
&& sequence.back().workClass == resource)
|
||||
sequence.back().amount = add(sequence.back().amount, ns);
|
||||
else {
|
||||
sequence.push_back({Event::Work, ns});
|
||||
sequence.back().workClass = resource;
|
||||
}
|
||||
}
|
||||
uint64_t localWork(uint64_t bytes, bool read, bool write) const {
|
||||
return getPimLocalTransferWorkNs(target, bytes, read, write);
|
||||
}
|
||||
void materialize(Value value, Sequence &sequence);
|
||||
bool accessSlice(tensor::ExtractSliceOp slice, Sequence &sequence, bool host);
|
||||
bool block(Block &body, Sequence &sequence);
|
||||
bool operation(Operation &op, Sequence &sequence);
|
||||
bool loop(scf::ForOp op, Sequence &sequence);
|
||||
bool core(Operation *op, int64_t physicalCore, unsigned lane);
|
||||
};
|
||||
|
||||
Residence Extraction::storage(Value value) {
|
||||
if (auto it = residence.find(value); it != residence.end()) return it->second;
|
||||
Attribute constant;
|
||||
if (matchPattern(value, m_Constant(&constant)))
|
||||
return isZeroBitPatternSplat(constant) ? Residence::Zero : Residence::Constant;
|
||||
if (auto slice = value.getDefiningOp<tensor::ExtractSliceOp>()) return storage(slice.getSource());
|
||||
if (auto collapse = value.getDefiningOp<tensor::CollapseShapeOp>()) return storage(collapse.getSrc());
|
||||
if (auto expand = value.getDefiningOp<tensor::ExpandShapeOp>()) return storage(expand.getSrc());
|
||||
if (auto cast = value.getDefiningOp<tensor::CastOp>()) return storage(cast.getSource());
|
||||
if (auto branch = value.getDefiningOp<scf::IfOp>()) {
|
||||
auto condition = evaluateDeferredIndex(branch.getCondition(), environment);
|
||||
if (succeeded(condition)) {
|
||||
Region ®ion = *condition ? branch.getThenRegion() : branch.getElseRegion();
|
||||
return storage(region.front().getTerminator()->getOperand(cast<OpResult>(value).getResultNumber()));
|
||||
}
|
||||
}
|
||||
if (auto arg = dyn_cast<BlockArgument>(value)) {
|
||||
if (auto loop = dyn_cast_or_null<scf::ForOp>(arg.getOwner()->getParentOp()))
|
||||
if (arg.getArgNumber()) return Residence::Local;
|
||||
return Residence::Host;
|
||||
}
|
||||
if (isa_and_nonnull<SpatVMMOp, SpatVAddOp, SpatVSubOp, SpatVMulOp,
|
||||
SpatVMaxOp, SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp,
|
||||
SpatChannelReceiveOp, SpatHostWaitLoadOp, SpatScheduledCompute,
|
||||
SpatScheduledComputeBatch, SpatConcatOp, tensor::EmptyOp,
|
||||
tensor::InsertSliceOp, tensor::PadOp, linalg::TransposeOp, scf::ForOp>(value.getDefiningOp()))
|
||||
return Residence::Local;
|
||||
error = "abstract storage residence is unavailable for "
|
||||
+ value.getDefiningOp()->getName().getStringRef().str();
|
||||
return Residence::Local; // The caller discards the estimate when error is set.
|
||||
}
|
||||
|
||||
void Extraction::materialize(Value value, Sequence &sequence) {
|
||||
auto place = storage(value);
|
||||
if (place == Residence::Host || place == Residence::Constant || place == Residence::CapturedWeight) {
|
||||
sequence.push_back({Event::Load, bytes(value.getType())});
|
||||
// Captured weights remain block arguments; only direct global zero loads fold to fills.
|
||||
if (place == Residence::Constant || place == Residence::CapturedWeight) {
|
||||
++constantLoads;
|
||||
sequence.back().constant = true;
|
||||
}
|
||||
} else if (place == Residence::Zero) {
|
||||
work(sequence, localWork(bytes(value.getType()), false, true), Event::Fill);
|
||||
}
|
||||
}
|
||||
|
||||
bool Extraction::accessSlice(tensor::ExtractSliceOp slice, Sequence &sequence, bool host) {
|
||||
auto source = computeRowMajorStridesChecked(slice.getSourceType().getShape());
|
||||
auto destination = computeRowMajorStridesChecked(slice.getStaticSizes());
|
||||
if (failed(source) || failed(destination)) { error = "abstract slice geometry unavailable"; return false; }
|
||||
SmallVector<int64_t> strides;
|
||||
for (auto [stride, scale] : llvm::zip(*source, slice.getStaticStrides())) {
|
||||
int64_t result;
|
||||
if (scale <= 0 || llvm::MulOverflow(stride, scale, result)) {
|
||||
error = "abstract slice stride unavailable"; return false;
|
||||
}
|
||||
strides.push_back(result);
|
||||
}
|
||||
auto copy = CopyAccessPlan::build(slice.getStaticSizes(), strides, *destination);
|
||||
if (failed(copy)) { error = "abstract slice access plan unavailable"; return false; }
|
||||
uint64_t size = ceil(mul(copy->getRunElements(), target.computeBitwidth), 8);
|
||||
if (host) {
|
||||
sequence.push_back({Event::Repeat, static_cast<uint64_t>(copy->getRunCount()),
|
||||
0, 0, 0, {{Event::Load, size}}});
|
||||
residence[slice.getResult()] = Residence::Local;
|
||||
compactStorage.insert(slice.getResult());
|
||||
} else if (compactStorage.contains(slice.getSource())
|
||||
&& slice.getSourceType().getRank() == slice.getType().getRank()
|
||||
&& llvm::all_of(slice.getResult().getUsers(), [](Operation *user) {
|
||||
return isa<SpatHostStoreSyncOp>(user);
|
||||
})) {
|
||||
// Publication-only slices retain the compact source's subview geometry.
|
||||
publicationAccess.insert_or_assign(slice.getResult(), *copy);
|
||||
} else if (copy->getRunCount() > 1) {
|
||||
if (diagnostics) ++gaps["local slice materialization and forwarding"];
|
||||
work(sequence, mul(copy->getRunCount(), localWork(size, true, true)));
|
||||
}
|
||||
if (compactStorage.contains(slice.getSource()) && copy->getRunCount() == 1)
|
||||
compactStorage.insert(slice.getResult());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Extraction::loop(scf::ForOp op, Sequence &sequence) {
|
||||
auto lower = evaluateDeferredIndex(op.getLowerBound(), environment);
|
||||
auto upper = evaluateDeferredIndex(op.getUpperBound(), environment);
|
||||
auto step = evaluateDeferredIndex(op.getStep(), environment);
|
||||
if (failed(lower) || failed(upper) || failed(step) || *step <= 0) {
|
||||
error = "abstract loop has unavailable static bounds"; return false;
|
||||
}
|
||||
for (Value initial : op.getInitArgs()) materialize(initial, sequence);
|
||||
int64_t distance = 0;
|
||||
if (*upper > *lower && llvm::SubOverflow(*upper, *lower, distance)) {
|
||||
error = "unrepresentable abstract loop trip count"; return false;
|
||||
}
|
||||
uint64_t count = ceil(distance, *step);
|
||||
bool indexSensitive = indexed.contains(op);
|
||||
if (indexSensitive && count > 100000) {
|
||||
error = "abstract indexed loop exceeds analysis budget"; return false;
|
||||
}
|
||||
if (!count) return true;
|
||||
if (indexSensitive) {
|
||||
for (uint64_t i = 0; i < count; ++i) {
|
||||
int64_t induction;
|
||||
if (llvm::AddOverflow(*lower, static_cast<int64_t>(mul(i, *step)), induction)) {
|
||||
error = "unrepresentable abstract induction value"; return false;
|
||||
}
|
||||
environment.bindings[op.getInductionVar()] = induction;
|
||||
if (!block(*op.getBody(), sequence)) return false;
|
||||
}
|
||||
} else {
|
||||
environment.bindings[op.getInductionVar()] = *lower;
|
||||
Event repeat{Event::Repeat, count};
|
||||
if (!block(*op.getBody(), repeat.body)) return false;
|
||||
sequence.push_back(std::move(repeat));
|
||||
}
|
||||
environment.bindings.erase(op.getInductionVar());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Extraction::operation(Operation &op, Sequence &sequence) {
|
||||
if (auto loopOp = dyn_cast<scf::ForOp>(op)) return loop(loopOp, sequence);
|
||||
if (auto branch = dyn_cast<scf::IfOp>(op)) {
|
||||
auto condition = evaluateDeferredIndex(branch.getCondition(), environment);
|
||||
if (failed(condition)) { error = "abstract branch condition is unavailable"; return false; }
|
||||
Region ®ion = *condition ? branch.getThenRegion() : branch.getElseRegion();
|
||||
return region.empty() || block(region.front(), sequence);
|
||||
}
|
||||
auto index = [&](Value value) -> int64_t {
|
||||
auto result = evaluateDeferredIndex(value, environment);
|
||||
if (failed(result)) { error = "abstract communication index is unavailable"; return -1; }
|
||||
return *result;
|
||||
};
|
||||
if (auto sync = dyn_cast<SpatSyncOp>(op)) {
|
||||
sequence.push_back({Event::Signal, 0, index(sync.getTargetCoreId()), index(sync.getEventRegister())});
|
||||
} else if (auto wait = dyn_cast<SpatWaitOp>(op)) {
|
||||
int64_t count = index(wait.getWaitValue());
|
||||
if (count) sequence.push_back({Event::Wait, 0, 0, index(wait.getEventRegister()), count});
|
||||
} else if (auto send = dyn_cast<SpatChannelSendOp>(op)) {
|
||||
materialize(send.getInput(), sequence);
|
||||
sequence.push_back({Event::Send, bytes(send.getInput().getType()), index(send.getTargetCoreId())});
|
||||
} else if (auto recv = dyn_cast<SpatChannelReceiveOp>(op)) {
|
||||
sequence.push_back({Event::Receive, bytes(recv.getOutput().getType()), index(recv.getSourceCoreId())});
|
||||
} else if (auto store = dyn_cast<SpatHostStoreSyncOp>(op)) {
|
||||
materialize(store.getInput(), sequence);
|
||||
if (auto plan = publicationAccess.find(store.getInput()); plan != publicationAccess.end())
|
||||
sequence.push_back({Event::Repeat, static_cast<uint64_t>(plan->second.getRunCount()),
|
||||
0, 0, 0, {{Event::Store, ceil(mul(plan->second.getRunElements(), target.computeBitwidth), 8)}}});
|
||||
else sequence.push_back({Event::Store, bytes(store.getInput().getType())});
|
||||
sequence.push_back({Event::Signal, 0, index(store.getTargetCoreId()), index(store.getEventRegister())});
|
||||
} else if (auto load = dyn_cast<SpatHostWaitLoadOp>(op)) {
|
||||
int64_t count = index(load.getWaitValue());
|
||||
if (count) sequence.push_back({Event::Wait, 0, 0, index(load.getEventRegister()), count});
|
||||
sequence.push_back({Event::Load, bytes(load.getOutput().getType())});
|
||||
} else if (auto vmm = dyn_cast<SpatVMMOp>(op)) {
|
||||
materialize(vmm.getInput(), sequence);
|
||||
auto weight = dyn_cast<RankedTensorType>(vmm.getWeight().getType());
|
||||
if (!weight || weight.getRank() != 2 || !weight.hasStaticShape() || !target.matrixColumns
|
||||
|| weight.getDimSize(1) <= 0
|
||||
|| static_cast<uint64_t>(weight.getDimSize(1)) % target.matrixColumns
|
||||
|| weight.getDimSize(0) <= 0
|
||||
|| static_cast<uint64_t>(weight.getDimSize(0)) > target.matrixRows
|
||||
|| weight.getEncoding() || !weight.getElementType().isIntOrFloat()
|
||||
|| weight.getElementTypeBitWidth() != 32) {
|
||||
error = "unavailable physical matrix weight geometry"; return false;
|
||||
}
|
||||
if (diagnostics && static_cast<uint64_t>(weight.getDimSize(0)) < target.matrixRows)
|
||||
++gaps["matrix input padding and copy forwarding"];
|
||||
uint64_t output = getPimMatrixOutputBytes(target, target.matrixCellBits,
|
||||
pim_binary::kMatrixWeightPrecisionBits, weight.getDimSize(1) / target.matrixColumns,
|
||||
ceil(target.computeBitwidth, 8));
|
||||
work(sequence, getPimMatrixWorkNs(target, target.computeBitwidth, output), Event::Matrix);
|
||||
} else if (isa<SpatVAddOp, SpatVSubOp, SpatVMulOp, SpatVMaxOp,
|
||||
SpatVVDMulOp, SpatVAvgOp, SpatReluOp, SpatSigmoidOp>(op)) {
|
||||
for (Value operand : op.getOperands()) materialize(operand, sequence);
|
||||
uint64_t width = ceil(target.computeBitwidth, 8);
|
||||
work(sequence, getPimVectorWorkNs(target, elements(op.getOperand(0).getType()),
|
||||
width, width, op.getNumOperands() == 2), Event::Vector);
|
||||
} else if (auto extract = dyn_cast<tensor::ExtractSliceOp>(op)) {
|
||||
auto place = storage(extract.getSource());
|
||||
if (place == Residence::Host || place == Residence::Local)
|
||||
if (!accessSlice(extract, sequence, place == Residence::Host)) return false;
|
||||
} else if (auto insert = dyn_cast<tensor::InsertSliceOp>(op)) {
|
||||
materialize(insert.getSource(), sequence);
|
||||
uint64_t size = bytes(insert.getSource().getType());
|
||||
work(sequence, localWork(size, true, true));
|
||||
if (compactStorage.contains(insert.getDest())) compactStorage.insert(insert.getResult());
|
||||
if (diagnostics) ++gaps["output copy forwarding"];
|
||||
} else if (auto transpose = dyn_cast<linalg::TransposeOp>(op)) {
|
||||
materialize(transpose.getInput(), sequence);
|
||||
auto type = cast<ShapedType>(transpose.getInput().getType());
|
||||
uint64_t volume = elements(type);
|
||||
if (!volume) {
|
||||
if (error.empty()) error = "zero-element transpose has no abstract timing contract";
|
||||
return false;
|
||||
}
|
||||
if (failed(computeRowMajorStridesChecked(type.getShape()))) {
|
||||
error = "abstract transpose geometry unavailable"; return false;
|
||||
}
|
||||
auto run = getTransposeRun(type.getShape(), transpose.getPermutation());
|
||||
uint64_t width = ceil(target.computeBitwidth, 8);
|
||||
uint64_t runBytes = mul(run.elements, width);
|
||||
uint64_t cost = run.sourceStride == 1
|
||||
? localWork(runBytes, true, true)
|
||||
: getPimVectorWorkNs(target, run.elements, width, width, false);
|
||||
work(sequence, mul(volume / run.elements, cost),
|
||||
run.sourceStride == 1 ? Event::LocalCopy : Event::VectorCopy);
|
||||
if (diagnostics) ++gaps["transpose alias and constant folding"];
|
||||
} else if (auto pad = dyn_cast<tensor::PadOp>(op)) {
|
||||
materialize(pad.getSource(), sequence);
|
||||
work(sequence, localWork(bytes(pad.getResult().getType()), false, true), Event::Fill);
|
||||
work(sequence, localWork(bytes(pad.getSource().getType()), true, true));
|
||||
if (diagnostics) ++gaps["padding fill and copy normalization"];
|
||||
} else if (auto concat = dyn_cast<SpatConcatOp>(op)) {
|
||||
for (Value input : concat.getInputs()) {
|
||||
materialize(input, sequence);
|
||||
uint64_t size = bytes(input.getType());
|
||||
work(sequence, localWork(size, true, true));
|
||||
}
|
||||
compactStorage.insert(concat.getOutput());
|
||||
if (diagnostics) ++gaps["concat alias and copy elimination"];
|
||||
} else if (auto empty = dyn_cast<tensor::EmptyOp>(op)) {
|
||||
compactStorage.insert(empty.getResult());
|
||||
} else if (isa<arith::ConstantOp, tensor::ExtractOp,
|
||||
tensor::CollapseShapeOp, tensor::ExpandShapeOp, tensor::CastOp,
|
||||
scf::YieldOp, SpatYieldOp>(op) || isPureIndexComputationOp(&op)) {
|
||||
// Compile-time shape/index work has no device execution cost.
|
||||
} else if (auto parallel = dyn_cast<SpatInParallelOp>(op)) {
|
||||
auto batch = parallel->getParentOfType<SpatScheduledComputeBatch>();
|
||||
auto first = batch ? batch.getOutputArgument(0) : std::nullopt;
|
||||
if (!first) { error = "unavailable abstract batch output mapping"; return false; }
|
||||
for (Operation &nested : parallel.getRegion().front()) {
|
||||
auto insert = dyn_cast<tensor::ParallelInsertSliceOp>(nested);
|
||||
auto destination = insert ? dyn_cast<BlockArgument>(insert.getDest()) : BlockArgument{};
|
||||
if (!destination || destination.getArgNumber() < first->getArgNumber()) {
|
||||
error = "unsupported abstract batch output"; return false;
|
||||
}
|
||||
unsigned result = destination.getArgNumber() - first->getArgNumber();
|
||||
if (result >= batch.getNumResults()) { error = "invalid abstract batch output"; return false; }
|
||||
if (succeeded(getDirectReturnOperandIndex(cast<OpResult>(batch.getResult(result))))) {
|
||||
materialize(insert.getSource(), sequence);
|
||||
sequence.push_back({Event::Store, bytes(insert.getSource().getType())});
|
||||
} else if (!batch.getResult(result).use_empty()) {
|
||||
error = "unavailable indirect batch output collection"; return false;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
error = "unsupported abstract operation: " + op.getName().getStringRef().str();
|
||||
return false;
|
||||
}
|
||||
return error.empty();
|
||||
}
|
||||
|
||||
bool Extraction::block(Block &body, Sequence &sequence) {
|
||||
for (Operation &op : body) {
|
||||
if (++visited > 1000000) { error = "abstract extraction budget exceeded"; return false; }
|
||||
if (!operation(op, sequence)) return false;
|
||||
for (Value result : op.getResults())
|
||||
if (auto outputs = directOutputs.find(result); outputs != directOutputs.end())
|
||||
// Lowering inserts each store immediately after the shared producer.
|
||||
for (unsigned destination : llvm::reverse(outputs->second)) {
|
||||
(void)destination; // Distinct host destinations require distinct stores.
|
||||
sequence.push_back({Event::Store, bytes(result.getType())});
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Extraction::core(Operation *op, int64_t physicalCore, unsigned lane) {
|
||||
if (physicalCore < 0 || physicalCore >= static_cast<int64_t>(program.size())) {
|
||||
loweringSupport = StaticLoweringSupport::Unsupported;
|
||||
error = "physical core ID is outside the target: " + std::to_string(physicalCore); return false;
|
||||
}
|
||||
if (!assignedCores.insert(physicalCore).second) {
|
||||
loweringSupport = StaticLoweringSupport::Unsupported;
|
||||
error = "duplicate physical core assignment: " + std::to_string(physicalCore); return false;
|
||||
}
|
||||
residence.clear(); publicationAccess.clear(); directOutputs.clear(); compactStorage.clear();
|
||||
environment.bindings.clear();
|
||||
ValueRange inputs, weights;
|
||||
auto single = dyn_cast<SpatScheduledCompute>(op);
|
||||
auto batch = dyn_cast<SpatScheduledComputeBatch>(op);
|
||||
if (single) { inputs = single.getInputs(); weights = single.getWeights(); }
|
||||
else { inputs = batch.getInputs(); weights = batch.getWeights(); environment.bindings[*batch.getLaneArgument()] = lane; }
|
||||
Sequence &sequence = program[physicalCore];
|
||||
if (single) {
|
||||
auto yield = cast<SpatYieldOp>(single.getBody().front().getTerminator());
|
||||
for (auto [result, value] : llvm::zip(single.getResults(), yield.getOperands()))
|
||||
if (auto destination = getDirectReturnOperandIndex(cast<OpResult>(result)); succeeded(destination)) {
|
||||
if (!value.getDefiningOp() || value.getDefiningOp()->getBlock() != &single.getBody().front()) {
|
||||
error = "unavailable direct output insertion position"; return false;
|
||||
}
|
||||
directOutputs[value].push_back(*destination);
|
||||
} else if (!result.use_empty()) {
|
||||
error = "unavailable indirect compute output collection"; return false;
|
||||
}
|
||||
}
|
||||
for (unsigned i = 0; i < weights.size(); ++i)
|
||||
residence[single ? *single.getWeightArgument(i) : *batch.getWeightArgument(i)] = Residence::CapturedWeight;
|
||||
for (unsigned i = 0; i < inputs.size(); ++i) {
|
||||
Value argument = single ? *single.getInputArgument(i) : *batch.getInputArgument(i);
|
||||
auto plan = planComputeInputMaterialization(op, i);
|
||||
if (failed(plan)) { error = "input materialization plan unavailable"; return false; }
|
||||
residence[argument] = storage(inputs[i]);
|
||||
// FuncOpArgToGlobalMemoryPattern removes this input and exposes the host
|
||||
// tensor in the body before entry-copy materialization runs.
|
||||
if (*plan == ComputeInputMaterialization::ForwardFunctionInput) {
|
||||
if (diagnostics) ++gaps["function-input whole-load versus slice materialization"];
|
||||
continue;
|
||||
}
|
||||
if (*plan == ComputeInputMaterialization::CopyAtEntry) {
|
||||
materialize(inputs[i], sequence);
|
||||
residence[argument] = Residence::Local;
|
||||
compactStorage.insert(argument);
|
||||
if (diagnostics) ++gaps["input host slicing and receive sinking"];
|
||||
}
|
||||
}
|
||||
return block(op->getRegion(0).front(), sequence);
|
||||
}
|
||||
|
||||
bool Extraction::build(ModuleOp module) {
|
||||
bool okay = true;
|
||||
module.walk<WalkOrder::PostOrder>([&](Operation *op) {
|
||||
if (indexed.contains(op) || isa<SpatChannelSendOp, SpatChannelReceiveOp,
|
||||
SpatSyncOp, SpatWaitOp, SpatHostStoreSyncOp, SpatHostWaitLoadOp, scf::IfOp>(op))
|
||||
if (Operation *parent = op->getParentOp()) indexed.insert(parent);
|
||||
});
|
||||
module.walk([&](Operation *op) {
|
||||
if (!okay) return WalkResult::interrupt();
|
||||
if (auto single = dyn_cast<SpatScheduledCompute>(op)) {
|
||||
auto coreId = op->getAttrOfType<IntegerAttr>("coreId");
|
||||
if (!coreId) { error = "missing physical core"; okay = false; }
|
||||
else okay = core(op, coreId.getInt(), 0);
|
||||
return WalkResult::skip();
|
||||
}
|
||||
if (auto batch = dyn_cast<SpatScheduledComputeBatch>(op)) {
|
||||
auto ids = op->getAttrOfType<DenseI32ArrayAttr>("coreIds");
|
||||
if (!ids || ids.size() != batch.getLaneCount()) { error = "missing physical batch cores"; okay = false; }
|
||||
else for (auto [lane, id] : llvm::enumerate(ids.asArrayRef())) if (!(okay = core(op, id, lane))) break;
|
||||
return WalkResult::skip();
|
||||
}
|
||||
return WalkResult::advance();
|
||||
});
|
||||
return okay && error.empty();
|
||||
}
|
||||
} // namespace
|
||||
|
||||
PimPeriodicEstimate onnx_mlir::evaluatePimSpatialSchedule(ModuleOp module,
|
||||
const PimInstructionTiming &target, PimPeriodicOptions options) try {
|
||||
if (!target.hasPeriodicTransferTiming() || !target.computeBitwidth)
|
||||
return unavailableCost("unsupported abstract timing contract");
|
||||
auto start = std::chrono::steady_clock::now();
|
||||
Extraction extraction(target, options.workBreakdown);
|
||||
bool available = extraction.build(module);
|
||||
double extractionMs = std::chrono::duration<double, std::milli>(
|
||||
std::chrono::steady_clock::now() - start).count();
|
||||
PimPeriodicEstimate estimate;
|
||||
if (available) estimate = evaluatePimExecution(extraction.program, target, options);
|
||||
else {
|
||||
estimate.cost.reason = extraction.error;
|
||||
estimate.report = llvm::json::Object{{"status", "unavailable"}, {"reason", extraction.error}};
|
||||
if (options.verifyPeriodicState)
|
||||
for (auto &recurrence : estimate.cost.recurrence)
|
||||
recurrence.status = ScheduleRecurrenceEvidence::Unproved;
|
||||
}
|
||||
auto &report = estimate.report;
|
||||
auto &cost = estimate.cost;
|
||||
cost.timing.extractionSeconds = extractionMs / 1000;
|
||||
if (available || extraction.loweringSupport == StaticLoweringSupport::Unsupported)
|
||||
cost.loweringSupport = extraction.loweringSupport;
|
||||
if (available && options.workBreakdown) {
|
||||
PimPeriodicOptions sensitivity;
|
||||
sensitivity.ignoreLocalCopies = true;
|
||||
auto withoutCopies = analyzePimExecution(extraction.program, target, sensitivity);
|
||||
if (auto *queue = withoutCopies.getObject("memory_controller"))
|
||||
report["without_local_copy_work"] = std::move(*queue);
|
||||
}
|
||||
auto convergence = cost.convergenceProved();
|
||||
auto support = cost.loweringSupported();
|
||||
report["cost_available"] = cost.primary.has_value() && cost.alternative.has_value();
|
||||
report["convergence_proved"] = convergence
|
||||
? llvm::json::Value(*convergence) : llvm::json::Value(nullptr);
|
||||
report["lowering_supported"] = support
|
||||
? llvm::json::Value(*support) : llvm::json::Value(nullptr);
|
||||
std::string failureReason = std::move(cost.reason);
|
||||
cost.reason = "Spatial prediction accuracy has no formal bound";
|
||||
if (!cost.available()) cost.reason = "Spatial queue cost is unavailable";
|
||||
else if (cost.loweringSupport != StaticLoweringSupport::Supported)
|
||||
cost.reason = "Spatial lowering support is unproved";
|
||||
else if (convergence != true)
|
||||
cost.reason = "complete-state recurrence is unproved under one or both arbitration policies";
|
||||
if (!extraction.loweringReason.empty()) cost.reason = extraction.loweringReason;
|
||||
if (!failureReason.empty()) cost.reason = std::move(failureReason);
|
||||
report["lowering_support_scope"] = "static operation and transfer contract; final backend verification is required";
|
||||
if (!extraction.loweringReason.empty()) report["lowering_reason"] = extraction.loweringReason;
|
||||
report["safe_for_selection"] = false;
|
||||
report["diagnostic_only"] = true;
|
||||
report["error_bound_available"] = false;
|
||||
report["extraction_ms"] = extractionMs;
|
||||
report["visited_operations"] = extraction.visited;
|
||||
if (!options.workBreakdown) return estimate;
|
||||
report["constant_load_sites"] = extraction.constantLoads;
|
||||
uint64_t workNs = 0, records = 0, constantRequests = 0, constantBytes = 0;
|
||||
constexpr const char *workClassNames[] = {
|
||||
"matrix", "vector", "local_copy", "fill", "vector_copy", "scalar_issue"};
|
||||
std::array<uint64_t, std::size(workClassNames)> workByClass{};
|
||||
static_assert(Event::ScalarIssue + 1 == std::size(workClassNames));
|
||||
std::array<uint64_t, 7> counts{};
|
||||
std::map<uint64_t, uint64_t> readSizes;
|
||||
std::function<void(const Sequence &, uint64_t)> summarize =
|
||||
[&](const Sequence &sequence, uint64_t multiplier) {
|
||||
for (const Event &event : sequence) {
|
||||
++records;
|
||||
if (event.kind == Event::Repeat) summarize(event.body, mul(multiplier, event.amount));
|
||||
else {
|
||||
counts[event.kind] = add(counts[event.kind], multiplier);
|
||||
if (event.kind == Event::Work) {
|
||||
workNs = add(workNs, mul(multiplier, event.amount));
|
||||
switch (event.workClass) {
|
||||
case Event::Matrix:
|
||||
case Event::Vector:
|
||||
case Event::LocalCopy:
|
||||
case Event::Fill:
|
||||
case Event::VectorCopy:
|
||||
case Event::ScalarIssue:
|
||||
workByClass[event.workClass] = add(workByClass[event.workClass], mul(multiplier, event.amount));
|
||||
break;
|
||||
default: throw pim_timing::Unrepresentable{"unsupported execution work class"};
|
||||
}
|
||||
} else if (event.kind == Event::Load) {
|
||||
readSizes[event.amount] = add(readSizes[event.amount], multiplier);
|
||||
if (event.constant) {
|
||||
constantRequests = add(constantRequests, multiplier);
|
||||
constantBytes = add(constantBytes, mul(multiplier, event.amount));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
for (const Sequence &sequence : extraction.program) summarize(sequence, 1);
|
||||
report["compute_and_local_work_ns"] = workNs;
|
||||
llvm::json::Array classes;
|
||||
for (uint64_t value : workByClass) classes.push_back(value);
|
||||
llvm::json::Array columns;
|
||||
for (const char *name : workClassNames) columns.push_back(name);
|
||||
report["work_class_columns"] = std::move(columns);
|
||||
report["work_class_ns"] = std::move(classes);
|
||||
llvm::json::Object eventCounts;
|
||||
const char *names[] = {"work_chunks", "load", "store", "send", "receive", "sync", "wait"};
|
||||
for (size_t i = 0; i < counts.size(); ++i) eventCounts[names[i]] = counts[i];
|
||||
report["event_counts"] = std::move(eventCounts);
|
||||
report["compact_records"] = records;
|
||||
report["constant_read_requests"] = constantRequests;
|
||||
report["constant_read_bytes"] = constantBytes;
|
||||
llvm::json::Array sizes;
|
||||
for (auto [bytes, count] : readSizes)
|
||||
sizes.push_back(llvm::json::Object{{"bytes", bytes}, {"requests", count}});
|
||||
report["read_sizes"] = std::move(sizes);
|
||||
llvm::json::Object gaps;
|
||||
for (const auto &[name, count] : extraction.gaps) gaps[name] = count;
|
||||
report["unresolved_decisions"] = std::move(gaps);
|
||||
report["omitted_effects"] = llvm::json::Array{"scalar register setup", "indirect output collection",
|
||||
"host slice splitting and folding", "copy forwarding and local reuse legality"};
|
||||
report["analysis_ms"] = std::chrono::duration<double, std::milli>(
|
||||
std::chrono::steady_clock::now() - start).count();
|
||||
return estimate;
|
||||
} catch (const pim_timing::Unrepresentable &failure) {
|
||||
return unavailableCost(failure.reason);
|
||||
}
|
||||
|
||||
llvm::json::Object onnx_mlir::analyzePimSpatialSchedule(ModuleOp module,
|
||||
const PimInstructionTiming &target, PimPeriodicOptions options) {
|
||||
return evaluatePimSpatialSchedule(module, target, options).report;
|
||||
}
|
||||
|
||||
void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target,
|
||||
PimInstructionTiming timing) {
|
||||
auto contract = getPimPeriodicTimingContract();
|
||||
if (contract) timing.matrixCellBits = *contract;
|
||||
auto hardware = std::make_shared<const PimInstructionTiming>(std::move(timing));
|
||||
target.reportPeriodicSchedule = [hardware](const spatial::ComputeGraph &graph,
|
||||
const spatial::ScheduledComputeMaterializationResult &materialization,
|
||||
const spatial::DeferredTransferPlan &transfers,
|
||||
const spatial::DeferredBoundaryPlan &boundaries) {
|
||||
dumpPeriodicScheduleEstimate(graph, materialization, transfers, boundaries, *hardware);
|
||||
};
|
||||
if (!contract) target.wholeScheduleCostUnavailable = llvm::toString(contract.takeError());
|
||||
if (!hardware->hasPeriodicTransferTiming())
|
||||
target.wholeScheduleCostUnavailable = "target has no supported queue timing contract";
|
||||
if (pimDisableSynchronization)
|
||||
target.wholeScheduleCostUnavailable = "synchronization ablation has no periodic queue timing contract";
|
||||
if (!target.wholeScheduleCostUnavailable.empty()) return;
|
||||
target.estimateRealizedScheduleCost = [hardware](mlir::ModuleOp module,
|
||||
llvm::json::Object &summary) {
|
||||
PimPeriodicOptions options;
|
||||
options.workBreakdown = pimPeriodicDiagnostics;
|
||||
options.measureTime = true;
|
||||
options.verifyPeriodicState = true;
|
||||
auto estimate = evaluatePimSpatialSchedule(module, *hardware, options);
|
||||
auto cost = std::move(estimate.cost);
|
||||
summary = std::move(estimate.report);
|
||||
if (estimate.queue)
|
||||
summary["interval_source"] = cost.convergenceProved() == true
|
||||
? "complete-state recurrence under both arbitration policies"
|
||||
: "recurrence is unproved under one or both arbitration policies";
|
||||
summary["cost_available"] = cost.available();
|
||||
return cost;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#pragma once
|
||||
|
||||
#include "PimPeriodicAnalysis.hpp"
|
||||
#include "mlir/IR/BuiltinOps.h"
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
void configurePimScheduleCosts(spatial::SchedulingTarget &target, PimInstructionTiming timing);
|
||||
|
||||
// Read-only, target-specific work extraction. No target dialect is constructed.
|
||||
PimPeriodicEstimate evaluatePimSpatialSchedule(mlir::ModuleOp module,
|
||||
const PimInstructionTiming &target, PimPeriodicOptions options = {});
|
||||
llvm::json::Object analyzePimSpatialSchedule(mlir::ModuleOp module,
|
||||
const PimInstructionTiming &target, PimPeriodicOptions options = {});
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
|
||||
namespace onnx_mlir::pim_timing {
|
||||
|
||||
struct Unrepresentable { const char *reason; };
|
||||
|
||||
// Optional timing queries must fail without wrapping or terminating compilation.
|
||||
struct Arithmetic {
|
||||
static constexpr uint64_t limit = std::numeric_limits<int64_t>::max();
|
||||
unsigned long long add(uint64_t a, uint64_t b) const {
|
||||
if (a > limit || b > limit - a) throw Unrepresentable{"timing addition overflow"};
|
||||
return a + b;
|
||||
}
|
||||
unsigned long long multiply(uint64_t a, uint64_t b) const {
|
||||
if (a > limit || b > limit || (a && b > limit / a))
|
||||
throw Unrepresentable{"timing multiplication overflow"};
|
||||
return a * b;
|
||||
}
|
||||
unsigned long long subtract(uint64_t a, uint64_t b) const {
|
||||
if (a > limit || b > a) throw Unrepresentable{"timing subtraction underflow"};
|
||||
return a - b;
|
||||
}
|
||||
unsigned long long divideUp(uint64_t a, uint64_t b) const {
|
||||
if (!b || a > limit || b > limit) throw Unrepresentable{"unrepresentable timing division"};
|
||||
return a / b + (a % b != 0);
|
||||
}
|
||||
};
|
||||
inline constexpr Arithmetic arithmetic{};
|
||||
inline uint64_t add(uint64_t a, uint64_t b) { return arithmetic.add(a, b); }
|
||||
inline uint64_t mul(uint64_t a, uint64_t b) { return arithmetic.multiply(a, b); }
|
||||
inline uint64_t sub(uint64_t a, uint64_t b) { return arithmetic.subtract(a, b); }
|
||||
inline uint64_t ceil(uint64_t a, uint64_t b) { return arithmetic.divideUp(a, b); }
|
||||
|
||||
} // namespace onnx_mlir::pim_timing
|
||||
@@ -15,6 +15,7 @@
|
||||
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
|
||||
|
||||
using namespace mlir;
|
||||
@@ -34,15 +35,6 @@ static bool isUsedOnlyByExtractSlices(Value value) {
|
||||
&& llvm::all_of(value.getUsers(), [](Operation* user) { return isa<tensor::ExtractSliceOp>(user); });
|
||||
}
|
||||
|
||||
static FailureOr<unsigned> getDirectReturnOperandIndex(OpResult result) {
|
||||
if (!result.hasOneUse())
|
||||
return failure();
|
||||
|
||||
auto returnOp = dyn_cast<func::ReturnOp>(*result.getUsers().begin());
|
||||
if (!returnOp)
|
||||
return failure();
|
||||
return result.getUses().begin()->getOperandNumber();
|
||||
}
|
||||
|
||||
static FailureOr<SmallVector<FragmentAssemblyCopy, 8>>
|
||||
collectFragmentAssemblyCopiesFromBlueprint(spatial::SpatBlueprintOp blueprint,
|
||||
@@ -85,7 +77,9 @@ collectFragmentAssemblyCopiesFromBlueprint(spatial::SpatBlueprintOp blueprint,
|
||||
flatStrides)))
|
||||
return failure();
|
||||
|
||||
SmallVector<int64_t> hostStrides = computeRowMajorStrides(resultType.getShape());
|
||||
auto checkedStrides = computeRowMajorStridesChecked(resultType.getShape());
|
||||
if (failed(checkedStrides)) return failure();
|
||||
SmallVector<int64_t> hostStrides = std::move(*checkedStrides);
|
||||
for (int64_t fragmentIndex = 0; fragmentIndex < static_cast<int64_t>(operandIndices.size()); ++fragmentIndex) {
|
||||
Value source = fixedSource ? fixedSource : mapper.lookupOrDefault(fragmentOperands[operandIndices[fragmentIndex]]);
|
||||
auto sourceType = dyn_cast<RankedTensorType>(source.getType());
|
||||
@@ -183,7 +177,9 @@ collectTopLevelFragmentAssemblyCopies(OpResult result, RankedTensorType packedRe
|
||||
flatSizes,
|
||||
flatStrides)))
|
||||
return failure();
|
||||
SmallVector<int64_t> hostStrides = computeRowMajorStrides(hostResultType.getShape());
|
||||
auto checkedStrides = computeRowMajorStridesChecked(hostResultType.getShape());
|
||||
if (failed(checkedStrides)) return failure();
|
||||
SmallVector<int64_t> hostStrides = std::move(*checkedStrides);
|
||||
for (int64_t fragmentIndex = 0; fragmentIndex < static_cast<int64_t>(operandIndices.size()); ++fragmentIndex) {
|
||||
if (operandIndices[fragmentIndex] != static_cast<int64_t>(use.getOperandNumber()))
|
||||
continue;
|
||||
@@ -239,14 +235,16 @@ static Value createScaledIndexValue(IRRewriter& rewriter, Location loc, Value ba
|
||||
return arith::MulIOp::create(rewriter, loc, base, scaleValue).getResult();
|
||||
}
|
||||
|
||||
static Value createHostTargetOffset(IRRewriter& rewriter,
|
||||
static FailureOr<Value> createHostTargetOffset(IRRewriter& rewriter,
|
||||
Location loc,
|
||||
ShapedType destinationType,
|
||||
ArrayRef<OpFoldResult> mixedOffsets,
|
||||
ArrayRef<int64_t> additionalOffsets,
|
||||
IRMapping& mapper) {
|
||||
int64_t elementBytes = static_cast<int64_t>(getElementTypeSizeInBytes(destinationType.getElementType()));
|
||||
SmallVector<int64_t> strides = computeRowMajorStrides(destinationType.getShape());
|
||||
auto checkedStrides = computeRowMajorStridesChecked(destinationType.getShape());
|
||||
if (failed(checkedStrides)) return failure();
|
||||
SmallVector<int64_t> strides = std::move(*checkedStrides);
|
||||
|
||||
Value totalOffset;
|
||||
for (auto [dim, offset] : llvm::enumerate(mixedOffsets)) {
|
||||
@@ -277,7 +275,7 @@ static Value createHostTargetOffset(IRRewriter& rewriter,
|
||||
return totalOffset;
|
||||
}
|
||||
|
||||
static Value createHostTargetOffset(IRRewriter& rewriter,
|
||||
static FailureOr<Value> createHostTargetOffset(IRRewriter& rewriter,
|
||||
tensor::ParallelInsertSliceOp insertSlice,
|
||||
ShapedType destinationType,
|
||||
IRMapping& mapper) {
|
||||
@@ -380,6 +378,9 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
|
||||
mapper.map(*oldWeightArg, coreBatchOp.getWeightArgument(weightIndex));
|
||||
}
|
||||
for (unsigned inputIndex = 0; inputIndex < computeBatchOp.getInputs().size(); ++inputIndex) {
|
||||
auto inputPlan = planComputeInputMaterialization(computeBatchOp, inputIndex);
|
||||
if (failed(inputPlan) || *inputPlan != ComputeInputMaterialization::CopyAtEntry)
|
||||
return computeBatchOp.emitOpError("expected body-entry compute_batch input materialization");
|
||||
auto oldArg = computeBatchOp.getInputArgument(inputIndex);
|
||||
if (!oldArg)
|
||||
return computeBatchOp.emitOpError("expected compute_batch input block arguments before lowering");
|
||||
@@ -491,19 +492,20 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
|
||||
if (failed(fragmentAssemblyRuns))
|
||||
return failure();
|
||||
SmallVector<int64_t> zeroOffsets(hostTargetType.getRank(), 0);
|
||||
Value baseHostOffset = createHostTargetOffset(rewriter,
|
||||
auto baseHostOffset = createHostTargetOffset(rewriter,
|
||||
blueprint.getLoc(),
|
||||
hostTargetType,
|
||||
insertSlice.getMixedOffsets(),
|
||||
zeroOffsets,
|
||||
mapper);
|
||||
if (failed(baseHostOffset)) return failure();
|
||||
FailureOr<Value> updatedHostTarget = emitFragmentAssemblyCopyRuns(rewriter,
|
||||
blueprint.getLoc(),
|
||||
*fragmentAssemblyRuns,
|
||||
hostTarget,
|
||||
coreBatchOp.getOperation(),
|
||||
std::nullopt,
|
||||
baseHostOffset);
|
||||
*baseHostOffset);
|
||||
if (failed(updatedHostTarget))
|
||||
return failure();
|
||||
hostOutputTensors[resultIndex] = *updatedHostTarget;
|
||||
@@ -511,7 +513,8 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
|
||||
}
|
||||
}
|
||||
|
||||
Value hostTargetOffset = createHostTargetOffset(rewriter, insertSlice, hostTargetType, mapper);
|
||||
auto hostTargetOffset = createHostTargetOffset(rewriter, insertSlice, hostTargetType, mapper);
|
||||
if (failed(hostTargetOffset)) return failure();
|
||||
Value zeroOffset = getOrCreateIndexConstant(rewriter, coreBatchOp.getOperation(), 0);
|
||||
auto sizeAttr = getTensorSizeInBytesAttr(rewriter, coreBatchOp.getOperation(), mappedSource);
|
||||
if (failed(sizeAttr))
|
||||
@@ -519,7 +522,7 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
|
||||
pim::PimMemCopyDevToHostOp::create(rewriter,
|
||||
insertSlice.getLoc(),
|
||||
hostTarget.getType(),
|
||||
hostTargetOffset,
|
||||
*hostTargetOffset,
|
||||
zeroOffset,
|
||||
hostTarget,
|
||||
mappedSource,
|
||||
|
||||
@@ -123,7 +123,7 @@ static bool isLoopCarriedOutput(mlir::Value operand, Operation* operation) {
|
||||
.getOperand(argument.getArgNumber() - 1) == operation->getResult(0);
|
||||
}
|
||||
|
||||
mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Operation* operation) {
|
||||
mlir::Value getReusableOutputTensor(Operation* operation) {
|
||||
assert("Only support operations with a single result" && operation->getNumResults() == 1);
|
||||
mlir::Value result = operation->getResult(0);
|
||||
auto resultType = result.getType();
|
||||
@@ -141,8 +141,13 @@ mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Op
|
||||
|
||||
if (bestOperand != validOperands.end())
|
||||
return *bestOperand;
|
||||
return {};
|
||||
}
|
||||
|
||||
auto resultShapedType = cast<ShapedType>(resultType);
|
||||
mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Operation* operation) {
|
||||
if (mlir::Value operand = getReusableOutputTensor(operation))
|
||||
return operand;
|
||||
auto resultShapedType = cast<ShapedType>(operation->getResult(0).getType());
|
||||
rewriter.setInsertionPoint(operation);
|
||||
return tensor::EmptyOp::create(
|
||||
rewriter, operation->getLoc(), resultShapedType.getShape(), resultShapedType.getElementType());
|
||||
|
||||
@@ -47,6 +47,10 @@ mlir::Operation* getEarliestUserWithinBlock(mlir::Value value);
|
||||
|
||||
mlir::SmallVector<mlir::Value> getOpOperandsSortedByUses(mlir::Operation* operation);
|
||||
|
||||
// Destination selection only. Bufferization still proves physical aliasing.
|
||||
// Valid for the current SSA graph only; recompute after use or order changes.
|
||||
mlir::Value getReusableOutputTensor(mlir::Operation* operation);
|
||||
|
||||
mlir::Value getBestOutputTensorFromOperandsOrAllocate(mlir::RewriterBase& rewriter, mlir::Operation* operation);
|
||||
|
||||
mlir::LogicalResult validateFragmentAssemblyMetadata(onnx_mlir::spatial::SpatBlueprintOp blueprint,
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
#include <cassert>
|
||||
#include "mlir/Dialect/Arith/IR/Arith.h"
|
||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||
|
||||
#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
@@ -7,6 +9,41 @@ using namespace mlir;
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
FailureOr<unsigned> getDirectReturnOperandIndex(OpResult result) {
|
||||
if (!result.hasOneUse())
|
||||
return failure();
|
||||
|
||||
auto returnOp = dyn_cast<func::ReturnOp>(*result.getUsers().begin());
|
||||
if (!returnOp)
|
||||
return failure();
|
||||
return result.getUses().begin()->getOperandNumber();
|
||||
}
|
||||
|
||||
|
||||
FailureOr<ComputeInputMaterialization>
|
||||
planComputeInputMaterialization(Operation* owner, unsigned inputIndex) {
|
||||
Value input;
|
||||
bool batched = false;
|
||||
if (auto compute = dyn_cast<spatial::SpatScheduledCompute>(owner)) {
|
||||
if (inputIndex >= compute.getInputs().size() || !compute.getInputArgument(inputIndex))
|
||||
return failure();
|
||||
if (compute.getInputArgument(inputIndex)->use_empty())
|
||||
return ComputeInputMaterialization::Unused;
|
||||
input = compute.getInputs()[inputIndex];
|
||||
} else if (auto batch = dyn_cast<spatial::SpatScheduledComputeBatch>(owner)) {
|
||||
if (inputIndex >= batch.getInputs().size() || !batch.getInputArgument(inputIndex))
|
||||
return failure();
|
||||
input = batch.getInputs()[inputIndex];
|
||||
batched = true;
|
||||
} else return failure();
|
||||
if (auto argument = dyn_cast<BlockArgument>(input); argument
|
||||
&& isa<func::FuncOp>(argument.getOwner()->getParentOp()))
|
||||
return ComputeInputMaterialization::ForwardFunctionInput;
|
||||
if (!batched && input.getDefiningOp<arith::ConstantOp>())
|
||||
return ComputeInputMaterialization::ForwardConstant;
|
||||
return ComputeInputMaterialization::CopyAtEntry;
|
||||
}
|
||||
|
||||
std::optional<unsigned> getDirectComputeLikeInputIndex(Operation* owner, unsigned operandNumber) {
|
||||
auto getInputIndex = [operandNumber](Operation* op, unsigned inputCount) -> std::optional<unsigned> {
|
||||
if (inputCount == 0)
|
||||
|
||||
@@ -7,6 +7,14 @@
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
// Input placement only; bufferization and folding determine physical requests.
|
||||
enum class ComputeInputMaterialization { Unused, ForwardConstant, ForwardFunctionInput, CopyAtEntry };
|
||||
|
||||
mlir::FailureOr<unsigned> getDirectReturnOperandIndex(mlir::OpResult result);
|
||||
|
||||
mlir::FailureOr<ComputeInputMaterialization>
|
||||
planComputeInputMaterialization(mlir::Operation* owner, unsigned inputIndex);
|
||||
|
||||
std::optional<unsigned> getDirectComputeLikeInputIndex(mlir::Operation* owner, unsigned operandNumber);
|
||||
|
||||
void replaceAndEraseDirectComputeLikeInput(mlir::PatternRewriter& rewriter,
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
|
||||
#include "src/Dialect/ONNX/ONNXOps.hpp"
|
||||
|
||||
@@ -80,7 +81,9 @@ static FailureOr<Value> lowerFragmentAssemblyBlueprint(IRRewriter& rewriter,
|
||||
return blueprint.emitOpError("contiguous row-major fragment storage does not match its logical result"), failure();
|
||||
return *reshaped;
|
||||
}
|
||||
SmallVector<int64_t> hostStrides = computeRowMajorStrides(resultType.getShape());
|
||||
auto checkedStrides = computeRowMajorStridesChecked(resultType.getShape());
|
||||
if (failed(checkedStrides)) return failure();
|
||||
SmallVector<int64_t> hostStrides = std::move(*checkedStrides);
|
||||
SmallVector<FragmentAssemblyCopy, 8> copies;
|
||||
for (int64_t fragmentIndex = 0; fragmentIndex < static_cast<int64_t>(operandIndices.size()); ++fragmentIndex) {
|
||||
int64_t operandIndex = operandIndices[fragmentIndex];
|
||||
@@ -430,10 +433,16 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeOp(spatial::SpatScheduledCom
|
||||
auto blockArg = computeOp.getInputArgument(inputIndex);
|
||||
if (!blockArg)
|
||||
return computeOp.emitOpError("expected compute input block arguments during input materialization");
|
||||
if (blockArg->use_empty())
|
||||
auto inputPlan = planComputeInputMaterialization(computeOp, inputIndex);
|
||||
if (failed(inputPlan))
|
||||
return computeOp.emitOpError("failed to plan compute input materialization");
|
||||
if (*inputPlan == ComputeInputMaterialization::Unused)
|
||||
continue;
|
||||
if (*inputPlan == ComputeInputMaterialization::ForwardFunctionInput)
|
||||
return computeOp.emitOpError("function inputs must be forwarded before core input materialization");
|
||||
|
||||
if (auto constantOp = input.getDefiningOp<arith::ConstantOp>()) {
|
||||
if (*inputPlan == ComputeInputMaterialization::ForwardConstant) {
|
||||
auto constantOp = input.getDefiningOp<arith::ConstantOp>();
|
||||
blockArg->replaceAllUsesWith(getOrCreateConstantLike(constantFolder, constantOp));
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -19,10 +19,12 @@ static void copyRaptorDebugAttrs(Operation* source, Operation* target) {
|
||||
}
|
||||
}
|
||||
|
||||
static Value createDestinationByteOffset(PatternRewriter& rewriter,
|
||||
static FailureOr<Value> createDestinationByteOffset(PatternRewriter& rewriter,
|
||||
tensor::InsertSliceOp insert) {
|
||||
auto destinationType = cast<RankedTensorType>(insert.getDestType());
|
||||
SmallVector<int64_t> strides = computeRowMajorStrides(destinationType.getShape());
|
||||
auto checkedStrides = computeRowMajorStridesChecked(destinationType.getShape());
|
||||
if (failed(checkedStrides)) return failure();
|
||||
SmallVector<int64_t> strides = std::move(*checkedStrides);
|
||||
int64_t elementBytes = getElementTypeSizeInBytes(destinationType.getElementType());
|
||||
Value total = arith::ConstantIndexOp::create(rewriter, insert.getLoc(), 0);
|
||||
for (auto [dimension, offset] : llvm::enumerate(insert.getMixedOffsets())) {
|
||||
@@ -114,9 +116,10 @@ static LogicalResult lowerReceive(
|
||||
}
|
||||
|
||||
rewriter.setInsertionPoint(destinationInsert);
|
||||
Value targetOffset = createDestinationByteOffset(rewriter, destinationInsert);
|
||||
auto targetOffset = createDestinationByteOffset(rewriter, destinationInsert);
|
||||
if (failed(targetOffset)) return failure();
|
||||
auto copy = pim::PimMemCopyOp::create(
|
||||
rewriter, op.getLoc(), destinationInsert.getDestType(), targetOffset, zero,
|
||||
rewriter, op.getLoc(), destinationInsert.getDestType(), *targetOffset, zero,
|
||||
destinationInsert.getDest(), *received, *sizeAttr);
|
||||
rewriter.replaceOp(destinationInsert, copy.getOutput());
|
||||
rewriter.eraseOp(op);
|
||||
|
||||
@@ -179,29 +179,18 @@ struct FuncOpArgToGlobalMemoryPattern final : OpRewritePattern<mlir::func::FuncO
|
||||
|
||||
for (auto& argUses : llvm::make_early_inc_range(arg.getUses())) {
|
||||
auto argUser = argUses.getOwner();
|
||||
if (auto spatCompute = dyn_cast<spatial::SpatScheduledCompute>(argUser)) {
|
||||
auto inputIndex = getDirectComputeLikeInputIndex(spatCompute, argUses.getOperandNumber());
|
||||
if (!inputIndex)
|
||||
if (isa<spatial::SpatScheduledCompute, spatial::SpatScheduledComputeBatch>(argUser)) {
|
||||
auto inputIndex = getDirectComputeLikeInputIndex(argUser, argUses.getOperandNumber());
|
||||
if (!inputIndex) return failure();
|
||||
auto plan = planComputeInputMaterialization(argUser, *inputIndex);
|
||||
if (failed(plan) || (*plan != ComputeInputMaterialization::ForwardFunctionInput
|
||||
&& *plan != ComputeInputMaterialization::Unused))
|
||||
return failure();
|
||||
auto BBArgIndex = *inputIndex;
|
||||
rewriter.setInsertionPoint(&spatCompute.getBody().front().front());
|
||||
rewriter.setInsertionPoint(&argUser->getRegion(0).front().front());
|
||||
auto getGlobalOp = memref::GetGlobalOp::create(rewriter, loc, memRefType, argName);
|
||||
auto toTensor = bufferization::ToTensorOp::create(
|
||||
rewriter, loc, argRankedTensorType, getGlobalOp, rewriter.getUnitAttr(), rewriter.getUnitAttr());
|
||||
|
||||
replaceAndEraseDirectComputeLikeInput(rewriter, spatCompute.getOperation(), BBArgIndex, toTensor);
|
||||
}
|
||||
else if (auto spatComputeBatch = dyn_cast<spatial::SpatScheduledComputeBatch>(argUser)) {
|
||||
auto inputIndex = getDirectComputeLikeInputIndex(spatComputeBatch, argUses.getOperandNumber());
|
||||
if (!inputIndex)
|
||||
return failure();
|
||||
auto BBArgIndex = *inputIndex;
|
||||
rewriter.setInsertionPoint(&spatComputeBatch.getBody().front().front());
|
||||
auto getGlobalOp = memref::GetGlobalOp::create(rewriter, loc, memRefType, argName);
|
||||
auto toTensor = bufferization::ToTensorOp::create(
|
||||
rewriter, loc, argRankedTensorType, getGlobalOp, rewriter.getUnitAttr(), rewriter.getUnitAttr());
|
||||
|
||||
replaceAndEraseDirectComputeLikeInput(rewriter, spatComputeBatch.getOperation(), BBArgIndex, toTensor);
|
||||
replaceAndEraseDirectComputeLikeInput(rewriter, argUser, *inputIndex, toTensor);
|
||||
}
|
||||
else {
|
||||
rewriter.setInsertionPoint(argUser);
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
#include "llvm/Support/MathExtras.h"
|
||||
|
||||
#include "ContiguityPatterns.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/LoopUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
|
||||
@@ -84,17 +85,6 @@ static FailureOr<int64_t> checkedPositiveMul(int64_t lhs, int64_t rhs) {
|
||||
return result;
|
||||
}
|
||||
|
||||
static FailureOr<int64_t> checkedPositiveProduct(ArrayRef<int64_t> values) {
|
||||
int64_t result = 1;
|
||||
for (int64_t value : values) {
|
||||
auto product = checkedPositiveMul(result, value);
|
||||
if (failed(product))
|
||||
return failure();
|
||||
result = *product;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static FailureOr<SmallVector<int64_t>> getStaticMemRefStrides(MemRefType type) {
|
||||
SmallVector<int64_t> strides;
|
||||
int64_t offset = 0;
|
||||
@@ -348,12 +338,9 @@ inferLogicalCopyShape(MemRefType targetType, MemRefType sourceType, int64_t size
|
||||
}
|
||||
|
||||
static FailureOr<int64_t> getContiguousSuffixRank(
|
||||
Value value, ArrayRef<int64_t> copyShape, Type elementType = {}) {
|
||||
Value value, ArrayRef<int64_t> copyShape) {
|
||||
auto type = dyn_cast<MemRefType>(value.getType());
|
||||
if (type && elementType && isPackedByteBuffer(type))
|
||||
return copyShape.size();
|
||||
if (!type || !type.hasStaticShape() || !hasByteSizedElementType(type.getElementType())
|
||||
|| (elementType && type.getElementType() != elementType)
|
||||
|| type.getRank() != static_cast<int64_t>(copyShape.size()))
|
||||
return failure();
|
||||
if (llvm::any_of(copyShape, [](int64_t dim) { return dim <= 0; }))
|
||||
@@ -363,42 +350,19 @@ static FailureOr<int64_t> getContiguousSuffixRank(
|
||||
if (failed(strides))
|
||||
return failure();
|
||||
|
||||
int64_t expectedStride = 1;
|
||||
int64_t contiguousSuffixRank = 0;
|
||||
for (int64_t dim = type.getRank() - 1; dim >= 0; --dim) {
|
||||
if (copyShape[dim] != 1 && (*strides)[dim] != expectedStride)
|
||||
break;
|
||||
++contiguousSuffixRank;
|
||||
auto nextStride = checkedPositiveMul(expectedStride, copyShape[dim]);
|
||||
if (failed(nextStride))
|
||||
return failure();
|
||||
expectedStride = *nextStride;
|
||||
}
|
||||
return contiguousSuffixRank;
|
||||
return onnx_mlir::getContiguousSuffixRank(copyShape, *strides);
|
||||
}
|
||||
|
||||
static FailureOr<SmallVector<int64_t>> getOuterByteStrides(
|
||||
Value value, const LogicalCopyShape ©Shape, size_t outerRank) {
|
||||
static FailureOr<SmallVector<int64_t>> getCopyElementStrides(
|
||||
Value value, const LogicalCopyShape ©Shape) {
|
||||
auto type = cast<MemRefType>(value.getType());
|
||||
SmallVector<int64_t> strides;
|
||||
if (isPackedByteBuffer(type))
|
||||
strides = computeRowMajorStrides(copyShape.dimensions);
|
||||
else {
|
||||
auto proven = getProvenMemRefStrides(value);
|
||||
if (failed(proven))
|
||||
return failure();
|
||||
strides = std::move(*proven);
|
||||
}
|
||||
int64_t elementByteWidth = static_cast<int64_t>(
|
||||
getElementTypeSizeInBytes(copyShape.elementType));
|
||||
SmallVector<int64_t> result;
|
||||
for (int64_t stride : ArrayRef<int64_t>(strides).take_front(outerRank)) {
|
||||
auto byteStride = checkedPositiveMul(stride, elementByteWidth);
|
||||
if (failed(byteStride))
|
||||
return failure();
|
||||
result.push_back(*byteStride);
|
||||
}
|
||||
return result;
|
||||
return computeRowMajorStridesChecked(copyShape.dimensions);
|
||||
if (!hasByteSizedElementType(type.getElementType())
|
||||
|| type.getElementType() != copyShape.elementType
|
||||
|| type.getRank() != static_cast<int64_t>(copyShape.dimensions.size()))
|
||||
return failure();
|
||||
return getProvenMemRefStrides(value);
|
||||
}
|
||||
|
||||
static FailureOr<CopyEndpointPlan> analyzeCopyEndpoint(Value value, Value initialByteOffset, MemRefType logicalType) {
|
||||
@@ -498,20 +462,19 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO
|
||||
if (failed(logicalCopyShape))
|
||||
return failure();
|
||||
|
||||
auto targetSuffixRank = getContiguousSuffixRank(
|
||||
target, logicalCopyShape->dimensions, logicalCopyShape->elementType);
|
||||
auto sourceSuffixRank = getContiguousSuffixRank(
|
||||
source, logicalCopyShape->dimensions, logicalCopyShape->elementType);
|
||||
if (failed(targetSuffixRank) || failed(sourceSuffixRank))
|
||||
auto targetStrides = getCopyElementStrides(target, *logicalCopyShape);
|
||||
auto sourceStrides = getCopyElementStrides(source, *logicalCopyShape);
|
||||
if (failed(targetStrides) || failed(sourceStrides))
|
||||
return failure();
|
||||
auto access = CopyAccessPlan::build(
|
||||
logicalCopyShape->dimensions, *sourceStrides, *targetStrides);
|
||||
if (failed(access))
|
||||
return failure();
|
||||
|
||||
CopyRewritePlan plan;
|
||||
plan.target = *targetPlan;
|
||||
plan.source = *sourcePlan;
|
||||
|
||||
int64_t contiguousSuffixRank = std::min(*targetSuffixRank, *sourceSuffixRank);
|
||||
if (contiguousSuffixRank
|
||||
== static_cast<int64_t>(logicalCopyShape->dimensions.size())) {
|
||||
if (access->getOuterShape().empty()) {
|
||||
plan.kind = CopyRewritePlan::Kind::Direct;
|
||||
plan.directBytes = size;
|
||||
return plan;
|
||||
@@ -522,30 +485,22 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO
|
||||
plan.kind = CopyRewritePlan::Kind::Loop;
|
||||
plan.loop.targetBaseOffset = plan.target.offset;
|
||||
plan.loop.sourceBaseOffset = plan.source.offset;
|
||||
plan.loop.outerShape.assign(
|
||||
logicalCopyShape->dimensions.begin(),
|
||||
logicalCopyShape->dimensions.end() - contiguousSuffixRank);
|
||||
SmallVector<int64_t> chunkShape(
|
||||
logicalCopyShape->dimensions.end() - contiguousSuffixRank,
|
||||
logicalCopyShape->dimensions.end());
|
||||
auto outerElements = checkedPositiveProduct(plan.loop.outerShape);
|
||||
auto chunkElements = checkedPositiveProduct(chunkShape);
|
||||
auto chunkBytes = failed(chunkElements)
|
||||
? FailureOr<int64_t>(failure())
|
||||
: checkedPositiveMul(*chunkElements, elementByteWidth);
|
||||
if (failed(outerElements) || failed(chunkBytes))
|
||||
plan.loop.outerShape.assign(access->getOuterShape().begin(), access->getOuterShape().end());
|
||||
auto chunkBytes = checkedPositiveMul(access->getRunElements(), elementByteWidth);
|
||||
if (failed(chunkBytes))
|
||||
return failure();
|
||||
plan.loop.outerElements = *outerElements;
|
||||
plan.loop.outerElements = access->getRunCount();
|
||||
plan.loop.chunkBytes = *chunkBytes;
|
||||
auto targetStrides = getOuterByteStrides(
|
||||
target, *logicalCopyShape, plan.loop.outerShape.size());
|
||||
auto sourceStrides = getOuterByteStrides(
|
||||
source, *logicalCopyShape, plan.loop.outerShape.size());
|
||||
if (failed(targetStrides) || failed(sourceStrides))
|
||||
return failure();
|
||||
plan.loop.targetOuterByteStrides = std::move(*targetStrides);
|
||||
plan.loop.sourceOuterByteStrides = std::move(*sourceStrides);
|
||||
if (plan.loop.chunkBytes <= 0)
|
||||
auto scaleStrides = [&](ArrayRef<int64_t> strides, SmallVectorImpl<int64_t> &result) {
|
||||
for (int64_t stride : strides) {
|
||||
auto byteStride = checkedPositiveMul(stride, elementByteWidth);
|
||||
if (failed(byteStride)) return failure();
|
||||
result.push_back(*byteStride);
|
||||
}
|
||||
return success();
|
||||
};
|
||||
if (failed(scaleStrides(access->getTargetOuterStrides(), plan.loop.targetOuterByteStrides))
|
||||
|| failed(scaleStrides(access->getSourceOuterStrides(), plan.loop.sourceOuterByteStrides)))
|
||||
return failure();
|
||||
return plan;
|
||||
}
|
||||
|
||||
@@ -94,8 +94,12 @@ foldDenseSubview(DenseElementsAttr denseAttr, ArrayRef<int64_t> staticOffsets, A
|
||||
if (numResultElements < 0)
|
||||
return failure();
|
||||
|
||||
auto sourceStrides = computeRowMajorStrides(sourceType.getShape());
|
||||
auto resultStrides = computeRowMajorStrides(resultTensorType.getShape());
|
||||
auto checkedSourceStrides = computeRowMajorStridesChecked(sourceType.getShape());
|
||||
if (failed(checkedSourceStrides)) return failure();
|
||||
auto sourceStrides = std::move(*checkedSourceStrides);
|
||||
auto checkedResultStrides = computeRowMajorStridesChecked(resultTensorType.getShape());
|
||||
if (failed(checkedResultStrides)) return failure();
|
||||
auto resultStrides = std::move(*checkedResultStrides);
|
||||
SmallVector<Attribute> sourceValues(denseAttr.getValues<Attribute>());
|
||||
SmallVector<Attribute> resultValues(numResultElements);
|
||||
for (int64_t i = 0; i < numResultElements; ++i) {
|
||||
|
||||
@@ -218,7 +218,9 @@ static FailureOr<DenseElementsAttr> foldConstantAlloc(memref::AllocOp allocOp, M
|
||||
return failure();
|
||||
|
||||
SmallVector<Attribute> resultValues(numElements, fillValue);
|
||||
auto resultStrides = computeRowMajorStrides(resultTensorType.getShape());
|
||||
auto checkedResultStrides = computeRowMajorStridesChecked(resultTensorType.getShape());
|
||||
if (failed(checkedResultStrides)) return failure();
|
||||
auto resultStrides = std::move(*checkedResultStrides);
|
||||
|
||||
llvm::sort(copies, [](const ConstantSubviewCopy& lhs, const ConstantSubviewCopy& rhs) {
|
||||
return lhs.copyOp->isBeforeInBlock(rhs.copyOp);
|
||||
@@ -232,7 +234,11 @@ static FailureOr<DenseElementsAttr> foldConstantAlloc(memref::AllocOp allocOp, M
|
||||
|| sourceType.getRank() != static_cast<int64_t>(copy.strides.size()))
|
||||
return failure();
|
||||
|
||||
auto sourceStrides = computeRowMajorStrides(sourceType.getShape());
|
||||
auto checkedSourceStrides = computeRowMajorStridesChecked(sourceType.getShape());
|
||||
|
||||
if (failed(checkedSourceStrides)) return failure();
|
||||
|
||||
auto sourceStrides = std::move(*checkedSourceStrides);
|
||||
SmallVector<Attribute> sourceValues(copy.source.getValues<Attribute>());
|
||||
for (auto [linearIndex, value] : llvm::enumerate(sourceValues)) {
|
||||
SmallVector<int64_t> sourceIndices =
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include <tuple>
|
||||
|
||||
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/Diagnostics.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Pim/Passes/Transforms/LocalMemoryPlanning/LocalMemoryPlanning.hpp"
|
||||
#include "src/Accelerators/PIM/Passes/PIMPasses.h"
|
||||
@@ -149,7 +150,7 @@ FailureOr<CoreMemoryPlan> buildCoreMemoryPlan(Operation* coreLikeOp) {
|
||||
plan.intervals = std::move(*intervals);
|
||||
auto placements = planLocalMemoryPlacements(plan.intervals, kPimLocalMemoryAddressLimit);
|
||||
if (failed(placements)) {
|
||||
coreLikeOp->emitError("Pim local-memory plan exceeds the signed int32 address range");
|
||||
pim::emitCandidateResourceError(coreLikeOp, "Pim local-memory plan exceeds the signed int32 address range");
|
||||
return failure();
|
||||
}
|
||||
plan.placements = std::move(*placements);
|
||||
|
||||
@@ -11,6 +11,7 @@ struct PreparedSpatialSchedules {
|
||||
std::vector<MergeScheduleResult> candidates;
|
||||
};
|
||||
|
||||
// Success always includes at least one candidate, including an empty compute graph.
|
||||
mlir::FailureOr<PreparedSpatialSchedules> prepareSpatialSchedules(
|
||||
mlir::ModuleOp module, const SchedulingTarget &target,
|
||||
size_t pipelineStages, bool collectCandidates);
|
||||
|
||||
+53
-34
@@ -15,6 +15,7 @@
|
||||
#include "llvm/ADT/DenseSet.h"
|
||||
#include "llvm/ADT/SetVector.h"
|
||||
#include "llvm/Support/Casting.h"
|
||||
#include "llvm/Support/CheckedArithmetic.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
@@ -30,6 +31,7 @@
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/DeferredProjectionAnalysis.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
|
||||
#include "src/Support/TypeUtilities.hpp"
|
||||
@@ -55,6 +57,7 @@ struct SchedulerCostModel {
|
||||
|
||||
const SchedulingTarget& target;
|
||||
ComputeCostKind kind = ComputeCostKind::Execution;
|
||||
mutable bool projectionFallbackReported = false;
|
||||
|
||||
static Cost ceilDiv(Cost numerator, Cost denominator) {
|
||||
assert(denominator > 0 && "denominator must be positive");
|
||||
@@ -534,18 +537,10 @@ static bool isSharedMemoryInput(Value input, const SchedulingTarget &target) {
|
||||
Attribute value;
|
||||
if (!matchPattern(input, m_Constant(&value)) || !isa<ElementsAttr>(value))
|
||||
return false;
|
||||
auto dense = dyn_cast<DenseElementsAttr>(value);
|
||||
if (!target.zeroSplatUsesSharedMemory && dense && dense.isSplat()) {
|
||||
auto element = dense.getSplatValue<Attribute>();
|
||||
if (auto floating = dyn_cast<FloatAttr>(element))
|
||||
return !floating.getValue().isZero() || floating.getValue().isNegative();
|
||||
if (auto integer = dyn_cast<IntegerAttr>(element))
|
||||
return !integer.getValue().isZero();
|
||||
}
|
||||
return true;
|
||||
return target.zeroSplatUsesSharedMemory || !isZeroBitPatternSplat(value);
|
||||
}
|
||||
|
||||
static std::pair<Cost, Cost> getProjectionMemoryRequests(Type type, Value projection,
|
||||
static FailureOr<std::pair<Cost, Cost>> getProjectionMemoryRequests(Type type, Value projection,
|
||||
const SchedulerCostModel &costModel) {
|
||||
Cost bytes = costModel.getByteSize(type, costModel.getComputeBitwidth(type));
|
||||
Cost requests = 1;
|
||||
@@ -553,40 +548,53 @@ static std::pair<Cost, Cost> getProjectionMemoryRequests(Type type, Value projec
|
||||
extract && extract.getSourceType().hasStaticShape()
|
||||
&& !ShapedType::isDynamicShape(extract.getStaticSizes())
|
||||
&& !ShapedType::isDynamicShape(extract.getStaticStrides())) {
|
||||
auto strides = computeRowMajorStrides(extract.getSourceType().getShape());
|
||||
Cost run = 1;
|
||||
for (size_t dim = strides.size(); dim-- > 0;) {
|
||||
Cost size = extract.getStaticSizes()[dim];
|
||||
if (size == 1) continue;
|
||||
if (checkedMultiply(static_cast<Cost>(strides[dim]),
|
||||
static_cast<Cost>(extract.getStaticStrides()[dim])) != run)
|
||||
break;
|
||||
run = checkedMultiply(run, size);
|
||||
auto unavailable = [&]() -> FailureOr<std::pair<Cost, Cost>> {
|
||||
if (!costModel.projectionFallbackReported) {
|
||||
extract.emitWarning("projection memory cost unavailable: using one request per element because static copy geometry exceeds supported offset or stride limits");
|
||||
costModel.projectionFallbackReported = true;
|
||||
}
|
||||
// A physical copy cannot need more than one request per element. Round
|
||||
// each element separately so sub-byte elements cannot undercharge service.
|
||||
return std::make_pair(SchedulerCostModel::ceilDiv(costModel.getComputeBitwidth(type), Cost{8}),
|
||||
static_cast<Cost>(cast<ShapedType>(type).getNumElements()));
|
||||
};
|
||||
auto strides = computeRowMajorStridesChecked(extract.getSourceType().getShape());
|
||||
auto targetStrides = computeRowMajorStridesChecked(extract.getStaticSizes());
|
||||
if (failed(strides) || failed(targetStrides)) return unavailable();
|
||||
for (size_t dim = 0; dim < strides->size(); ++dim) {
|
||||
auto stride = llvm::checkedMul((*strides)[dim], extract.getStaticStrides()[dim]);
|
||||
if (!stride) return unavailable();
|
||||
(*strides)[dim] = *stride;
|
||||
}
|
||||
Cost elements = cast<ShapedType>(type).getNumElements();
|
||||
if (elements) {
|
||||
auto access = CopyAccessPlan::build(extract.getStaticSizes(), *strides, *targetStrides);
|
||||
if (failed(access))
|
||||
return unavailable();
|
||||
Cost run = access->getRunElements();
|
||||
requests = checkedMultiply(requests, elements / run);
|
||||
bytes = SchedulerCostModel::ceilDiv(
|
||||
checkedMultiply(run, costModel.getComputeBitwidth(type)), Cost{8});
|
||||
}
|
||||
}
|
||||
return {bytes, requests};
|
||||
return std::make_pair(bytes, requests);
|
||||
}
|
||||
|
||||
static SharedMemoryDemand getInputMemoryDemand(const InputFragment &fragment,
|
||||
static FailureOr<SharedMemoryDemand> getInputMemoryDemand(const InputFragment &fragment,
|
||||
const SchedulerCostModel &costModel) {
|
||||
auto [bytes, runs] = getProjectionMemoryRequests(fragment.type, fragment.projection, costModel);
|
||||
return costModel.target.getSharedMemoryDemand(bytes,
|
||||
checkedMultiply(runs, fragment.count), false);
|
||||
auto access = getProjectionMemoryRequests(fragment.type, fragment.projection, costModel);
|
||||
if (failed(access)) return failure();
|
||||
return costModel.target.getSharedMemoryDemand(access->first,
|
||||
checkedMultiply(access->second, fragment.count), false);
|
||||
}
|
||||
|
||||
static SharedMemoryDemand getCapturedConstantMemoryDemand(
|
||||
static FailureOr<SharedMemoryDemand> getCapturedConstantMemoryDemand(
|
||||
Operation *owner, const SchedulerCostModel &costModel) {
|
||||
// shortcut: later aliasing/load folding can change baseline requests; exact
|
||||
// accounting requires a placed, bufferized program.
|
||||
SharedMemoryDemand demand;
|
||||
owner->walk([&](Operation *user) {
|
||||
if (user == owner) return;
|
||||
auto result = owner->walk([&](Operation *user) -> WalkResult {
|
||||
if (user == owner) return WalkResult::advance();
|
||||
for (Value value : user->getOperands()) {
|
||||
if (isa<BlockArgument>(value) || !isSharedMemoryInput(value, costModel.target)) continue;
|
||||
if (auto extract = dyn_cast<tensor::ExtractOp>(user)) {
|
||||
@@ -618,9 +626,13 @@ static SharedMemoryDemand getCapturedConstantMemoryDemand(
|
||||
}
|
||||
auto extract = dyn_cast<tensor::ExtractSliceOp>(user);
|
||||
Type type = extract ? extract.getResult().getType() : value.getType();
|
||||
demand += getInputMemoryDemand({type, count, extract ? extract.getResult() : value}, costModel);
|
||||
auto memory = getInputMemoryDemand({type, count, extract ? extract.getResult() : value}, costModel);
|
||||
if (failed(memory)) return WalkResult::interrupt();
|
||||
demand += *memory;
|
||||
}
|
||||
return WalkResult::advance();
|
||||
});
|
||||
if (result.wasInterrupted()) return failure();
|
||||
return demand;
|
||||
}
|
||||
|
||||
@@ -840,10 +852,11 @@ static LogicalResult collectPublicationTransfer(Value input, size_t targetIndex,
|
||||
if (auto fragments = getProjectedInputFragments(input, consumer)) {
|
||||
for (const InputFragment &fragment : *fragments) {
|
||||
Cost bytes = costModel.getByteSize(fragment.type, costModel.getComputeBitwidth(fragment.type));
|
||||
auto [storeBytes, storeRuns] = getProjectionMemoryRequests(
|
||||
auto access = getProjectionMemoryRequests(
|
||||
fragment.type, fragment.projection, costModel);
|
||||
if (failed(access)) return failure();
|
||||
graph.publicationTransfers.push_back({sourceIndex, targetIndex, value,
|
||||
0, fragment.projection, bytes, fragment.count, {}, storeBytes, storeRuns});
|
||||
0, fragment.projection, bytes, fragment.count, {}, access->first, access->second});
|
||||
}
|
||||
} else {
|
||||
Cost bytes = costModel.getByteSize(input.getType(), costModel.getComputeBitwidth(input.getType()));
|
||||
@@ -1292,8 +1305,11 @@ FailureOr<ComputeGraph> buildComputeGraph(Operation* entryOp,
|
||||
DenseMap<Operation *, SharedMemoryDemand> capturedMemory;
|
||||
for (const auto& [targetIndex, node] : llvm::enumerate(graph.nodes)) {
|
||||
auto [captured, inserted] = capturedMemory.try_emplace(node.instance.op);
|
||||
if (inserted)
|
||||
captured->second = getCapturedConstantMemoryDemand(node.instance.op, costModel);
|
||||
if (inserted) {
|
||||
auto memory = getCapturedConstantMemoryDemand(node.instance.op, costModel);
|
||||
if (failed(memory)) return failure();
|
||||
captured->second = *memory;
|
||||
}
|
||||
auto memory = captured->second;
|
||||
memory.readBytes = checkedMultiply(memory.readBytes, static_cast<Cost>(node.instance.laneCount));
|
||||
memory.readRequests = checkedMultiply(memory.readRequests, static_cast<Cost>(node.instance.laneCount));
|
||||
@@ -1304,8 +1320,11 @@ FailureOr<ComputeGraph> buildComputeGraph(Operation* entryOp,
|
||||
size_t publicationsBefore = graph.publicationTransfers.size();
|
||||
if (isSharedMemoryInput(input, target)) {
|
||||
if (auto fragments = getProjectedInputFragments(input, node.instance)) {
|
||||
for (const InputFragment &fragment : *fragments)
|
||||
graph.baselineMemory += getInputMemoryDemand(fragment, costModel);
|
||||
for (const InputFragment &fragment : *fragments) {
|
||||
auto memory = getInputMemoryDemand(fragment, costModel);
|
||||
if (failed(memory)) return failure();
|
||||
graph.baselineMemory += *memory;
|
||||
}
|
||||
} else {
|
||||
graph.baselineMemory += target.getSharedMemoryDemand(
|
||||
costModel.getByteSize(input.getType(), costModel.getComputeBitwidth(input.getType())),
|
||||
|
||||
+30
-8
@@ -6,11 +6,33 @@
|
||||
|
||||
namespace onnx_mlir::spatial {
|
||||
|
||||
bool WholeScheduleCost::eligible() const {
|
||||
bool ScheduleRecurrenceEvidence::proved() const {
|
||||
return status == Proved && periodGenerations && intervalNs
|
||||
&& std::isfinite(*intervalNs) && *intervalNs > 0;
|
||||
}
|
||||
|
||||
std::optional<bool> WholeScheduleCost::convergenceProved() const {
|
||||
if (recurrence[0].status == ScheduleRecurrenceEvidence::NotChecked
|
||||
&& recurrence[1].status == ScheduleRecurrenceEvidence::NotChecked)
|
||||
return std::nullopt;
|
||||
return recurrence[0].proved() && recurrence[1].proved();
|
||||
}
|
||||
|
||||
std::optional<bool> WholeScheduleCost::loweringSupported() const {
|
||||
if (loweringSupport == StaticLoweringSupport::Unknown) return std::nullopt;
|
||||
return loweringSupport == StaticLoweringSupport::Supported;
|
||||
}
|
||||
|
||||
bool WholeScheduleCost::available() const {
|
||||
return primary && alternative && std::isfinite(*primary) && std::isfinite(*alternative)
|
||||
&& *primary > 0 && *alternative > 0 && relativeUncertainty
|
||||
&& std::isfinite(*relativeUncertainty) && *relativeUncertainty >= 0
|
||||
&& *relativeUncertainty < 1;
|
||||
&& *primary > 0 && *alternative > 0;
|
||||
}
|
||||
|
||||
bool WholeScheduleCost::eligible() const {
|
||||
return available() && convergenceProved() == true
|
||||
&& loweringSupport == StaticLoweringSupport::Supported && selectionMargin
|
||||
&& std::isfinite(*selectionMargin) && *selectionMargin >= 0
|
||||
&& *selectionMargin < 1;
|
||||
}
|
||||
|
||||
size_t selectPipelineCandidate(llvm::ArrayRef<WholeScheduleCost> scores, std::string &reason) {
|
||||
@@ -21,17 +43,17 @@ size_t selectPipelineCandidate(llvm::ArrayRef<WholeScheduleCost> scores, std::st
|
||||
size_t best = 0;
|
||||
auto upper = [&](size_t i) {
|
||||
return std::max(*scores[i].primary, *scores[i].alternative)
|
||||
/ (1 - *scores[i].relativeUncertainty);
|
||||
/ (1 - *scores[i].selectionMargin);
|
||||
};
|
||||
for (size_t i = 1; i < scores.size(); ++i)
|
||||
if (scores[i].eligible() && upper(i) < upper(best)) best = i;
|
||||
if (!best) { reason = "original has the lowest conservative estimate"; return 0; }
|
||||
double originalLower = std::min(*scores.front().primary, *scores.front().alternative);
|
||||
if (upper(best) >= originalLower / (1 + *scores.front().relativeUncertainty)) {
|
||||
reason = "candidate and original overlap static prediction uncertainty";
|
||||
if (upper(best) >= originalLower / (1 + *scores.front().selectionMargin)) {
|
||||
reason = "candidate and original overlap the selection policy margin";
|
||||
return 0;
|
||||
}
|
||||
reason = "candidate improves on original under both arbitration tie policies and prediction uncertainty";
|
||||
reason = "candidate improves on original under both arbitration tie policies and the selection margin";
|
||||
return best;
|
||||
}
|
||||
|
||||
|
||||
+1
@@ -8,6 +8,7 @@
|
||||
namespace onnx_mlir::spatial {
|
||||
|
||||
struct MergeScheduleResult;
|
||||
inline constexpr double kPipelineSelectionMargin = 0.021;
|
||||
|
||||
size_t selectPipelineCandidate(llvm::ArrayRef<WholeScheduleCost> scores,
|
||||
std::string &reason);
|
||||
|
||||
+57
-48
@@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <cassert>
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <optional>
|
||||
@@ -39,20 +40,36 @@ struct ScheduledComputeMaterializationResult;
|
||||
struct DeferredTransferPlan;
|
||||
struct DeferredBoundaryPlan;
|
||||
|
||||
// Unknown lacks a proof. Unsupported identifies a known static backend violation.
|
||||
// Supported covers checked static forms, not final allocation or code generation.
|
||||
enum class StaticLoweringSupport { Unknown, Supported, Unsupported };
|
||||
|
||||
struct ScheduleRecurrenceEvidence {
|
||||
enum Status { NotChecked, Unproved, Proved };
|
||||
Status status = NotChecked;
|
||||
size_t periodGenerations = 0;
|
||||
std::optional<double> intervalNs;
|
||||
std::string reason;
|
||||
bool proved() const;
|
||||
};
|
||||
|
||||
struct WholeScheduleCost {
|
||||
// Steady-state intervals in nanoseconds. The alternative tests sensitivity.
|
||||
std::optional<double> primary, alternative;
|
||||
std::string reason;
|
||||
std::string identity;
|
||||
std::optional<double> relativeUncertainty;
|
||||
// Selection policy only. This is not a prediction error bound.
|
||||
std::optional<double> selectionMargin;
|
||||
std::array<ScheduleRecurrenceEvidence, 2> recurrence;
|
||||
StaticLoweringSupport loweringSupport = StaticLoweringSupport::Unknown;
|
||||
struct AnalysisTiming {
|
||||
double extractionSeconds = 0, expansionSeconds = 0, evaluationSeconds = 0;
|
||||
} timing;
|
||||
std::optional<bool> convergenceProved() const;
|
||||
std::optional<bool> loweringSupported() const;
|
||||
bool available() const;
|
||||
bool eligible() const;
|
||||
};
|
||||
|
||||
struct ScheduleCostQuery {
|
||||
std::string identity;
|
||||
std::function<WholeScheduleCost(llvm::json::Object &)> evaluate;
|
||||
};
|
||||
|
||||
struct SchedulingTarget {
|
||||
size_t processorCount = 0;
|
||||
size_t residentWeightCapacity = 0;
|
||||
@@ -76,38 +93,38 @@ struct SchedulingTarget {
|
||||
Cost sharedMemoryWriteLatencyNs = 0;
|
||||
bool zeroSplatUsesSharedMemory = true;
|
||||
|
||||
// Input is a verified, target-lowered candidate. Preparation and evaluation
|
||||
// are read-only. An absent query keeps the original schedule.
|
||||
std::function<mlir::FailureOr<ScheduleCostQuery>(mlir::ModuleOp,
|
||||
llvm::json::Object &)> prepareScheduleCost;
|
||||
// Optional read-only estimate over realized Spatial. Finite costs do not
|
||||
// establish convergence or lowering support. Callers must validate these
|
||||
// properties and apply the Spatial selection policy.
|
||||
std::function<WholeScheduleCost(mlir::ModuleOp,
|
||||
llvm::json::Object &)> estimateRealizedScheduleCost;
|
||||
std::string wholeScheduleCostUnavailable;
|
||||
std::function<void(const ComputeGraph &,
|
||||
const ScheduledComputeMaterializationResult &, const DeferredTransferPlan &,
|
||||
const DeferredBoundaryPlan &)> reportPeriodicSchedule;
|
||||
|
||||
Cost getLocalMemoryServiceNs(Cost bytes, bool write) const {
|
||||
Cost accesses = bytes / localMemoryWidthBytes
|
||||
+ (bytes % localMemoryWidthBytes != 0);
|
||||
return checkedMultiply(accesses, checkedMultiply(localMemoryPeriodNs,
|
||||
write ? localMemoryWriteLatencyCycles : localMemoryReadLatencyCycles));
|
||||
template <typename Arithmetic = SchedulingCostArithmetic>
|
||||
Cost getLocalMemoryServiceNs(Cost bytes, bool write, Arithmetic math = {}) const {
|
||||
return math.multiply(math.divideUp(bytes, localMemoryWidthBytes),
|
||||
math.multiply(localMemoryPeriodNs,
|
||||
write ? localMemoryWriteLatencyCycles : localMemoryReadLatencyCycles));
|
||||
}
|
||||
|
||||
template <typename Arithmetic = SchedulingCostArithmetic>
|
||||
SharedMemoryDemand getSharedMemoryDemand(
|
||||
Cost bytesPerRequest, Cost requests, bool write) const {
|
||||
Cost bytesPerRequest, Cost requests, bool write, Arithmetic math = {}) const {
|
||||
SharedMemoryDemand demand;
|
||||
if (write) {
|
||||
demand.writeBytes = checkedMultiply(bytesPerRequest, requests);
|
||||
demand.writeBytes = math.multiply(bytesPerRequest, requests);
|
||||
demand.writeRequests = requests;
|
||||
} else {
|
||||
demand.readBytes = checkedMultiply(bytesPerRequest, requests);
|
||||
demand.readBytes = math.multiply(bytesPerRequest, requests);
|
||||
demand.readRequests = requests;
|
||||
}
|
||||
if (sharedMemoryWidthBytes) {
|
||||
Cost accesses = bytesPerRequest / sharedMemoryWidthBytes
|
||||
+ (bytesPerRequest % sharedMemoryWidthBytes != 0);
|
||||
demand.serviceNs = checkedMultiply(requests, checkedMultiply(accesses,
|
||||
if (sharedMemoryWidthBytes)
|
||||
demand.serviceNs = math.multiply(requests, math.multiply(
|
||||
math.divideUp(bytesPerRequest, sharedMemoryWidthBytes),
|
||||
write ? sharedMemoryWriteLatencyNs : sharedMemoryReadLatencyNs));
|
||||
}
|
||||
return demand;
|
||||
}
|
||||
|
||||
@@ -127,33 +144,25 @@ struct SchedulingTarget {
|
||||
Cost matrixInputBufferLatencyCycles = 0;
|
||||
bool matrixPipeline = true;
|
||||
|
||||
Cost getMatrixComputeLatencyNs(Cost inputBitwidth) const {
|
||||
auto divideRoundedUp = [](Cost value, Cost width) {
|
||||
return value / width + (value % width != 0);
|
||||
};
|
||||
Cost inputTimes = divideRoundedUp(inputBitwidth, matrixInputResolutionBits);
|
||||
Cost inputPasses = divideRoundedUp(matrixRows, matrixInputParallelism);
|
||||
Cost outputPasses = divideRoundedUp(matrixColumns, matrixOutputParallelism);
|
||||
Cost readCycles = divideRoundedUp(matrixReadLatencyNs, matrixPeriodNs);
|
||||
Cost frontStage = matrixInputBufferLatencyCycles + matrixInputLatencyCycles
|
||||
+ readCycles + matrixSampleLatencyCycles;
|
||||
template <typename Arithmetic = SchedulingCostArithmetic>
|
||||
Cost getMatrixComputeLatencyNs(Cost inputBitwidth, Arithmetic math = {}) const {
|
||||
Cost inputTimes = math.divideUp(inputBitwidth, matrixInputResolutionBits);
|
||||
Cost inputPasses = math.divideUp(matrixRows, matrixInputParallelism);
|
||||
Cost outputPasses = math.divideUp(matrixColumns, matrixOutputParallelism);
|
||||
Cost readCycles = math.divideUp(matrixReadLatencyNs, matrixPeriodNs);
|
||||
Cost frontStage = math.add(math.add(matrixInputBufferLatencyCycles,
|
||||
matrixInputLatencyCycles), math.add(readCycles, matrixSampleLatencyCycles));
|
||||
Cost backPipe = std::max(matrixOutputLatencyCycles,
|
||||
checkedAdd(matrixShiftLatencyCycles,
|
||||
matrixBufferLatencyCycles));
|
||||
Cost backStage = checkedAdd(
|
||||
checkedAdd(matrixOutputLatencyCycles, matrixShiftLatencyCycles),
|
||||
matrixBufferLatencyCycles);
|
||||
backStage = checkedAdd(backStage, checkedMultiply(outputPasses - 1, backPipe));
|
||||
Cost totalTimes = checkedMultiply(inputTimes, inputPasses);
|
||||
math.add(matrixShiftLatencyCycles, matrixBufferLatencyCycles));
|
||||
Cost backStage = math.add(math.add(matrixOutputLatencyCycles,
|
||||
matrixShiftLatencyCycles), matrixBufferLatencyCycles);
|
||||
backStage = math.add(backStage, math.multiply(math.subtract(outputPasses, 1), backPipe));
|
||||
Cost totalTimes = math.multiply(inputTimes, inputPasses);
|
||||
if (!matrixPipeline)
|
||||
return checkedMultiply(
|
||||
checkedMultiply(checkedAdd(frontStage, backStage), totalTimes),
|
||||
matrixPeriodNs);
|
||||
return math.multiply(math.multiply(math.add(frontStage, backStage), totalTimes), matrixPeriodNs);
|
||||
Cost stagePipe = std::max(frontStage, backStage);
|
||||
return checkedMultiply(
|
||||
checkedAdd(checkedAdd(frontStage, backStage),
|
||||
checkedMultiply(totalTimes - 1, stagePipe)),
|
||||
matrixPeriodNs);
|
||||
return math.multiply(math.add(math.add(frontStage, backStage),
|
||||
math.multiply(math.subtract(totalTimes, 1), stagePipe)), matrixPeriodNs);
|
||||
}
|
||||
|
||||
Cost getInterProcessorLatencyNs(size_t source,
|
||||
|
||||
@@ -36,6 +36,13 @@ inline T checkedMultiply(T lhs, T rhs) {
|
||||
return lhs * rhs;
|
||||
}
|
||||
|
||||
struct SchedulingCostArithmetic {
|
||||
Cost add(Cost a, Cost b) const { return checkedAdd(a, b); }
|
||||
Cost multiply(Cost a, Cost b) const { return checkedMultiply(a, b); }
|
||||
Cost subtract(Cost a, Cost b) const { assert(a >= b); return a - b; }
|
||||
Cost divideUp(Cost a, Cost b) const { assert(b); return a / b + (a % b != 0); }
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
inline T addOrMax(T lhs, T rhs) {
|
||||
static_assert(std::is_unsigned_v<T>, "addOrMax only supports unsigned types");
|
||||
|
||||
Reference in New Issue
Block a user