This commit is contained in:
@@ -1,10 +1,23 @@
|
||||
#include "Dialect/Pim/Transforms/Bufferization/Common.hpp"
|
||||
#include "mlir/Dialect/SCF/IR/SCF.h"
|
||||
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
|
||||
|
||||
using namespace mlir;
|
||||
|
||||
static SmallVector<Region *> getSelectionRegions(OpResult result) {
|
||||
SmallVector<Region *> regions;
|
||||
if (auto selection = dyn_cast<scf::IndexSwitchOp>(result.getOwner()))
|
||||
for (Region ®ion : selection->getRegions())
|
||||
regions.push_back(®ion);
|
||||
else if (auto selection = dyn_cast<scf::IfOp>(result.getOwner())) {
|
||||
regions.push_back(&selection.getThenRegion());
|
||||
regions.push_back(&selection.getElseRegion());
|
||||
}
|
||||
return regions;
|
||||
}
|
||||
|
||||
static bool isCoreBatchInputArgument(Value value) {
|
||||
auto blockArg = dyn_cast<BlockArgument>(value);
|
||||
if (!blockArg)
|
||||
@@ -92,20 +105,46 @@ FailureOr<Value> onnx_mlir::pim::getPimAddressBase(Value value, const StaticValu
|
||||
}
|
||||
|
||||
bool onnx_mlir::pim::isHostBackedPimAddress(Value value, const StaticValueKnowledge& knowledge) {
|
||||
auto base = getPimStorageBase(value, knowledge);
|
||||
if (failed(base))
|
||||
return false;
|
||||
|
||||
if (isCoreBatchInputArgument(*base))
|
||||
return true;
|
||||
|
||||
return isa_and_nonnull<memref::GetGlobalOp>(base->getDefiningOp());
|
||||
llvm::SmallPtrSet<Value, 8> visited;
|
||||
std::function<bool(Value)> isHost = [&](Value current) {
|
||||
auto base = getPimStorageBase(current, knowledge);
|
||||
if (failed(base) || !visited.insert(*base).second)
|
||||
return false;
|
||||
bool resultIsHost = isCoreBatchInputArgument(*base)
|
||||
|| isa_and_nonnull<memref::GetGlobalOp>(base->getDefiningOp());
|
||||
auto result = dyn_cast<OpResult>(*base);
|
||||
SmallVector<Region *> regions = result ? getSelectionRegions(result)
|
||||
: SmallVector<Region *>();
|
||||
if (!resultIsHost && !regions.empty())
|
||||
resultIsHost = llvm::all_of(regions, [&](Region *region) {
|
||||
auto yield = dyn_cast<scf::YieldOp>(region->front().getTerminator());
|
||||
return yield && result.getResultNumber() < yield.getNumOperands()
|
||||
&& isHost(yield.getOperand(result.getResultNumber()));
|
||||
});
|
||||
visited.erase(*base);
|
||||
return resultIsHost;
|
||||
};
|
||||
return isHost(value);
|
||||
}
|
||||
|
||||
bool onnx_mlir::pim::isDeviceLocalPimAddress(Value value, const StaticValueKnowledge& knowledge) {
|
||||
auto base = getPimStorageBase(value, knowledge);
|
||||
if (failed(base))
|
||||
return false;
|
||||
|
||||
return isa_and_nonnull<memref::AllocOp>(base->getDefiningOp());
|
||||
llvm::SmallPtrSet<Value, 8> visited;
|
||||
std::function<bool(Value)> isDevice = [&](Value current) {
|
||||
auto base = getPimStorageBase(current, knowledge);
|
||||
if (failed(base) || !visited.insert(*base).second)
|
||||
return false;
|
||||
bool resultIsDevice = isa_and_nonnull<memref::AllocOp>(base->getDefiningOp());
|
||||
auto result = dyn_cast<OpResult>(*base);
|
||||
SmallVector<Region *> regions = result ? getSelectionRegions(result)
|
||||
: SmallVector<Region *>();
|
||||
if (!resultIsDevice && !regions.empty())
|
||||
resultIsDevice = llvm::all_of(regions, [&](Region *region) {
|
||||
auto yield = dyn_cast<scf::YieldOp>(region->front().getTerminator());
|
||||
return yield && result.getResultNumber() < yield.getNumOperands()
|
||||
&& isDevice(yield.getOperand(result.getResultNumber()));
|
||||
});
|
||||
visited.erase(*base);
|
||||
return resultIsDevice;
|
||||
};
|
||||
return isDevice(value);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
#include "mlir/Dialect/MemRef/IR/MemRef.h"
|
||||
#include "mlir/Dialect/SCF/IR/SCF.h"
|
||||
|
||||
#include "llvm/Support/MathExtras.h"
|
||||
|
||||
#include "ContiguityPatterns.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/LoopUtils.hpp"
|
||||
@@ -33,6 +35,7 @@ struct CopyEndpointPlan {
|
||||
|
||||
struct CopyLoopPlan {
|
||||
SmallVector<int64_t> outerShape;
|
||||
int64_t outerElements = 0;
|
||||
int64_t chunkBytes = 0;
|
||||
ByteOffsetExpr targetBaseOffset;
|
||||
ByteOffsetExpr sourceBaseOffset;
|
||||
@@ -74,6 +77,24 @@ static void appendTerm(ByteOffsetExpr& expr, Value value, int64_t scale) {
|
||||
expr.terms.push_back(ByteOffsetTerm {value, scale});
|
||||
}
|
||||
|
||||
static FailureOr<int64_t> checkedPositiveMul(int64_t lhs, int64_t rhs) {
|
||||
int64_t result = 0;
|
||||
if (lhs < 0 || rhs < 0 || llvm::MulOverflow(lhs, rhs, result))
|
||||
return failure();
|
||||
return result;
|
||||
}
|
||||
|
||||
static FailureOr<int64_t> checkedPositiveProduct(ArrayRef<int64_t> values) {
|
||||
int64_t result = 1;
|
||||
for (int64_t value : values) {
|
||||
auto product = checkedPositiveMul(result, value);
|
||||
if (failed(product))
|
||||
return failure();
|
||||
result = *product;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
static FailureOr<SmallVector<int64_t>> getStaticMemRefStrides(MemRefType type) {
|
||||
SmallVector<int64_t> strides;
|
||||
int64_t offset = 0;
|
||||
@@ -84,6 +105,165 @@ static FailureOr<SmallVector<int64_t>> getStaticMemRefStrides(MemRefType type) {
|
||||
return strides;
|
||||
}
|
||||
|
||||
static FailureOr<SmallVector<int64_t>> getProvenMemRefStrides(Value value) {
|
||||
llvm::SmallPtrSet<Value, 8> visiting;
|
||||
std::function<FailureOr<SmallVector<int64_t>>(Value)> prove =
|
||||
[&](Value current) -> FailureOr<SmallVector<int64_t>> {
|
||||
auto type = dyn_cast<MemRefType>(current.getType());
|
||||
if (!type || !visiting.insert(current).second)
|
||||
return failure();
|
||||
if (auto strides = getStaticMemRefStrides(type); succeeded(strides)) {
|
||||
visiting.erase(current);
|
||||
return strides;
|
||||
}
|
||||
if (auto castOp = current.getDefiningOp<memref::CastOp>()) {
|
||||
auto strides = prove(castOp.getSource());
|
||||
visiting.erase(current);
|
||||
return strides;
|
||||
}
|
||||
if (auto subview = current.getDefiningOp<memref::SubViewOp>()) {
|
||||
auto sourceStrides = prove(subview.getSource());
|
||||
if (failed(sourceStrides) || subview.getSourceType().getRank() != subview.getType().getRank()) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
SmallVector<int64_t> strides;
|
||||
for (auto [sourceStride, viewStride] :
|
||||
llvm::zip_equal(*sourceStrides, subview.getStaticStrides())) {
|
||||
if (ShapedType::isDynamic(viewStride) || viewStride < 0) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
auto stride = checkedPositiveMul(sourceStride, viewStride);
|
||||
if (failed(stride)) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
strides.push_back(*stride);
|
||||
}
|
||||
visiting.erase(current);
|
||||
return strides;
|
||||
}
|
||||
if (auto expand = current.getDefiningOp<memref::ExpandShapeOp>()) {
|
||||
auto sourceStrides = prove(expand.getSrc());
|
||||
auto resultType = dyn_cast<MemRefType>(expand.getResult().getType());
|
||||
auto sourceType = dyn_cast<MemRefType>(expand.getSrc().getType());
|
||||
if (failed(sourceStrides) || !sourceType || !resultType
|
||||
|| !resultType.hasStaticShape()
|
||||
|| sourceStrides->size() != static_cast<size_t>(sourceType.getRank())
|
||||
|| llvm::any_of(resultType.getShape(), [](int64_t dim) {
|
||||
return dim <= 0;
|
||||
})) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
SmallVector<int64_t> strides(resultType.getRank());
|
||||
SmallVector<bool> assigned(resultType.getRank(), false);
|
||||
for (auto [sourceDim, group] :
|
||||
llvm::enumerate(expand.getReassociationIndices())) {
|
||||
if (sourceDim >= sourceStrides->size() || group.empty()) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
int64_t stride = (*sourceStrides)[sourceDim];
|
||||
for (int64_t resultDim : llvm::reverse(group)) {
|
||||
if (resultDim < 0 || resultDim >= resultType.getRank()
|
||||
|| assigned[resultDim]) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
strides[resultDim] = stride;
|
||||
assigned[resultDim] = true;
|
||||
auto nextStride = checkedPositiveMul(
|
||||
stride, resultType.getDimSize(resultDim));
|
||||
if (failed(nextStride)) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
stride = *nextStride;
|
||||
}
|
||||
}
|
||||
if (llvm::is_contained(assigned, false)) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
visiting.erase(current);
|
||||
return strides;
|
||||
}
|
||||
if (auto collapse = current.getDefiningOp<memref::CollapseShapeOp>()) {
|
||||
auto sourceStrides = prove(collapse.getSrc());
|
||||
auto sourceType = dyn_cast<MemRefType>(collapse.getSrc().getType());
|
||||
if (failed(sourceStrides) || !sourceType
|
||||
|| !sourceType.hasStaticShape()
|
||||
|| sourceStrides->size() != static_cast<size_t>(sourceType.getRank())) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
SmallVector<int64_t> strides;
|
||||
for (ArrayRef<int64_t> group : collapse.getReassociationIndices()) {
|
||||
if (group.empty()) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
for (int64_t dim : group)
|
||||
if (dim < 0 || dim >= sourceType.getRank()
|
||||
|| sourceType.getDimSize(dim) <= 0
|
||||
|| (*sourceStrides)[dim] < 0) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
for (auto pair : llvm::zip(group.drop_back(), group.drop_front())) {
|
||||
int64_t outer = std::get<0>(pair);
|
||||
int64_t inner = std::get<1>(pair);
|
||||
auto expectedOuterStride = checkedPositiveMul(
|
||||
(*sourceStrides)[inner], sourceType.getDimSize(inner));
|
||||
if (failed(expectedOuterStride)
|
||||
|| (*sourceStrides)[outer] != *expectedOuterStride) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
}
|
||||
strides.push_back((*sourceStrides)[group.back()]);
|
||||
}
|
||||
visiting.erase(current);
|
||||
return strides;
|
||||
}
|
||||
auto result = dyn_cast<OpResult>(current);
|
||||
SmallVector<Region *> regions;
|
||||
if (result) {
|
||||
if (auto selection = dyn_cast<scf::IndexSwitchOp>(result.getOwner()))
|
||||
for (Region ®ion : selection->getRegions())
|
||||
regions.push_back(®ion);
|
||||
else if (auto selection = dyn_cast<scf::IfOp>(result.getOwner())) {
|
||||
regions.push_back(&selection.getThenRegion());
|
||||
regions.push_back(&selection.getElseRegion());
|
||||
}
|
||||
}
|
||||
if (regions.empty()) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
std::optional<SmallVector<int64_t>> common;
|
||||
for (Region *region : regions) {
|
||||
auto yield = dyn_cast<scf::YieldOp>(region->front().getTerminator());
|
||||
if (!yield || result.getResultNumber() >= yield.getNumOperands()) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
auto strides = prove(yield.getOperand(result.getResultNumber()));
|
||||
if (failed(strides) || (common && *common != *strides)) {
|
||||
visiting.erase(current);
|
||||
return failure();
|
||||
}
|
||||
common = std::move(*strides);
|
||||
}
|
||||
visiting.erase(current);
|
||||
return common ? FailureOr<SmallVector<int64_t>>(std::move(*common))
|
||||
: FailureOr<SmallVector<int64_t>>(failure());
|
||||
};
|
||||
return prove(value);
|
||||
}
|
||||
|
||||
static FailureOr<int64_t> getShapedByteSize(MemRefType type) {
|
||||
if (!type.hasStaticShape() || !hasByteSizedElementType(type.getElementType()))
|
||||
return failure();
|
||||
@@ -119,12 +299,15 @@ inferLogicalCopyShape(MemRefType targetType, MemRefType sourceType, int64_t size
|
||||
return failure();
|
||||
}
|
||||
|
||||
static FailureOr<int64_t> getContiguousSuffixRank(MemRefType type, ArrayRef<int64_t> copyShape) {
|
||||
if (!type.hasStaticShape() || !hasByteSizedElementType(type.getElementType())
|
||||
static FailureOr<int64_t> getContiguousSuffixRank(Value value, ArrayRef<int64_t> copyShape) {
|
||||
auto type = dyn_cast<MemRefType>(value.getType());
|
||||
if (!type || !type.hasStaticShape() || !hasByteSizedElementType(type.getElementType())
|
||||
|| type.getRank() != static_cast<int64_t>(copyShape.size()))
|
||||
return failure();
|
||||
if (llvm::any_of(copyShape, [](int64_t dim) { return dim <= 0; }))
|
||||
return failure();
|
||||
|
||||
auto strides = getStaticMemRefStrides(type);
|
||||
auto strides = getProvenMemRefStrides(value);
|
||||
if (failed(strides))
|
||||
return failure();
|
||||
|
||||
@@ -134,7 +317,10 @@ static FailureOr<int64_t> getContiguousSuffixRank(MemRefType type, ArrayRef<int6
|
||||
if ((*strides)[dim] != expectedStride)
|
||||
break;
|
||||
++contiguousSuffixRank;
|
||||
expectedStride *= copyShape[dim];
|
||||
auto nextStride = checkedPositiveMul(expectedStride, copyShape[dim]);
|
||||
if (failed(nextStride))
|
||||
return failure();
|
||||
expectedStride = *nextStride;
|
||||
}
|
||||
return contiguousSuffixRank;
|
||||
}
|
||||
@@ -174,18 +360,25 @@ static FailureOr<CopyEndpointPlan> analyzeCopyEndpoint(Value value, Value initia
|
||||
if (!sourceType || !sourceType.hasStaticShape() || !hasByteSizedElementType(sourceType.getElementType()))
|
||||
return failure();
|
||||
|
||||
auto sourceStrides = getStaticMemRefStrides(sourceType);
|
||||
auto sourceStrides = getProvenMemRefStrides(subviewOp.getSource());
|
||||
if (failed(sourceStrides))
|
||||
return failure();
|
||||
|
||||
int64_t elementByteWidth = static_cast<int64_t>(getElementTypeSizeInBytes(sourceType.getElementType()));
|
||||
for (auto [offset, stride] : llvm::zip_equal(subviewOp.getMixedOffsets(), *sourceStrides)) {
|
||||
int64_t byteScale = stride * elementByteWidth;
|
||||
auto byteScale = checkedPositiveMul(stride, elementByteWidth);
|
||||
if (failed(byteScale))
|
||||
return failure();
|
||||
if (auto attr = dyn_cast<Attribute>(offset)) {
|
||||
endpoint.offset.constant += cast<IntegerAttr>(attr).getInt() * byteScale;
|
||||
auto constantOffset = checkedPositiveMul(
|
||||
cast<IntegerAttr>(attr).getInt(), *byteScale);
|
||||
if (failed(constantOffset)
|
||||
|| llvm::AddOverflow(endpoint.offset.constant, *constantOffset,
|
||||
endpoint.offset.constant))
|
||||
return failure();
|
||||
continue;
|
||||
}
|
||||
appendTerm(endpoint.offset, cast<Value>(offset), byteScale);
|
||||
appendTerm(endpoint.offset, cast<Value>(offset), *byteScale);
|
||||
}
|
||||
|
||||
endpoint.base = subviewOp.getSource();
|
||||
@@ -213,8 +406,8 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO
|
||||
auto sourceBytes = getShapedByteSize(sourceType);
|
||||
if (targetType.getElementType() == sourceType.getElementType() && succeeded(targetBytes) && succeeded(sourceBytes)
|
||||
&& *targetBytes == size && *sourceBytes == size) {
|
||||
auto targetSuffixRank = getContiguousSuffixRank(targetType, targetType.getShape());
|
||||
auto sourceSuffixRank = getContiguousSuffixRank(sourceType, sourceType.getShape());
|
||||
auto targetSuffixRank = getContiguousSuffixRank(target, targetType.getShape());
|
||||
auto sourceSuffixRank = getContiguousSuffixRank(source, sourceType.getShape());
|
||||
if (succeeded(targetSuffixRank) && succeeded(sourceSuffixRank)
|
||||
&& *targetSuffixRank == targetType.getRank() && *sourceSuffixRank == sourceType.getRank()) {
|
||||
CopyRewritePlan plan;
|
||||
@@ -230,8 +423,8 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO
|
||||
if (failed(logicalCopyShape))
|
||||
return failure();
|
||||
|
||||
auto targetSuffixRank = getContiguousSuffixRank(targetType, *logicalCopyShape);
|
||||
auto sourceSuffixRank = getContiguousSuffixRank(sourceType, *logicalCopyShape);
|
||||
auto targetSuffixRank = getContiguousSuffixRank(target, *logicalCopyShape);
|
||||
auto sourceSuffixRank = getContiguousSuffixRank(source, *logicalCopyShape);
|
||||
if (failed(targetSuffixRank) || failed(sourceSuffixRank))
|
||||
return failure();
|
||||
|
||||
@@ -246,8 +439,8 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO
|
||||
return plan;
|
||||
}
|
||||
|
||||
auto targetStrides = getStaticMemRefStrides(targetType);
|
||||
auto sourceStrides = getStaticMemRefStrides(sourceType);
|
||||
auto targetStrides = getProvenMemRefStrides(target);
|
||||
auto sourceStrides = getProvenMemRefStrides(source);
|
||||
if (failed(targetStrides) || failed(sourceStrides))
|
||||
return failure();
|
||||
|
||||
@@ -257,11 +450,27 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO
|
||||
plan.loop.sourceBaseOffset = plan.source.offset;
|
||||
plan.loop.outerShape.assign(logicalCopyShape->begin(), logicalCopyShape->end() - contiguousSuffixRank);
|
||||
SmallVector<int64_t> chunkShape(logicalCopyShape->end() - contiguousSuffixRank, logicalCopyShape->end());
|
||||
plan.loop.chunkBytes = getNumElements(chunkShape) * elementByteWidth;
|
||||
for (int64_t stride : ArrayRef<int64_t>(*targetStrides).take_front(plan.loop.outerShape.size()))
|
||||
plan.loop.targetOuterByteStrides.push_back(stride * elementByteWidth);
|
||||
for (int64_t stride : ArrayRef<int64_t>(*sourceStrides).take_front(plan.loop.outerShape.size()))
|
||||
plan.loop.sourceOuterByteStrides.push_back(stride * elementByteWidth);
|
||||
auto outerElements = checkedPositiveProduct(plan.loop.outerShape);
|
||||
auto chunkElements = checkedPositiveProduct(chunkShape);
|
||||
auto chunkBytes = failed(chunkElements)
|
||||
? FailureOr<int64_t>(failure())
|
||||
: checkedPositiveMul(*chunkElements, elementByteWidth);
|
||||
if (failed(outerElements) || failed(chunkBytes))
|
||||
return failure();
|
||||
plan.loop.outerElements = *outerElements;
|
||||
plan.loop.chunkBytes = *chunkBytes;
|
||||
for (int64_t stride : ArrayRef<int64_t>(*targetStrides).take_front(plan.loop.outerShape.size())) {
|
||||
auto byteStride = checkedPositiveMul(stride, elementByteWidth);
|
||||
if (failed(byteStride))
|
||||
return failure();
|
||||
plan.loop.targetOuterByteStrides.push_back(*byteStride);
|
||||
}
|
||||
for (int64_t stride : ArrayRef<int64_t>(*sourceStrides).take_front(plan.loop.outerShape.size())) {
|
||||
auto byteStride = checkedPositiveMul(stride, elementByteWidth);
|
||||
if (failed(byteStride))
|
||||
return failure();
|
||||
plan.loop.sourceOuterByteStrides.push_back(*byteStride);
|
||||
}
|
||||
if (plan.loop.chunkBytes <= 0)
|
||||
return failure();
|
||||
return plan;
|
||||
@@ -361,7 +570,7 @@ static LogicalResult rewriteCopyLikeOp(CopyOp copyOp,
|
||||
}
|
||||
|
||||
Value c0 = createIndexConstant(rewriter, anchorOp, 0);
|
||||
Value cUpper = createIndexConstant(rewriter, anchorOp, getNumElements(plan->loop.outerShape));
|
||||
Value cUpper = createIndexConstant(rewriter, anchorOp, plan->loop.outerElements);
|
||||
Value cStep = createIndexConstant(rewriter, anchorOp, 1);
|
||||
auto loop = buildNormalizedScfFor(
|
||||
rewriter,
|
||||
|
||||
@@ -17,6 +17,7 @@ add_pim_library(SpatialOps
|
||||
Transforms/MergeComputeNodes/ScheduledComputeMaterialization.cpp
|
||||
Transforms/MergeComputeNodes/ScheduledComputeReport.cpp
|
||||
Transforms/MergeComputeNodes/ScheduledComputeVerification.cpp
|
||||
Transforms/MergeComputeNodes/SpatialDataflowCsvExporter.cpp
|
||||
Transforms/MergeComputeNodes/Scheduling/MergeSchedulingAnalysis.cpp
|
||||
Transforms/MergeComputeNodes/Scheduling/PeftScheduler.cpp
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include "llvm/Support/LogicalResult.h"
|
||||
|
||||
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
|
||||
using namespace mlir;
|
||||
@@ -130,7 +131,7 @@ struct CanonicalizeSingleLaneComputeBatchPattern : OpRewritePattern<ComputeBatch
|
||||
rewriter.setInsertionPointToStart(newBlock);
|
||||
|
||||
IRMapping mapper;
|
||||
Value zero = arith::ConstantIndexOp::create(rewriter, compute.getLoc(), 0);
|
||||
Value zero = getOrCreateIndexConstant(rewriter, compute.getOperation(), 0);
|
||||
mapper.map(*oldLaneArg, zero);
|
||||
for (auto [index, weight] : llvm::enumerate(compute.getWeights())) {
|
||||
auto oldArg = compute.getWeightArgument(index);
|
||||
|
||||
+219
-157
@@ -4,7 +4,6 @@
|
||||
|
||||
#include "llvm/ADT/DenseMap.h"
|
||||
#include "llvm/ADT/DenseSet.h"
|
||||
#include "llvm/ADT/DenseSet.h"
|
||||
|
||||
namespace onnx_mlir::spatial {
|
||||
|
||||
@@ -23,54 +22,73 @@ static LogicalResult simulate(Operation *anchor,
|
||||
ArrayRef<SmallVector<Event>> streams,
|
||||
StringRef phase) {
|
||||
SmallVector<size_t> cursor(streams.size());
|
||||
while (true) {
|
||||
bool allFinished = true;
|
||||
bool progressed = false;
|
||||
for (unsigned stream = 0; stream < streams.size(); ++stream) {
|
||||
if (cursor[stream] == streams[stream].size())
|
||||
continue;
|
||||
allFinished = false;
|
||||
if (streams[stream][cursor[stream]].kind == EventKind::Compute) {
|
||||
++cursor[stream];
|
||||
progressed = true;
|
||||
}
|
||||
}
|
||||
if (allFinished)
|
||||
return success();
|
||||
DenseMap<uint64_t, unsigned> headSends;
|
||||
DenseMap<uint64_t, unsigned> headReceives;
|
||||
SmallVector<unsigned> readyComputes;
|
||||
SmallVector<uint64_t> readyExchanges;
|
||||
size_t computeCursor = 0;
|
||||
size_t exchangeCursor = 0;
|
||||
unsigned finishedStreams = 0;
|
||||
|
||||
for (unsigned source = 0; source < streams.size(); ++source) {
|
||||
if (cursor[source] == streams[source].size())
|
||||
continue;
|
||||
const Event &send = streams[source][cursor[source]];
|
||||
if (send.kind != EventKind::Send)
|
||||
continue;
|
||||
for (unsigned target = 0; target < streams.size(); ++target) {
|
||||
if (cursor[target] == streams[target].size())
|
||||
continue;
|
||||
const Event &receive = streams[target][cursor[target]];
|
||||
if (receive.kind == EventKind::Receive && receive.exchangeId == send.exchangeId) {
|
||||
++cursor[source];
|
||||
++cursor[target];
|
||||
progressed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto registerHead = [&](unsigned stream) {
|
||||
if (cursor[stream] == streams[stream].size()) {
|
||||
++finishedStreams;
|
||||
return;
|
||||
}
|
||||
if (!progressed) {
|
||||
InFlightDiagnostic diagnostic = anchor->emitError()
|
||||
<< phase << " communication rendezvous simulation made no progress";
|
||||
unsigned reported = 0;
|
||||
for (unsigned stream = 0; stream < streams.size() && reported < 8; ++stream) {
|
||||
if (cursor[stream] == streams[stream].size())
|
||||
continue;
|
||||
const Event &event = streams[stream][cursor[stream]];
|
||||
diagnostic << (reported == 0 ? "; blocked " : ", ") << "stream " << stream
|
||||
<< " at exchange " << event.exchangeId;
|
||||
++reported;
|
||||
}
|
||||
return failure();
|
||||
const Event &event = streams[stream][cursor[stream]];
|
||||
if (event.kind == EventKind::Compute) {
|
||||
readyComputes.push_back(stream);
|
||||
return;
|
||||
}
|
||||
DenseMap<uint64_t, unsigned> &heads =
|
||||
event.kind == EventKind::Send ? headSends : headReceives;
|
||||
DenseMap<uint64_t, unsigned> &peers =
|
||||
event.kind == EventKind::Send ? headReceives : headSends;
|
||||
heads[event.exchangeId] = stream;
|
||||
if (peers.contains(event.exchangeId))
|
||||
readyExchanges.push_back(event.exchangeId);
|
||||
};
|
||||
for (unsigned stream = 0; stream < streams.size(); ++stream)
|
||||
registerHead(stream);
|
||||
|
||||
while (computeCursor != readyComputes.size()
|
||||
|| exchangeCursor != readyExchanges.size()) {
|
||||
if (computeCursor != readyComputes.size()) {
|
||||
unsigned stream = readyComputes[computeCursor++];
|
||||
++cursor[stream];
|
||||
registerHead(stream);
|
||||
continue;
|
||||
}
|
||||
|
||||
uint64_t exchange = readyExchanges[exchangeCursor++];
|
||||
auto send = headSends.find(exchange);
|
||||
auto receive = headReceives.find(exchange);
|
||||
if (send == headSends.end() || receive == headReceives.end())
|
||||
continue;
|
||||
unsigned source = send->second;
|
||||
unsigned target = receive->second;
|
||||
headSends.erase(send);
|
||||
headReceives.erase(receive);
|
||||
++cursor[source];
|
||||
++cursor[target];
|
||||
registerHead(source);
|
||||
registerHead(target);
|
||||
}
|
||||
|
||||
if (finishedStreams == streams.size())
|
||||
return success();
|
||||
InFlightDiagnostic diagnostic = anchor->emitError()
|
||||
<< phase << " communication rendezvous simulation made no progress";
|
||||
unsigned reported = 0;
|
||||
for (unsigned stream = 0; stream < streams.size() && reported < 8; ++stream) {
|
||||
if (cursor[stream] == streams[stream].size())
|
||||
continue;
|
||||
const Event &event = streams[stream][cursor[stream]];
|
||||
diagnostic << (reported == 0 ? "; blocked " : ", ") << "stream " << stream
|
||||
<< " at exchange " << event.exchangeId;
|
||||
++reported;
|
||||
}
|
||||
return failure();
|
||||
}
|
||||
|
||||
static std::optional<int64_t> getI64Attr(Operation *op, StringRef name) {
|
||||
@@ -79,6 +97,98 @@ static std::optional<int64_t> getI64Attr(Operation *op, StringRef name) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
static LogicalResult getI64ArrayAttr(
|
||||
Operation *op, StringRef name,
|
||||
std::optional<SmallVector<int64_t>> &values) {
|
||||
Attribute attr = op->getAttr(name);
|
||||
if (!attr)
|
||||
return success();
|
||||
if (auto array = dyn_cast<DenseI64ArrayAttr>(attr)) {
|
||||
values.emplace(array.asArrayRef());
|
||||
return success();
|
||||
}
|
||||
auto elements = dyn_cast<DenseIntElementsAttr>(attr);
|
||||
auto type = elements ? dyn_cast<RankedTensorType>(elements.getType())
|
||||
: RankedTensorType();
|
||||
if (!elements || !type || type.getRank() != 1
|
||||
|| !type.getElementType().isInteger(64))
|
||||
return op->emitOpError() << "has invalid " << name << " metadata";
|
||||
values.emplace();
|
||||
values->reserve(elements.getNumElements());
|
||||
for (const APInt &value : elements.getValues<APInt>())
|
||||
values->push_back(value.getSExtValue());
|
||||
return success();
|
||||
}
|
||||
|
||||
struct RealizedLogicalTransfer {
|
||||
int64_t channelId = -1;
|
||||
int64_t parentExchangeId = -1;
|
||||
int64_t parentTransferCount = 0;
|
||||
int64_t sourceCore = -1;
|
||||
int64_t targetCore = -1;
|
||||
};
|
||||
|
||||
static LogicalResult forEachRealizedLogicalTransfer(
|
||||
Operation *op,
|
||||
function_ref<LogicalResult(const RealizedLogicalTransfer &)> callback) {
|
||||
auto scalarChannel = getI64Attr(op, "raptor.channel_id");
|
||||
std::optional<SmallVector<int64_t>> batchChannels;
|
||||
if (failed(getI64ArrayAttr(
|
||||
op, "raptor.batch_channel_ids", batchChannels)))
|
||||
return failure();
|
||||
if (scalarChannel && batchChannels)
|
||||
return op->emitOpError(
|
||||
"mixes scalar and compact logical transfer metadata");
|
||||
|
||||
if (scalarChannel) {
|
||||
auto exchange = getI64Attr(op, "raptor.exchange_id");
|
||||
auto parent = getI64Attr(op, "raptor.parent_exchange_id");
|
||||
auto count = getI64Attr(op, "raptor.parent_transfer_count");
|
||||
auto source = getI64Attr(op, "raptor.source_core");
|
||||
auto target = getI64Attr(op, "raptor.target_core");
|
||||
if (!exchange || !parent || !count || !source || !target)
|
||||
return op->emitOpError(
|
||||
"is missing scalar logical transfer metadata");
|
||||
RealizedLogicalTransfer transfer {
|
||||
*scalarChannel, *parent, *count, *source, *target};
|
||||
if (*exchange != transfer.channelId || transfer.channelId < 0
|
||||
|| transfer.parentExchangeId < 0 || transfer.parentTransferCount <= 0
|
||||
|| transfer.sourceCore < 0 || transfer.targetCore < 0)
|
||||
return op->emitOpError("has invalid scalar logical transfer metadata");
|
||||
return callback(transfer);
|
||||
}
|
||||
|
||||
std::optional<SmallVector<int64_t>> sources, targets, parents, counts;
|
||||
if (failed(getI64ArrayAttr(op, "raptor.batch_source_cores", sources))
|
||||
|| failed(getI64ArrayAttr(op, "raptor.batch_target_cores", targets))
|
||||
|| failed(getI64ArrayAttr(
|
||||
op, "raptor.batch_parent_exchange_ids", parents))
|
||||
|| failed(getI64ArrayAttr(
|
||||
op, "raptor.batch_parent_transfer_counts", counts)))
|
||||
return failure();
|
||||
if (!batchChannels || !sources || !targets || !parents || !counts)
|
||||
return op->emitOpError(
|
||||
"is missing compact logical transfer metadata");
|
||||
size_t size = batchChannels->size();
|
||||
if (size == 0 || sources->size() != size || targets->size() != size
|
||||
|| parents->size() != size || counts->size() != size)
|
||||
return op->emitOpError(
|
||||
"has non-parallel compact logical transfer metadata");
|
||||
for (auto values : llvm::zip_equal(
|
||||
*batchChannels, *parents, *counts, *sources, *targets)) {
|
||||
RealizedLogicalTransfer transfer {
|
||||
std::get<0>(values), std::get<1>(values), std::get<2>(values),
|
||||
std::get<3>(values), std::get<4>(values)};
|
||||
if (transfer.channelId < 0 || transfer.parentExchangeId < 0
|
||||
|| transfer.parentTransferCount <= 0 || transfer.sourceCore < 0
|
||||
|| transfer.targetCore < 0)
|
||||
return op->emitOpError("has invalid compact logical transfer metadata");
|
||||
if (failed(callback(transfer)))
|
||||
return failure();
|
||||
}
|
||||
return success();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
LogicalResult verifyPlannedCommunicationDeadlockFree(
|
||||
@@ -118,7 +228,11 @@ LogicalResult verifyPlannedCommunicationDeadlockFree(
|
||||
}
|
||||
|
||||
LogicalResult verifyRealizedCommunicationDeadlockFree(func::FuncOp funcOp) {
|
||||
DenseMap<int64_t, SmallVector<Operation *, 2>> operationsByExchange;
|
||||
struct LogicalOperation {
|
||||
Operation *op = nullptr;
|
||||
RealizedLogicalTransfer transfer;
|
||||
};
|
||||
DenseMap<int64_t, SmallVector<LogicalOperation, 2>> operationsByExchange;
|
||||
struct ParentExchange {
|
||||
std::optional<int64_t> expectedTransfers;
|
||||
DenseSet<int64_t> channels;
|
||||
@@ -130,39 +244,25 @@ LogicalResult verifyRealizedCommunicationDeadlockFree(func::FuncOp funcOp) {
|
||||
funcOp.walk([&](Operation *op) {
|
||||
if (!isa<SpatChannelSendOp, SpatChannelReceiveOp>(op))
|
||||
return;
|
||||
std::optional<int64_t> exchangeId = getI64Attr(op, "raptor.exchange_id");
|
||||
if (exchangeId)
|
||||
operationsByExchange[*exchangeId].push_back(op);
|
||||
if (auto channels = op->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_channel_ids"))
|
||||
for (int64_t channel : channels.asArrayRef())
|
||||
operationsByExchange[channel].push_back(op);
|
||||
if (std::optional<int64_t> parent = getI64Attr(op, "raptor.parent_exchange_id")) {
|
||||
ParentExchange &group = parentExchanges[*parent];
|
||||
std::optional<int64_t> expected =
|
||||
getI64Attr(op, "raptor.parent_transfer_count");
|
||||
if (!expected || *expected <= 0
|
||||
|| (group.expectedTransfers && group.expectedTransfers != expected)) {
|
||||
op->emitOpError(
|
||||
"realized parent exchange has missing or inconsistent transfer count metadata");
|
||||
invalid = true;
|
||||
} else {
|
||||
group.expectedTransfers = expected;
|
||||
}
|
||||
if (exchangeId)
|
||||
group.channels.insert(*exchangeId);
|
||||
if (auto channels =
|
||||
op->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_channel_ids"))
|
||||
group.channels.insert(channels.asArrayRef().begin(),
|
||||
channels.asArrayRef().end());
|
||||
}
|
||||
for (StringRef attrName : {"raptor.source_core", "raptor.target_core"})
|
||||
if (std::optional<int64_t> core = getI64Attr(op, attrName); core && !llvm::is_contained(cores, *core))
|
||||
cores.push_back(*core);
|
||||
for (StringRef attrName : {"raptor.batch_source_cores", "raptor.batch_target_cores"})
|
||||
if (auto batchCores = op->getAttrOfType<DenseI64ArrayAttr>(attrName))
|
||||
for (int64_t core : batchCores.asArrayRef())
|
||||
if (!llvm::is_contained(cores, core))
|
||||
cores.push_back(core);
|
||||
if (failed(forEachRealizedLogicalTransfer(
|
||||
op, [&](const RealizedLogicalTransfer &transfer) -> LogicalResult {
|
||||
operationsByExchange[transfer.channelId].push_back(
|
||||
{op, transfer});
|
||||
ParentExchange &parent =
|
||||
parentExchanges[transfer.parentExchangeId];
|
||||
if (parent.expectedTransfers
|
||||
&& *parent.expectedTransfers
|
||||
!= transfer.parentTransferCount)
|
||||
return op->emitOpError(
|
||||
"declares an inconsistent parent transfer count");
|
||||
parent.expectedTransfers = transfer.parentTransferCount;
|
||||
parent.channels.insert(transfer.channelId);
|
||||
for (int64_t core : {transfer.sourceCore, transfer.targetCore})
|
||||
if (!llvm::is_contained(cores, core))
|
||||
cores.push_back(core);
|
||||
return success();
|
||||
})))
|
||||
invalid = true;
|
||||
});
|
||||
llvm::sort(cores);
|
||||
for (auto [index, core] : llvm::enumerate(cores))
|
||||
@@ -172,38 +272,18 @@ LogicalResult verifyRealizedCommunicationDeadlockFree(func::FuncOp funcOp) {
|
||||
funcOp.walk([&](Operation *op) {
|
||||
if (!isa<SpatChannelSendOp, SpatChannelReceiveOp>(op))
|
||||
return;
|
||||
auto exchangeId = getI64Attr(op, "raptor.exchange_id");
|
||||
if (!exchangeId) {
|
||||
auto channels = op->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_channel_ids");
|
||||
auto sourceCores = op->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_source_cores");
|
||||
auto targetCores = op->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_target_cores");
|
||||
if (!channels || !sourceCores || !targetCores
|
||||
|| channels.size() != sourceCores.size() || channels.size() != targetCores.size()) {
|
||||
op->emitOpError("realized compact batch communication is missing channel/core metadata");
|
||||
invalid = true;
|
||||
return;
|
||||
}
|
||||
if (isa<SpatChannelSendOp>(op))
|
||||
for (auto [channel, sourceCore] : llvm::zip(channels.asArrayRef(), sourceCores.asArrayRef()))
|
||||
streams[streamByCore.lookup(sourceCore)].push_back(
|
||||
{EventKind::Send, static_cast<uint64_t>(channel)});
|
||||
else
|
||||
for (auto [channel, targetCore] : llvm::zip(channels.asArrayRef(), targetCores.asArrayRef()))
|
||||
streams[streamByCore.lookup(targetCore)].push_back(
|
||||
{EventKind::Receive, static_cast<uint64_t>(channel)});
|
||||
return;
|
||||
}
|
||||
auto sourceCore = getI64Attr(op, "raptor.source_core");
|
||||
auto targetCore = getI64Attr(op, "raptor.target_core");
|
||||
if (!sourceCore || !targetCore) {
|
||||
op->emitOpError("realized communication is missing core metadata");
|
||||
if (failed(forEachRealizedLogicalTransfer(
|
||||
op, [&](const RealizedLogicalTransfer &transfer) {
|
||||
unsigned stream = streamByCore.lookup(
|
||||
isa<SpatChannelSendOp>(op) ? transfer.sourceCore
|
||||
: transfer.targetCore);
|
||||
streams[stream].push_back(
|
||||
{isa<SpatChannelSendOp>(op) ? EventKind::Send
|
||||
: EventKind::Receive,
|
||||
static_cast<uint64_t>(transfer.channelId)});
|
||||
return success();
|
||||
})))
|
||||
invalid = true;
|
||||
return;
|
||||
}
|
||||
if (isa<SpatChannelSendOp>(op))
|
||||
streams[streamByCore.lookup(*sourceCore)].push_back({EventKind::Send, static_cast<uint64_t>(*exchangeId)});
|
||||
else if (isa<SpatChannelReceiveOp>(op))
|
||||
streams[streamByCore.lookup(*targetCore)].push_back({EventKind::Receive, static_cast<uint64_t>(*exchangeId)});
|
||||
});
|
||||
if (invalid)
|
||||
return failure();
|
||||
@@ -216,56 +296,38 @@ LogicalResult verifyRealizedCommunicationDeadlockFree(func::FuncOp funcOp) {
|
||||
<< " does not contain its declared lane transfer set";
|
||||
|
||||
for (const auto &entry : operationsByExchange) {
|
||||
if (entry.second.size() != 2 || !isa<SpatChannelSendOp>(entry.second[0])
|
||||
== !isa<SpatChannelSendOp>(entry.second[1]))
|
||||
return funcOp.emitOpError() << "exchange " << entry.first << " does not have exactly one send and one receive";
|
||||
auto send = dyn_cast<SpatChannelSendOp>(entry.second[0]);
|
||||
auto receive = dyn_cast<SpatChannelReceiveOp>(entry.second[1]);
|
||||
if (!send) {
|
||||
send = cast<SpatChannelSendOp>(entry.second[1]);
|
||||
receive = cast<SpatChannelReceiveOp>(entry.second[0]);
|
||||
if (entry.second.size() != 2
|
||||
|| isa<SpatChannelSendOp>(entry.second[0].op)
|
||||
== isa<SpatChannelSendOp>(entry.second[1].op)) {
|
||||
return funcOp.emitOpError()
|
||||
<< "exchange " << entry.first
|
||||
<< " does not have exactly one send and one receive (sends="
|
||||
<< llvm::count_if(entry.second, [](const LogicalOperation &item) {
|
||||
return isa<SpatChannelSendOp>(item.op);
|
||||
})
|
||||
<< ", receives="
|
||||
<< llvm::count_if(entry.second, [](const LogicalOperation &item) {
|
||||
return isa<SpatChannelReceiveOp>(item.op);
|
||||
})
|
||||
<< ")";
|
||||
}
|
||||
const LogicalOperation &first = entry.second[0];
|
||||
const LogicalOperation &second = entry.second[1];
|
||||
const LogicalOperation &sendRecord =
|
||||
isa<SpatChannelSendOp>(first.op) ? first : second;
|
||||
const LogicalOperation &receiveRecord =
|
||||
isa<SpatChannelReceiveOp>(first.op) ? first : second;
|
||||
auto send = cast<SpatChannelSendOp>(sendRecord.op);
|
||||
auto receive = cast<SpatChannelReceiveOp>(receiveRecord.op);
|
||||
if (send.getInput().getType() != receive.getOutput().getType())
|
||||
return send.emitOpError("send and receive payload types do not match");
|
||||
int64_t sendSource = 0;
|
||||
int64_t sendTarget = 0;
|
||||
if (auto channels = send->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_channel_ids")) {
|
||||
auto channelIt = llvm::find(channels.asArrayRef(), entry.first);
|
||||
auto sources = send->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_source_cores");
|
||||
auto targets = send->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_target_cores");
|
||||
if (channelIt == channels.asArrayRef().end() || !sources || !targets)
|
||||
return send.emitOpError("batch send channel metadata is incomplete");
|
||||
size_t index = std::distance(channels.asArrayRef().begin(), channelIt);
|
||||
sendSource = sources.asArrayRef()[index];
|
||||
sendTarget = targets.asArrayRef()[index];
|
||||
} else {
|
||||
auto source = getI64Attr(send, "raptor.source_core");
|
||||
auto target = getI64Attr(send, "raptor.target_core");
|
||||
if (!source || !target)
|
||||
return send.emitOpError("send core metadata is incomplete");
|
||||
sendSource = *source;
|
||||
sendTarget = *target;
|
||||
}
|
||||
int64_t receiveSource = 0;
|
||||
int64_t receiveTarget = 0;
|
||||
if (auto channels = receive->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_channel_ids")) {
|
||||
auto channelIt = llvm::find(channels.asArrayRef(), entry.first);
|
||||
auto sources = receive->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_source_cores");
|
||||
auto targets = receive->getAttrOfType<DenseI64ArrayAttr>("raptor.batch_target_cores");
|
||||
if (channelIt == channels.asArrayRef().end() || !sources || !targets)
|
||||
return receive.emitOpError("batch receive channel metadata is incomplete");
|
||||
size_t index = std::distance(channels.asArrayRef().begin(), channelIt);
|
||||
receiveSource = sources.asArrayRef()[index];
|
||||
receiveTarget = targets.asArrayRef()[index];
|
||||
} else {
|
||||
auto source = getI64Attr(receive, "raptor.source_core");
|
||||
auto target = getI64Attr(receive, "raptor.target_core");
|
||||
if (!source || !target)
|
||||
return receive.emitOpError("receive core metadata is incomplete");
|
||||
receiveSource = *source;
|
||||
receiveTarget = *target;
|
||||
}
|
||||
if (receiveSource != sendSource || receiveTarget != sendTarget)
|
||||
if (receiveRecord.transfer.sourceCore != sendRecord.transfer.sourceCore
|
||||
|| receiveRecord.transfer.targetCore
|
||||
!= sendRecord.transfer.targetCore
|
||||
|| receiveRecord.transfer.parentExchangeId
|
||||
!= sendRecord.transfer.parentExchangeId
|
||||
|| receiveRecord.transfer.parentTransferCount
|
||||
!= sendRecord.transfer.parentTransferCount)
|
||||
return receive.emitOpError("receive core metadata does not match its send");
|
||||
}
|
||||
return simulate(funcOp, streams, "realized");
|
||||
|
||||
+84
-113
@@ -8,6 +8,10 @@
|
||||
|
||||
#include "llvm/ADT/SmallPtrSet.h"
|
||||
|
||||
#include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/CompileTime.hpp"
|
||||
|
||||
namespace onnx_mlir::spatial {
|
||||
using namespace mlir;
|
||||
namespace {
|
||||
@@ -112,11 +116,34 @@ static FailureOr<Value> buildBlueprintReconstruction(
|
||||
return result;
|
||||
}
|
||||
|
||||
static bool isSupportedDeferredShapingOp(Operation *op) {
|
||||
return isa<tensor::ExtractSliceOp, tensor::InsertSliceOp, tensor::CollapseShapeOp,
|
||||
tensor::ExpandShapeOp, tensor::CastOp, tensor::EmptyOp, tensor::ExtractOp,
|
||||
arith::ConstantOp, arith::IndexCastOp, arith::AddIOp, arith::SubIOp,
|
||||
arith::MulIOp, affine::AffineApplyOp>(op);
|
||||
static FailureOr<Value> buildIndexSwitchSelection(OpBuilder &builder, Location loc,
|
||||
Value selector, ValueRange candidates,
|
||||
Operation *diagnosticOwner) {
|
||||
if (candidates.empty())
|
||||
return diagnosticOwner->emitOpError("direct selection requires at least one candidate"), failure();
|
||||
Type type = candidates.front().getType();
|
||||
if (llvm::any_of(candidates, [&](Value candidate) { return candidate.getType() != type; }))
|
||||
return diagnosticOwner->emitOpError("direct selection requires identical candidate types"), failure();
|
||||
if (candidates.size() == 1)
|
||||
return candidates.front();
|
||||
|
||||
SmallVector<int64_t> cases;
|
||||
for (int64_t index = 0; index < static_cast<int64_t>(candidates.size()) - 1; ++index)
|
||||
cases.push_back(index);
|
||||
auto selection = scf::IndexSwitchOp::create(
|
||||
builder, loc, TypeRange {type}, selector, cases, cases.size());
|
||||
auto buildYield = [&](Region ®ion, Value candidate) {
|
||||
OpBuilder::InsertionGuard guard(builder);
|
||||
Block *block = builder.createBlock(®ion);
|
||||
builder.setInsertionPointToEnd(block);
|
||||
scf::YieldOp::create(builder, loc, candidate);
|
||||
};
|
||||
for (auto [region, candidate] : llvm::zip(selection.getCaseRegions(), candidates.drop_back()))
|
||||
buildYield(region, candidate);
|
||||
// The scheduled-lane verifier guarantees an in-range selector, so default is
|
||||
// the final lane without an otherwise-unreachable extra branch.
|
||||
buildYield(selection.getDefaultRegion(), candidates.back());
|
||||
return selection.getResult(0);
|
||||
}
|
||||
|
||||
static FailureOr<Value> buildSelectedDeferredSource(OpBuilder &builder, Location loc,
|
||||
@@ -128,46 +155,28 @@ static FailureOr<Value> buildSelectedDeferredSource(OpBuilder &builder, Location
|
||||
return sourceBlockArgs.front();
|
||||
if (!scheduledLane || sourceOperandForScheduledLane.empty())
|
||||
return transfer.emitOpError("multiple deferred sources require the enclosing scheduled lane"), failure();
|
||||
Value table = createI64LookupTableConstant(builder, transfer.getOperation(), sourceOperandForScheduledLane);
|
||||
Value i64 = tensor::ExtractOp::create(builder, loc, table, ValueRange {scheduledLane}).getResult();
|
||||
Value index = arith::IndexCastOp::create(builder, loc, builder.getIndexType(), i64).getResult();
|
||||
auto type = dyn_cast<RankedTensorType>(sourceBlockArgs.front().getType());
|
||||
if (!type || !type.hasStaticShape())
|
||||
return transfer.emitOpError("multiple deferred sources require static ranked tensors"), failure();
|
||||
for (Value source : sourceBlockArgs)
|
||||
if (source.getType() != type)
|
||||
return transfer.emitOpError("multiple deferred sources require identical tensor types"), failure();
|
||||
SmallVector<int64_t> shape {static_cast<int64_t>(sourceBlockArgs.size())};
|
||||
llvm::append_range(shape, type.getShape());
|
||||
auto stacked = createEmptyTensorForType(builder, loc, RankedTensorType::get(shape, type.getElementType()));
|
||||
if (failed(stacked))
|
||||
return failure();
|
||||
Value value = *stacked;
|
||||
SmallVector<OpFoldResult> sizes, strides(shape.size(), builder.getIndexAttr(1));
|
||||
sizes.push_back(builder.getIndexAttr(1));
|
||||
for (int64_t dim : type.getShape()) sizes.push_back(builder.getIndexAttr(dim));
|
||||
for (auto [i, source] : llvm::enumerate(sourceBlockArgs)) {
|
||||
SmallVector<int64_t> expandedShape {1}; llvm::append_range(expandedShape, type.getShape());
|
||||
SmallVector<ReassociationIndices> reassociation {{0, 1}};
|
||||
for (int64_t dim = 1; dim < type.getRank(); ++dim) reassociation.push_back({dim + 1});
|
||||
Value expanded = tensor::ExpandShapeOp::create(builder, loc,
|
||||
RankedTensorType::get(expandedShape, type.getElementType()), source, reassociation).getResult();
|
||||
SmallVector<OpFoldResult> offsets(shape.size(), builder.getIndexAttr(0));
|
||||
offsets[0] = builder.getIndexAttr(i);
|
||||
value = tensor::InsertSliceOp::create(builder, loc, expanded, value, offsets, sizes, strides).getResult();
|
||||
auto scheduled = transfer->getParentOfType<SpatScheduledComputeBatch>();
|
||||
if (!scheduled || sourceOperandForScheduledLane.size() != static_cast<size_t>(scheduled.getLaneCount()))
|
||||
return transfer.emitOpError("deferred source mapping must cover every scheduled lane"), failure();
|
||||
SmallVector<Value> candidates;
|
||||
candidates.reserve(sourceOperandForScheduledLane.size());
|
||||
for (int64_t sourceIndex : sourceOperandForScheduledLane) {
|
||||
if (sourceIndex < 0 || sourceIndex >= static_cast<int64_t>(sourceBlockArgs.size()))
|
||||
return transfer.emitOpError("deferred source mapping operand is out of range"), failure();
|
||||
candidates.push_back(sourceBlockArgs[sourceIndex]);
|
||||
}
|
||||
SmallVector<OpFoldResult> offsets(shape.size(), builder.getIndexAttr(0)); offsets[0] = index;
|
||||
SmallVector<int64_t> sliceShape {1}; llvm::append_range(sliceShape, type.getShape());
|
||||
auto slice = tensor::ExtractSliceOp::create(builder, loc,
|
||||
RankedTensorType::get(sliceShape, type.getElementType()), value, offsets, sizes, strides);
|
||||
// extract has a leading unit dimension; remove it without changing the payload.
|
||||
SmallVector<ReassociationIndices> reassociation {{0, 1}};
|
||||
for (int64_t dim = 1; dim < type.getRank(); ++dim) reassociation.push_back({dim + 1});
|
||||
return tensor::CollapseShapeOp::create(builder, loc, type, slice.getResult(), reassociation).getResult();
|
||||
return buildIndexSwitchSelection(builder, loc, scheduledLane, candidates, transfer.getOperation());
|
||||
}
|
||||
|
||||
static bool isTopLevelShaping(Operation *op, Block &body) {
|
||||
return op->getBlock() == &body && isSupportedDeferredShapingOp(op);
|
||||
static bool isDeferredPayloadCandidateOp(Operation *op) {
|
||||
return isShapingOnlyOp(op) || isCompileTimeOp(op) || isPureIndexComputationOp(op);
|
||||
}
|
||||
|
||||
static bool isTopLevelDeferredOperation(Operation *op, Block &body,
|
||||
const DeferredInputPlan &plan) {
|
||||
(void)plan;
|
||||
return op->getBlock() == &body
|
||||
&& (isDeferredPayloadCandidateOp(op) || isa<scf::ForOp>(op));
|
||||
}
|
||||
|
||||
static bool isEligible(Value value, Block &body, const DeferredInputPlan &plan,
|
||||
@@ -180,7 +189,7 @@ static bool isEligible(Value value, Block &body, const DeferredInputPlan &plan,
|
||||
Operation *op = value.getDefiningOp();
|
||||
if (op && op->hasTrait<OpTrait::ConstantLike>())
|
||||
return true;
|
||||
if (!op || !isTopLevelShaping(op, body) || !seen.insert(op).second)
|
||||
if (!op || !isTopLevelDeferredOperation(op, body, plan) || !seen.insert(op).second)
|
||||
return op && seen.contains(op);
|
||||
return llvm::all_of(op->getOperands(), [&](Value operand) { return isEligible(operand, body, plan, seen); });
|
||||
}
|
||||
@@ -196,7 +205,7 @@ static FailureOr<Value> clonePayloadRoot(Value root, Block &body, const Deferred
|
||||
if (isa<BlockArgument>(value))
|
||||
return transfer.emitOpError("phase 1 payload shaping captures an unsupported block argument"), failure();
|
||||
Operation *op = value.getDefiningOp();
|
||||
if (!op || !isSupportedDeferredShapingOp(op))
|
||||
if (!op || (!isDeferredPayloadCandidateOp(op) && !op->hasTrait<OpTrait::ConstantLike>()))
|
||||
return transfer.emitOpError("phase 1 cannot clone the scheduled graph-lane expression"), failure();
|
||||
for (Value operand : op->getOperands()) if (failed(cloneScheduledLane(operand))) return failure();
|
||||
Operation *copy = builder.clone(*op, mapping);
|
||||
@@ -214,7 +223,7 @@ static FailureOr<Value> clonePayloadRoot(Value root, Block &body, const Deferred
|
||||
if (isa<BlockArgument>(value))
|
||||
return transfer.emitOpError("phase 1 payload shaping captures an unsupported block argument"), failure();
|
||||
Operation *op = value.getDefiningOp();
|
||||
if (!op || (!isTopLevelShaping(op, body) && !op->hasTrait<OpTrait::ConstantLike>()))
|
||||
if (!op || (!isTopLevelDeferredOperation(op, body, plan) && !op->hasTrait<OpTrait::ConstantLike>()))
|
||||
return transfer.emitOpError("phase 1 payload shaping contains an unsupported operation"), failure();
|
||||
for (Value operand : op->getOperands()) if (failed(clone(operand))) return failure();
|
||||
Operation *copy = builder.clone(*op, mapping);
|
||||
@@ -225,65 +234,32 @@ static FailureOr<Value> clonePayloadRoot(Value root, Block &body, const Deferred
|
||||
}
|
||||
|
||||
static bool dependsOnGraphLane(Value value, Value graphLane, Block &body,
|
||||
const DeferredInputPlan &plan,
|
||||
llvm::SmallPtrSetImpl<Operation *> &seen) {
|
||||
if (value == graphLane)
|
||||
return true;
|
||||
Operation *op = value.getDefiningOp();
|
||||
if (!op || !isTopLevelShaping(op, body) || !seen.insert(op).second)
|
||||
if (!op || !isTopLevelDeferredOperation(op, body, plan) || !seen.insert(op).second)
|
||||
return false;
|
||||
return llvm::any_of(op->getOperands(), [&](Value operand) {
|
||||
return dependsOnGraphLane(operand, graphLane, body, seen);
|
||||
});
|
||||
}
|
||||
|
||||
static FailureOr<Value> buildPayloadAggregate(OpBuilder &builder, Location loc,
|
||||
ArrayRef<Value> payloads) {
|
||||
auto payloadType = dyn_cast<RankedTensorType>(payloads.front().getType());
|
||||
if (!payloadType || !payloadType.hasStaticShape())
|
||||
return failure();
|
||||
SmallVector<int64_t> shape {static_cast<int64_t>(payloads.size())};
|
||||
llvm::append_range(shape, payloadType.getShape());
|
||||
auto aggregateType = RankedTensorType::get(shape, payloadType.getElementType());
|
||||
auto empty = createEmptyTensorForType(builder, loc, aggregateType);
|
||||
if (failed(empty)) return failure();
|
||||
Value aggregate = *empty;
|
||||
SmallVector<OpFoldResult> sizes, strides(shape.size(), builder.getIndexAttr(1));
|
||||
sizes.push_back(builder.getIndexAttr(1));
|
||||
for (int64_t dim : payloadType.getShape()) sizes.push_back(builder.getIndexAttr(dim));
|
||||
SmallVector<ReassociationIndices> reassociation {{0, 1}};
|
||||
for (int64_t dim = 1; dim < payloadType.getRank(); ++dim) reassociation.push_back({dim + 1});
|
||||
SmallVector<int64_t> expandedShape {1}; llvm::append_range(expandedShape, payloadType.getShape());
|
||||
auto expandedType = RankedTensorType::get(expandedShape, payloadType.getElementType());
|
||||
for (auto [index, payload] : llvm::enumerate(payloads)) {
|
||||
Value expanded = tensor::ExpandShapeOp::create(builder, loc, expandedType, payload, reassociation).getResult();
|
||||
SmallVector<OpFoldResult> offsets(shape.size(), builder.getIndexAttr(0));
|
||||
offsets[0] = builder.getIndexAttr(index);
|
||||
aggregate = tensor::InsertSliceOp::create(builder, loc, expanded, aggregate, offsets, sizes, strides).getResult();
|
||||
if (auto loop = dyn_cast<scf::ForOp>(op)) {
|
||||
bool depends = false;
|
||||
loop.getRegion().walk([&](Operation *nested) {
|
||||
depends |= llvm::is_contained(nested->getOperands(), graphLane);
|
||||
});
|
||||
if (depends)
|
||||
return true;
|
||||
}
|
||||
return aggregate;
|
||||
}
|
||||
|
||||
static FailureOr<Value> selectPayloadAggregate(OpBuilder &builder, Location loc, Value aggregate,
|
||||
Value localLane) {
|
||||
auto aggregateType = cast<RankedTensorType>(aggregate.getType());
|
||||
SmallVector<int64_t> payloadShape(aggregateType.getShape().begin() + 1, aggregateType.getShape().end());
|
||||
auto payloadType = RankedTensorType::get(payloadShape, aggregateType.getElementType());
|
||||
SmallVector<OpFoldResult> offsets(aggregateType.getRank(), builder.getIndexAttr(0)); offsets[0] = localLane;
|
||||
SmallVector<OpFoldResult> sizes, strides(aggregateType.getRank(), builder.getIndexAttr(1));
|
||||
sizes.push_back(builder.getIndexAttr(1));
|
||||
for (int64_t dim : payloadShape) sizes.push_back(builder.getIndexAttr(dim));
|
||||
SmallVector<int64_t> unitShape {1}; llvm::append_range(unitShape, payloadShape);
|
||||
Value unit = tensor::ExtractSliceOp::create(builder, loc,
|
||||
RankedTensorType::get(unitShape, aggregateType.getElementType()), aggregate, offsets, sizes, strides).getResult();
|
||||
SmallVector<ReassociationIndices> reassociation {{0, 1}};
|
||||
for (int64_t dim = 1; dim < payloadType.getRank(); ++dim) reassociation.push_back({dim + 1});
|
||||
return tensor::CollapseShapeOp::create(builder, loc, payloadType, unit, reassociation).getResult();
|
||||
return llvm::any_of(op->getOperands(), [&](Value operand) {
|
||||
return dependsOnGraphLane(operand, graphLane, body, plan, seen);
|
||||
});
|
||||
}
|
||||
|
||||
static void collectClosure(Value value, Block &body, const DeferredInputPlan &plan,
|
||||
llvm::SmallPtrSetImpl<Operation *> &ops) {
|
||||
Operation *op = value.getDefiningOp();
|
||||
if (!op || !isTopLevelShaping(op, body) || !ops.insert(op).second) return;
|
||||
if (!op || !isTopLevelDeferredOperation(op, body, plan) || !ops.insert(op).second) return;
|
||||
if (auto loop = dyn_cast<scf::ForOp>(op))
|
||||
loop.getRegion().walk([&](Operation *nested) { ops.insert(nested); });
|
||||
for (Value operand : op->getOperands())
|
||||
if (operand != plan.graphInput && operand != plan.graphLane) collectClosure(operand, body, plan, ops);
|
||||
}
|
||||
@@ -371,12 +347,12 @@ LogicalResult materializeDeferredPayloadDemands(OpBuilder &builder, Location loc
|
||||
if (!seen.insert(value).second) continue;
|
||||
for (OpOperand &use : value.getUses()) {
|
||||
Operation *user = use.getOwner();
|
||||
if (!isTopLevelShaping(user, body)) { needsIdentity = true; continue; }
|
||||
if (!isTopLevelDeferredOperation(user, body, plan)) { needsIdentity = true; continue; }
|
||||
llvm::SmallPtrSet<Operation *, 16> eligibility;
|
||||
if (!isEligible(user->getResult(0), body, plan, eligibility)) { needsIdentity = true; continue; }
|
||||
for (Value result : user->getResults()) {
|
||||
bool hasShapingUse = llvm::any_of(result.getUses(), [&](OpOperand &next) { return isTopLevelShaping(next.getOwner(), body); });
|
||||
bool hasOtherUse = llvm::any_of(result.getUses(), [&](OpOperand &next) { return !isTopLevelShaping(next.getOwner(), body); });
|
||||
bool hasShapingUse = llvm::any_of(result.getUses(), [&](OpOperand &next) { return isTopLevelDeferredOperation(next.getOwner(), body, plan); });
|
||||
bool hasOtherUse = llvm::any_of(result.getUses(), [&](OpOperand &next) { return !isTopLevelDeferredOperation(next.getOwner(), body, plan); });
|
||||
if (hasOtherUse) roots.push_back(result);
|
||||
if (hasShapingUse) worklist.push_back(result);
|
||||
}
|
||||
@@ -388,7 +364,7 @@ LogicalResult materializeDeferredPayloadDemands(OpBuilder &builder, Location loc
|
||||
for (Value root : roots) {
|
||||
llvm::SmallPtrSet<Operation *, 16> laneDependencies;
|
||||
bool scalarize = plan.scalarizedGraphLaneBase
|
||||
&& dependsOnGraphLane(root, plan.graphLane, body, laneDependencies);
|
||||
&& dependsOnGraphLane(root, plan.graphLane, body, plan, laneDependencies);
|
||||
OpBuilder::InsertPoint restore = builder.saveInsertionPoint();
|
||||
Operation *loop = nullptr;
|
||||
if (scalarize) {
|
||||
@@ -419,9 +395,8 @@ LogicalResult materializeDeferredPayloadDemands(OpBuilder &builder, Location loc
|
||||
if (failed(selected)) return failure();
|
||||
Value boundGraphLane;
|
||||
if (scalarize) {
|
||||
Value offsetValue = arith::ConstantIndexOp::create(builder, loc, offset);
|
||||
boundGraphLane = offset ? arith::AddIOp::create(builder, loc, plan.scalarizedGraphLaneBase, offsetValue).getResult()
|
||||
: plan.scalarizedGraphLaneBase;
|
||||
boundGraphLane = affineAddConst(
|
||||
builder, loc, plan.scalarizedGraphLaneBase, offset, transfer.getOperation());
|
||||
}
|
||||
auto payload = clonePayloadRoot(root, body, plan, builder, transfer, *selected, boundGraphLane);
|
||||
if (failed(payload)) return failure();
|
||||
@@ -431,30 +406,26 @@ LogicalResult materializeDeferredPayloadDemands(OpBuilder &builder, Location loc
|
||||
}
|
||||
if (scalarize) {
|
||||
builder.restoreInsertionPoint(restore);
|
||||
if (payloads.size() == 1) {
|
||||
mapper.map(root, payloads.front());
|
||||
} else {
|
||||
if (loop) builder.setInsertionPoint(loop);
|
||||
else builder.setInsertionPointToEnd(plan.scalarizedHoistBlock);
|
||||
auto aggregate = buildPayloadAggregate(builder, loc, payloads);
|
||||
if (failed(aggregate)) return failure();
|
||||
builder.restoreInsertionPoint(restore);
|
||||
auto selected = selectPayloadAggregate(builder, loc, *aggregate, plan.scalarizedLocalLane);
|
||||
if (failed(selected)) return failure();
|
||||
mapper.map(root, *selected);
|
||||
}
|
||||
auto selected = buildIndexSwitchSelection(
|
||||
builder, loc, plan.scalarizedLocalLane, payloads, root.getDefiningOp());
|
||||
if (failed(selected)) return failure();
|
||||
mapper.map(root, *selected);
|
||||
} else {
|
||||
mapper.map(root, payloads.front());
|
||||
}
|
||||
collectClosure(root, body, plan, absorbed);
|
||||
}
|
||||
}
|
||||
SmallVector<Operation *> notFullyAbsorbed;
|
||||
for (Operation *op : absorbed) {
|
||||
bool allResultsMapped = llvm::all_of(op->getResults(), [&](Value result) {
|
||||
return mapper.contains(result) || llvm::all_of(result.getUses(), [&](OpOperand &use) { return absorbed.contains(use.getOwner()); });
|
||||
});
|
||||
if (!allResultsMapped) absorbed.erase(op);
|
||||
if (!allResultsMapped)
|
||||
notFullyAbsorbed.push_back(op);
|
||||
}
|
||||
for (Operation *op : notFullyAbsorbed)
|
||||
absorbed.erase(op);
|
||||
return success();
|
||||
}
|
||||
|
||||
|
||||
+2637
-279
File diff suppressed because it is too large
Load Diff
+500
-74
@@ -1,9 +1,11 @@
|
||||
#include "DeferredProjectionAnalysis.hpp"
|
||||
|
||||
#include "mlir/Dialect/Affine/IR/AffineOps.h"
|
||||
#include "mlir/Dialect/Arith/IR/Arith.h"
|
||||
#include "mlir/Dialect/SCF/IR/SCF.h"
|
||||
#include "mlir/IR/Matchers.h"
|
||||
#include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp"
|
||||
#include "llvm/ADT/SmallPtrSet.h"
|
||||
#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/CompileTime.hpp"
|
||||
|
||||
#include <limits>
|
||||
|
||||
@@ -11,43 +13,83 @@ namespace onnx_mlir::spatial {
|
||||
using namespace mlir;
|
||||
namespace {
|
||||
|
||||
static FailureOr<int64_t> getSignedInt64(IntegerAttr value) {
|
||||
return value.getValue().isSignedIntN(64) ? FailureOr<int64_t>(value.getValue().getSExtValue())
|
||||
: FailureOr<int64_t>(failure());
|
||||
}
|
||||
|
||||
static FailureOr<int64_t> evaluate(Value value, const StaticIndexEnvironment &environment,
|
||||
llvm::SmallDenseSet<Value, 16> &visiting);
|
||||
|
||||
static FailureOr<int64_t> evaluateDenseExtract(tensor::ExtractOp extract,
|
||||
const StaticIndexEnvironment &environment,
|
||||
llvm::SmallDenseSet<Value, 16> &visiting) {
|
||||
auto constant = extract.getTensor().getDefiningOp<arith::ConstantOp>();
|
||||
auto elements = constant ? dyn_cast<DenseIntElementsAttr>(constant.getValue()) : DenseIntElementsAttr();
|
||||
auto type = elements ? dyn_cast<RankedTensorType>(elements.getType()) : RankedTensorType();
|
||||
if (!elements || !type || !type.hasStaticShape()
|
||||
|| extract.getIndices().size() != static_cast<size_t>(type.getRank()))
|
||||
return failure();
|
||||
int64_t linear = 0;
|
||||
for (auto [index, dim] : llvm::zip(extract.getIndices(), type.getShape())) {
|
||||
auto folded = evaluate(index, environment, visiting);
|
||||
if (failed(folded) || *folded < 0 || *folded >= dim
|
||||
|| llvm::MulOverflow(linear, dim, linear) || llvm::AddOverflow(linear, *folded, linear))
|
||||
return failure();
|
||||
}
|
||||
APInt value = elements.getValues<APInt>()[linear];
|
||||
return value.isSignedIntN(64) ? FailureOr<int64_t>(value.getSExtValue())
|
||||
: FailureOr<int64_t>(failure());
|
||||
}
|
||||
|
||||
static FailureOr<int64_t> evaluate(Value value, const StaticIndexEnvironment &environment,
|
||||
llvm::SmallDenseSet<Value, 16> &visiting) {
|
||||
if (auto it = environment.bindings.find(value); it != environment.bindings.end())
|
||||
return it->second;
|
||||
Attribute constant;
|
||||
if (matchPattern(value, m_Constant(&constant)))
|
||||
if (auto integer = dyn_cast_or_null<IntegerAttr>(constant))
|
||||
return getSignedInt64(integer);
|
||||
if (isa<BlockArgument>(value) || !value.getDefiningOp())
|
||||
return failure();
|
||||
if (!visiting.insert(value).second)
|
||||
return failure();
|
||||
if (auto constant = value.getDefiningOp<arith::ConstantOp>())
|
||||
if (auto integer = dyn_cast<IntegerAttr>(constant.getValue()))
|
||||
{ visiting.erase(value); return integer.getInt(); }
|
||||
if (auto cast = value.getDefiningOp<arith::IndexCastOp>()) {
|
||||
auto result = evaluate(cast.getIn(), environment, visiting); visiting.erase(value); return result;
|
||||
}
|
||||
auto binary = [&](auto op, auto fn) -> FailureOr<int64_t> {
|
||||
auto lhs = evaluate(op.getLhs(), environment, visiting);
|
||||
auto rhs = evaluate(op.getRhs(), environment, visiting);
|
||||
if (failed(lhs) || failed(rhs)) return failure();
|
||||
return fn(*lhs, *rhs);
|
||||
};
|
||||
if (auto op = value.getDefiningOp<arith::AddIOp>()) { auto result = binary(op, [](int64_t a, int64_t b) -> FailureOr<int64_t> { int64_t r; if (llvm::AddOverflow(a, b, r)) return failure(); return r; }); visiting.erase(value); return result; }
|
||||
if (auto op = value.getDefiningOp<arith::SubIOp>()) { auto result = binary(op, [](int64_t a, int64_t b) -> FailureOr<int64_t> { int64_t r; if (llvm::SubOverflow(a, b, r)) return failure(); return r; }); visiting.erase(value); return result; }
|
||||
if (auto op = value.getDefiningOp<arith::MulIOp>()) { auto result = binary(op, [](int64_t a, int64_t b) -> FailureOr<int64_t> { int64_t r; if (llvm::MulOverflow(a, b, r)) return failure(); return r; }); visiting.erase(value); return result; }
|
||||
if (auto apply = value.getDefiningOp<affine::AffineApplyOp>()) { auto result = evaluateAffineApply(apply, [&](Value operand) { return evaluate(operand, environment, visiting); }); visiting.erase(value); return result; }
|
||||
if (auto extract = value.getDefiningOp<tensor::ExtractOp>()) {
|
||||
auto constant = extract.getTensor().getDefiningOp<arith::ConstantOp>();
|
||||
auto elements = constant ? dyn_cast<DenseIntElementsAttr>(constant.getValue()) : DenseIntElementsAttr();
|
||||
auto type = elements ? dyn_cast<RankedTensorType>(elements.getType()) : RankedTensorType();
|
||||
if (!elements || !type || extract.getIndices().size() != static_cast<size_t>(type.getRank())) return failure();
|
||||
int64_t linear = 0;
|
||||
for (auto [index, dim] : llvm::zip(extract.getIndices(), type.getShape())) {
|
||||
auto i = evaluate(index, environment, visiting);
|
||||
if (failed(i) || *i < 0 || *i >= dim) return failure();
|
||||
linear = linear * dim + *i;
|
||||
auto result = evaluateDenseExtract(extract, environment, visiting);
|
||||
visiting.erase(value);
|
||||
return result;
|
||||
}
|
||||
|
||||
Operation *definingOp = value.getDefiningOp();
|
||||
if (definingOp->getNumRegions() != 0 || !isPureIndexComputationOp(definingOp)) {
|
||||
visiting.erase(value);
|
||||
return failure();
|
||||
}
|
||||
SmallVector<Attribute> operandConstants;
|
||||
operandConstants.reserve(definingOp->getNumOperands());
|
||||
Builder builder(definingOp->getContext());
|
||||
for (Value operand : definingOp->getOperands()) {
|
||||
auto folded = evaluate(operand, environment, visiting);
|
||||
if (failed(folded)) {
|
||||
visiting.erase(value);
|
||||
return failure();
|
||||
}
|
||||
visiting.erase(value); return elements.getValues<APInt>()[linear].getSExtValue();
|
||||
operandConstants.push_back(builder.getIntegerAttr(operand.getType(), *folded));
|
||||
}
|
||||
SmallVector<OpFoldResult> results;
|
||||
if (failed(definingOp->fold(operandConstants, results)) || results.size() != 1) {
|
||||
visiting.erase(value);
|
||||
return failure();
|
||||
}
|
||||
FailureOr<int64_t> result = failure();
|
||||
if (auto integer = dyn_cast<Attribute>(results.front())) {
|
||||
if (auto attr = dyn_cast<IntegerAttr>(integer))
|
||||
result = getSignedInt64(attr);
|
||||
} else if (auto foldedValue = dyn_cast<Value>(results.front())) {
|
||||
result = evaluate(foldedValue, environment, visiting);
|
||||
}
|
||||
visiting.erase(value);
|
||||
return failure();
|
||||
return result;
|
||||
}
|
||||
|
||||
static FailureOr<std::optional<unsigned>> sourceArgument(Value value, SpatDeferredCommunicationOp deferred,
|
||||
@@ -57,50 +99,29 @@ static FailureOr<std::optional<unsigned>> sourceArgument(Value value, SpatDeferr
|
||||
argument && argument.getOwner() == &deferred.getBody().front()
|
||||
&& argument.getArgNumber() < deferred.getSources().size())
|
||||
return std::optional<unsigned>(argument.getArgNumber());
|
||||
// Phase 1's selector ends in collapse(extract_slice(stacked, table[lane])).
|
||||
auto collapse = value.getDefiningOp<tensor::CollapseShapeOp>();
|
||||
if (!collapse) return std::optional<unsigned>();
|
||||
value = collapse.getSrc();
|
||||
auto slice = value.getDefiningOp<tensor::ExtractSliceOp>();
|
||||
if (!slice) return std::optional<unsigned>();
|
||||
auto sourceType = dyn_cast<RankedTensorType>(slice.getSourceType());
|
||||
if (!sourceType || slice.getMixedOffsets().size() != static_cast<size_t>(sourceType.getRank()))
|
||||
auto result = dyn_cast<OpResult>(value);
|
||||
auto selection = result ? dyn_cast<scf::IndexSwitchOp>(result.getOwner()) : scf::IndexSwitchOp();
|
||||
if (!selection || result.getResultNumber() != 0 || selection.getNumResults() != 1)
|
||||
return std::optional<unsigned>();
|
||||
for (unsigned dim = 1; dim < sourceType.getRank(); ++dim) {
|
||||
auto offset = evaluateDeferredIndex(slice.getMixedOffsets()[dim], environment);
|
||||
auto size = evaluateDeferredIndex(slice.getMixedSizes()[dim], environment);
|
||||
auto stride = evaluateDeferredIndex(slice.getMixedStrides()[dim], environment);
|
||||
if (failed(offset) || failed(size) || failed(stride) || *offset != 0 || *size != sourceType.getDimSize(dim) || *stride != 1)
|
||||
return std::optional<unsigned>();
|
||||
}
|
||||
auto leadingSize = evaluateDeferredIndex(slice.getMixedSizes().front(), environment);
|
||||
auto leadingStride = evaluateDeferredIndex(slice.getMixedStrides().front(), environment);
|
||||
if (failed(leadingSize) || failed(leadingStride) || *leadingSize != 1 || *leadingStride != 1)
|
||||
return std::optional<unsigned>();
|
||||
auto selected = evaluateDeferredIndex(slice.getMixedOffsets().front(), environment);
|
||||
if (failed(selected)) return failure();
|
||||
Value stacked = slice.getSource();
|
||||
while (auto cast = stacked.getDefiningOp<tensor::CastOp>()) stacked = cast.getSource();
|
||||
while (auto insert = stacked.getDefiningOp<tensor::InsertSliceOp>()) stacked = insert.getDest();
|
||||
// The stack is a chain. Find the insertion at the selected leading offset.
|
||||
for (Value cursor = slice.getSource(); auto insert = cursor.getDefiningOp<tensor::InsertSliceOp>(); cursor = insert.getDest()) {
|
||||
auto offset = evaluateDeferredIndex(insert.getMixedOffsets().front(), environment);
|
||||
if (succeeded(offset) && *offset == *selected) {
|
||||
Value source = insert.getSource();
|
||||
if (auto expand = source.getDefiningOp<tensor::ExpandShapeOp>()) source = expand.getSrc();
|
||||
if (auto arg = dyn_cast<BlockArgument>(source); arg && arg.getOwner() == &deferred.getBody().front())
|
||||
return std::optional<unsigned>(arg.getArgNumber());
|
||||
auto selector = evaluateDeferredIndex(selection.getArg(), environment);
|
||||
if (failed(selector))
|
||||
return failure();
|
||||
Region *selectedRegion = &selection.getDefaultRegion();
|
||||
for (auto [caseValue, region] : llvm::zip(selection.getCases(), selection.getCaseRegions()))
|
||||
if (caseValue == *selector) {
|
||||
selectedRegion = ®ion;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return std::optional<unsigned>();
|
||||
}
|
||||
|
||||
static bool isResidual(Operation *op) {
|
||||
return isa<tensor::CollapseShapeOp, tensor::ExpandShapeOp, tensor::CastOp,
|
||||
tensor::ExtractSliceOp, tensor::InsertSliceOp, tensor::ConcatOp,
|
||||
tensor::EmptyOp, tensor::ExtractOp, arith::ConstantOp,
|
||||
arith::IndexCastOp, arith::AddIOp, arith::SubIOp, arith::MulIOp,
|
||||
affine::AffineApplyOp>(op);
|
||||
if (!selectedRegion->hasOneBlock())
|
||||
return failure();
|
||||
Block &block = selectedRegion->front();
|
||||
auto yield = dyn_cast<scf::YieldOp>(block.getTerminator());
|
||||
if (!yield || yield.getResults().size() != 1)
|
||||
return failure();
|
||||
for (Operation &op : block.without_terminator())
|
||||
if (!isa<tensor::CastOp>(op))
|
||||
return failure();
|
||||
return sourceArgument(yield.getResults().front(), deferred, environment);
|
||||
}
|
||||
|
||||
static SpatGraphComputeBatch graphBatchOwner(Value value) {
|
||||
@@ -119,15 +140,394 @@ static Value getEnclosingScheduledLane(SpatDeferredCommunicationOp deferred,
|
||||
return block && !block->empty() ? block->getArgument(0) : Value();
|
||||
}
|
||||
|
||||
static bool isInsideDeferredLoop(Operation *op,
|
||||
SpatDeferredCommunicationOp deferred) {
|
||||
for (Operation *parent = op->getParentOp(); parent && parent != deferred;
|
||||
parent = parent->getParentOp())
|
||||
if (isa<scf::ForOp>(parent))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool isAllowedStaticIndexExpression(
|
||||
Value value, Value scheduledLane,
|
||||
llvm::SmallDenseSet<Value, 16> &visiting) {
|
||||
if (value == scheduledLane)
|
||||
return true;
|
||||
Attribute constant;
|
||||
if (matchPattern(value, m_Constant(&constant)))
|
||||
return true;
|
||||
if (isa<BlockArgument>(value) || !value.getDefiningOp()
|
||||
|| !visiting.insert(value).second)
|
||||
return false;
|
||||
Operation *op = value.getDefiningOp();
|
||||
bool allowed = op->getNumRegions() == 0
|
||||
&& (isPureIndexComputationOp(op) || isCompileTimeOp(op))
|
||||
&& llvm::all_of(op->getOperands(), [&](Value operand) {
|
||||
return isAllowedStaticIndexExpression(operand, scheduledLane,
|
||||
visiting);
|
||||
});
|
||||
visiting.erase(value);
|
||||
return allowed;
|
||||
}
|
||||
|
||||
static bool isAllowedStaticIndexExpression(Value value, Value scheduledLane) {
|
||||
llvm::SmallDenseSet<Value, 16> visiting;
|
||||
return isAllowedStaticIndexExpression(value, scheduledLane, visiting);
|
||||
}
|
||||
|
||||
static bool originatesFromDeferredSource(
|
||||
Value value, SpatDeferredCommunicationOp deferred,
|
||||
llvm::SmallDenseSet<Value, 16> &visited) {
|
||||
if (!visited.insert(value).second)
|
||||
return false;
|
||||
if (auto argument = dyn_cast<BlockArgument>(value))
|
||||
return argument.getOwner() == &deferred.getBody().front()
|
||||
&& argument.getArgNumber() < deferred.getSources().size();
|
||||
Operation *op = value.getDefiningOp();
|
||||
if (!op)
|
||||
return false;
|
||||
if (isa<scf::IndexSwitchOp>(op) && op->getBlock() == &deferred.getBody().front())
|
||||
return true;
|
||||
return llvm::any_of(op->getOperands(), [&](Value operand) {
|
||||
return originatesFromDeferredSource(operand, deferred, visited);
|
||||
});
|
||||
}
|
||||
|
||||
static bool originatesFromDeferredSource(
|
||||
Value value, SpatDeferredCommunicationOp deferred) {
|
||||
llvm::SmallDenseSet<Value, 16> visited;
|
||||
return originatesFromDeferredSource(value, deferred, visited);
|
||||
}
|
||||
|
||||
static LogicalResult verifyCanonicalSourceSelector(
|
||||
scf::IndexSwitchOp selection, SpatDeferredCommunicationOp deferred,
|
||||
Value scheduledLane, int64_t laneCount) {
|
||||
if (selection->getBlock() != &deferred.getBody().front()
|
||||
|| selection.getNumResults() != 1
|
||||
|| !selection.getArg().getType().isIndex()
|
||||
|| !isAllowedStaticIndexExpression(selection.getArg(), scheduledLane))
|
||||
return selection.emitOpError("is not a canonical deferred source selector");
|
||||
if (laneCount < 2
|
||||
|| selection.getCases().size() != static_cast<size_t>(laneCount - 1)
|
||||
|| selection.getCaseRegions().size() != selection.getCases().size())
|
||||
return selection.emitOpError(
|
||||
"must cover every non-default scheduled lane");
|
||||
for (auto [index, caseValue] : llvm::enumerate(selection.getCases()))
|
||||
if (caseValue != static_cast<int64_t>(index))
|
||||
return selection.emitOpError(
|
||||
"must use consecutive scheduled-lane cases starting at zero");
|
||||
|
||||
auto verifyRegion = [&](Region ®ion) -> LogicalResult {
|
||||
if (!region.hasOneBlock())
|
||||
return selection.emitOpError("source-selector region must have one block");
|
||||
Block &block = region.front();
|
||||
auto yield = dyn_cast<scf::YieldOp>(block.getTerminator());
|
||||
if (!yield || yield.getResults().size() != 1
|
||||
|| yield.getResults().front().getType() != selection.getResult(0).getType())
|
||||
return selection.emitOpError(
|
||||
"source-selector region must yield one exact result type");
|
||||
for (Operation &op : block.without_terminator())
|
||||
if (!isa<tensor::CastOp>(op))
|
||||
return selection.emitOpError(
|
||||
"source-selector regions may contain only tensor.cast before scf.yield");
|
||||
Value source = yield.getResults().front();
|
||||
while (auto cast = source.getDefiningOp<tensor::CastOp>()) {
|
||||
if (cast->getBlock() != &block)
|
||||
return selection.emitOpError(
|
||||
"source-selector casts must be local to their region");
|
||||
source = cast.getSource();
|
||||
}
|
||||
auto argument = dyn_cast<BlockArgument>(source);
|
||||
if (!argument || argument.getOwner() != &deferred.getBody().front()
|
||||
|| argument.getArgNumber() >= deferred.getSources().size())
|
||||
return selection.emitOpError(
|
||||
"source-selector branch must resolve to a deferred source argument");
|
||||
return success();
|
||||
};
|
||||
for (Region ®ion : selection.getCaseRegions())
|
||||
if (failed(verifyRegion(region)))
|
||||
return failure();
|
||||
return verifyRegion(selection.getDefaultRegion());
|
||||
}
|
||||
|
||||
static bool valueDependsOnAny(
|
||||
Value value, const llvm::SmallDenseSet<Value, 16> &dependencies,
|
||||
llvm::SmallDenseSet<Value, 16> &visited) {
|
||||
if (dependencies.contains(value))
|
||||
return true;
|
||||
if (!visited.insert(value).second)
|
||||
return false;
|
||||
Operation *op = value.getDefiningOp();
|
||||
return op && llvm::any_of(op->getOperands(), [&](Value operand) {
|
||||
return valueDependsOnAny(operand, dependencies, visited);
|
||||
});
|
||||
}
|
||||
|
||||
static bool valueDependsOnAny(
|
||||
Value value, const llvm::SmallDenseSet<Value, 16> &dependencies) {
|
||||
llvm::SmallDenseSet<Value, 16> visited;
|
||||
return valueDependsOnAny(value, dependencies, visited);
|
||||
}
|
||||
|
||||
static LogicalResult specializeDeferredLoopStaticValues(
|
||||
scf::ForOp loop, SpatDeferredCommunicationOp deferred,
|
||||
const StaticIndexEnvironment &environment,
|
||||
SpecializedDeferredProgram &program,
|
||||
llvm::SmallDenseSet<Value, 16> dynamicIndices = {}) {
|
||||
auto record = [&](Value value, StringRef diagnostic) -> LogicalResult {
|
||||
auto folded = evaluateDeferredIndex(value, environment);
|
||||
if (failed(folded))
|
||||
return deferred.emitOpError(diagnostic);
|
||||
program.staticValues.try_emplace(value, *folded);
|
||||
return success();
|
||||
};
|
||||
auto step = evaluateDeferredIndex(loop.getStep(), environment);
|
||||
if (failed(record(loop.getLowerBound(),
|
||||
"deferred shaping loop lower bound did not specialize"))
|
||||
|| failed(record(loop.getUpperBound(),
|
||||
"deferred shaping loop upper bound did not specialize"))
|
||||
|| failed(step) || *step <= 0)
|
||||
return deferred.emitOpError(
|
||||
"deferred shaping loop requires specialized bounds and a positive step");
|
||||
program.staticValues.try_emplace(loop.getStep(), *step);
|
||||
|
||||
dynamicIndices.insert(loop.getInductionVar());
|
||||
for (Operation &nested : loop.getBody()->without_terminator()) {
|
||||
if (auto nestedLoop = dyn_cast<scf::ForOp>(nested)) {
|
||||
if (failed(specializeDeferredLoopStaticValues(
|
||||
nestedLoop, deferred, environment, program, dynamicIndices)))
|
||||
return failure();
|
||||
continue;
|
||||
}
|
||||
for (Value operand : nested.getOperands()) {
|
||||
if ((!operand.getType().isIndex()
|
||||
&& !isa<IntegerType>(operand.getType()))
|
||||
|| valueDependsOnAny(operand, dynamicIndices))
|
||||
continue;
|
||||
if (failed(record(operand,
|
||||
"deferred shaping loop captured an index that did not specialize")))
|
||||
return failure();
|
||||
}
|
||||
}
|
||||
return success();
|
||||
}
|
||||
|
||||
static FailureOr<std::optional<DeferredInsertAssembly>>
|
||||
analyzeDeferredInsertAssembly(
|
||||
const SpecializedDeferredProgram &program,
|
||||
const StaticIndexEnvironment &environment) {
|
||||
auto finalInsert = program.yieldedValue.getDefiningOp<tensor::InsertSliceOp>();
|
||||
if (!finalInsert || program.leaves.empty())
|
||||
return std::optional<DeferredInsertAssembly>();
|
||||
|
||||
DenseMap<Value, unsigned> leafByRoot;
|
||||
for (auto [leafIndex, leaf] : llvm::enumerate(program.leaves)) {
|
||||
if (leaf.physicalSlots.size() != 1
|
||||
|| !leafByRoot.try_emplace(leaf.replacementRoot, leafIndex).second)
|
||||
return std::optional<DeferredInsertAssembly>();
|
||||
}
|
||||
|
||||
SmallPtrSet<Operation *, 32> consumed;
|
||||
SmallVector<DeferredInsertAssemblyEntry> reverseEntries;
|
||||
llvm::SmallDenseSet<unsigned, 16> insertedLeaves;
|
||||
Value current = program.yieldedValue;
|
||||
while (auto insert = current.getDefiningOp<tensor::InsertSliceOp>()) {
|
||||
Value source = insert.getSource();
|
||||
SmallVector<Operation *> rankShaping;
|
||||
while (Operation *shape = source.getDefiningOp()) {
|
||||
if (auto collapse = dyn_cast<tensor::CollapseShapeOp>(shape))
|
||||
source = collapse.getSrc();
|
||||
else if (auto expand = dyn_cast<tensor::ExpandShapeOp>(shape))
|
||||
source = expand.getSrc();
|
||||
else
|
||||
break;
|
||||
rankShaping.push_back(shape);
|
||||
}
|
||||
auto leafIt = leafByRoot.find(source);
|
||||
if (leafIt == leafByRoot.end()
|
||||
|| !insertedLeaves.insert(leafIt->second).second)
|
||||
return std::optional<DeferredInsertAssembly>();
|
||||
|
||||
DeferredInsertAssemblyEntry entry;
|
||||
entry.leafIndex = leafIt->second;
|
||||
entry.requirementIndex = leafIt->second;
|
||||
auto foldGeometry = [&](ArrayRef<OpFoldResult> values,
|
||||
SmallVectorImpl<int64_t> &folded) {
|
||||
for (OpFoldResult value : values) {
|
||||
auto result = evaluateDeferredIndex(value, environment);
|
||||
if (failed(result))
|
||||
return failure();
|
||||
folded.push_back(*result);
|
||||
}
|
||||
return success();
|
||||
};
|
||||
if (failed(foldGeometry(insert.getMixedOffsets(),
|
||||
entry.targetGeometry.offsets))
|
||||
|| failed(foldGeometry(insert.getMixedSizes(),
|
||||
entry.targetGeometry.sizes))
|
||||
|| failed(foldGeometry(insert.getMixedStrides(),
|
||||
entry.targetGeometry.strides)))
|
||||
return std::optional<DeferredInsertAssembly>();
|
||||
reverseEntries.push_back(std::move(entry));
|
||||
consumed.insert(insert);
|
||||
consumed.insert(rankShaping.begin(), rankShaping.end());
|
||||
current = insert.getDest();
|
||||
}
|
||||
|
||||
auto initial = current.getDefiningOp<tensor::EmptyOp>();
|
||||
auto resultType = dyn_cast<RankedTensorType>(program.yieldedValue.getType());
|
||||
if (!initial || !initial.getDynamicSizes().empty() || !resultType
|
||||
|| !resultType.hasStaticShape() || initial.getType() != resultType
|
||||
|| insertedLeaves.size() != program.leaves.size())
|
||||
return std::optional<DeferredInsertAssembly>();
|
||||
consumed.insert(initial);
|
||||
|
||||
for (const DeferredInsertAssemblyEntry &entry : reverseEntries) {
|
||||
const StaticSliceGeometry &geometry = entry.targetGeometry;
|
||||
if (geometry.offsets.size() != static_cast<size_t>(resultType.getRank())
|
||||
|| geometry.sizes.size() != geometry.offsets.size()
|
||||
|| geometry.strides.size() != geometry.offsets.size())
|
||||
return std::optional<DeferredInsertAssembly>();
|
||||
for (auto [offset, size, stride, dim] :
|
||||
llvm::zip_equal(geometry.offsets, geometry.sizes,
|
||||
geometry.strides, resultType.getShape())) {
|
||||
int64_t span = 0;
|
||||
int64_t last = 0;
|
||||
if (offset < 0 || size <= 0 || stride <= 0
|
||||
|| llvm::MulOverflow(size - 1, stride, span)
|
||||
|| llvm::AddOverflow(offset, span, last) || last >= dim)
|
||||
return std::optional<DeferredInsertAssembly>();
|
||||
}
|
||||
}
|
||||
if (llvm::any_of(program.residualOps,
|
||||
[&](Operation *op) { return !consumed.contains(op); })
|
||||
|| consumed.size() != program.residualOps.size())
|
||||
return std::optional<DeferredInsertAssembly>();
|
||||
|
||||
DeferredInsertAssembly assembly;
|
||||
assembly.initialValue = initial;
|
||||
assembly.resultType = resultType;
|
||||
assembly.entries.assign(reverseEntries.rbegin(), reverseEntries.rend());
|
||||
return std::optional<DeferredInsertAssembly>(std::move(assembly));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
LogicalResult verifyDeferredProgramContract(
|
||||
SpatDeferredCommunicationOp deferred) {
|
||||
if (!deferred.getBody().hasOneBlock())
|
||||
return deferred.emitOpError("deferred program must have exactly one body block");
|
||||
Block &body = deferred.getBody().front();
|
||||
auto terminator = dyn_cast<SpatYieldOp>(body.getTerminator());
|
||||
if (!terminator || terminator.getOutputs().size() != 1)
|
||||
return deferred.emitOpError(
|
||||
"deferred program must have exactly one yielded value");
|
||||
|
||||
Value scheduledLane;
|
||||
int64_t laneCount = 1;
|
||||
if (auto scheduled = deferred->getParentOfType<SpatScheduledComputeBatch>()) {
|
||||
scheduledLane = getEnclosingScheduledLane(deferred, scheduled);
|
||||
laneCount = scheduled.getLaneCount();
|
||||
if (!scheduledLane || laneCount <= 0)
|
||||
return deferred.emitOpError(
|
||||
"deferred program cannot locate a valid enclosing scheduled lane");
|
||||
}
|
||||
|
||||
bool invalid = false;
|
||||
WalkResult result = deferred.getBody().walk([&](Operation *op) {
|
||||
if (invalid)
|
||||
return WalkResult::interrupt();
|
||||
auto reject = [&](StringRef message) {
|
||||
op->emitOpError(message);
|
||||
invalid = true;
|
||||
return WalkResult::interrupt();
|
||||
};
|
||||
|
||||
if (auto yield = dyn_cast<SpatYieldOp>(op)) {
|
||||
if (op != body.getTerminator())
|
||||
return reject("must be the unique top-level deferred terminator");
|
||||
return WalkResult::advance();
|
||||
}
|
||||
if (auto yield = dyn_cast<scf::YieldOp>(op)) {
|
||||
Operation *parent = op->getParentOp();
|
||||
if (!parent || !isa<scf::ForOp, scf::IndexSwitchOp>(parent)
|
||||
|| op != op->getBlock()->getTerminator())
|
||||
return reject("is not a valid deferred control-flow terminator");
|
||||
return WalkResult::advance();
|
||||
}
|
||||
if (auto selection = dyn_cast<scf::IndexSwitchOp>(op)) {
|
||||
if (failed(verifyCanonicalSourceSelector(
|
||||
selection, deferred, scheduledLane, laneCount))) {
|
||||
invalid = true;
|
||||
return WalkResult::interrupt();
|
||||
}
|
||||
return WalkResult::advance();
|
||||
}
|
||||
if (auto loop = dyn_cast<scf::ForOp>(op)) {
|
||||
auto isStaticRankedTensor = [](Type type) {
|
||||
auto tensor = dyn_cast<RankedTensorType>(type);
|
||||
return tensor && tensor.hasStaticShape();
|
||||
};
|
||||
if (!llvm::all_of(loop.getInitArgs(), [&](Value value) {
|
||||
return isStaticRankedTensor(value.getType());
|
||||
})
|
||||
|| !llvm::all_of(loop.getResultTypes(), isStaticRankedTensor))
|
||||
return reject(
|
||||
"requires static ranked tensor iter arguments and results");
|
||||
auto yield = dyn_cast<scf::YieldOp>(loop.getBody()->getTerminator());
|
||||
if (!yield || yield.getResults().getTypes() != loop.getResultTypes())
|
||||
return reject("yield types must exactly match loop result types");
|
||||
for (Value bound : {loop.getLowerBound(), loop.getUpperBound(),
|
||||
loop.getStep()})
|
||||
if (!isAllowedStaticIndexExpression(bound, scheduledLane))
|
||||
return reject(
|
||||
"bounds must use only constants, the scheduled lane, and pure index computation");
|
||||
for (int64_t lane = 0; lane < laneCount; ++lane) {
|
||||
StaticIndexEnvironment environment;
|
||||
if (scheduledLane)
|
||||
environment.bindings[scheduledLane] = lane;
|
||||
auto lower = evaluateDeferredIndex(loop.getLowerBound(), environment);
|
||||
auto upper = evaluateDeferredIndex(loop.getUpperBound(), environment);
|
||||
auto step = evaluateDeferredIndex(loop.getStep(), environment);
|
||||
if (failed(lower) || failed(upper) || failed(step) || *step <= 0)
|
||||
return reject(
|
||||
"bounds must specialize for every scheduled lane with a positive step");
|
||||
}
|
||||
return WalkResult::advance();
|
||||
}
|
||||
if (op->getNumRegions() != 0)
|
||||
return reject("unsupported region operation in deferred program");
|
||||
if (!isShapingOnlyOp(op) && !isCompileTimeOp(op)
|
||||
&& !isPureIndexComputationOp(op))
|
||||
return reject(
|
||||
"deferred program permits only shaping, compile-time, or pure index operations");
|
||||
|
||||
for (Value operand : op->getOperands()) {
|
||||
if (auto argument = dyn_cast<BlockArgument>(operand)) {
|
||||
Operation *owner = argument.getOwner()->getParentOp();
|
||||
bool local = owner == deferred
|
||||
|| (owner && deferred->isAncestor(owner));
|
||||
if (!local && operand != scheduledLane)
|
||||
return reject("captures an unsupported external block argument");
|
||||
}
|
||||
if (isInsideDeferredLoop(op, deferred)
|
||||
&& originatesFromDeferredSource(operand, deferred))
|
||||
return reject(
|
||||
"deferred source projection must remain outside residual loops");
|
||||
}
|
||||
return WalkResult::advance();
|
||||
});
|
||||
return success(!invalid && !result.wasInterrupted());
|
||||
}
|
||||
|
||||
FailureOr<int64_t> evaluateDeferredIndex(Value value, const StaticIndexEnvironment &environment) {
|
||||
llvm::SmallDenseSet<Value, 16> visiting;
|
||||
return evaluate(value, environment, visiting);
|
||||
}
|
||||
FailureOr<int64_t> evaluateDeferredIndex(OpFoldResult value, const StaticIndexEnvironment &environment) {
|
||||
if (auto attr = dyn_cast<Attribute>(value))
|
||||
if (auto integer = dyn_cast<IntegerAttr>(attr)) return integer.getInt();
|
||||
if (auto integer = dyn_cast<IntegerAttr>(attr)) return getSignedInt64(integer);
|
||||
if (auto dynamic = dyn_cast<Value>(value)) return evaluateDeferredIndex(dynamic, environment);
|
||||
return failure();
|
||||
}
|
||||
@@ -152,6 +552,8 @@ FailureOr<ResolvedDeferredSource> requireResolvedDeferredSource(Value value, Spa
|
||||
|
||||
FailureOr<SpecializedDeferredProgram> analyzeDeferredProgram(SpatDeferredCommunicationOp deferred,
|
||||
std::optional<unsigned> targetScheduledLane) {
|
||||
// The Phase 1 verifier must establish the deferred-program contract once;
|
||||
// this analysis only specializes lane-dependent static semantics.
|
||||
Block &body = deferred.getBody().front();
|
||||
auto yield = dyn_cast<SpatYieldOp>(body.getTerminator());
|
||||
if (!yield || yield.getOutputs().size() != 1)
|
||||
@@ -229,13 +631,37 @@ FailureOr<SpecializedDeferredProgram> analyzeDeferredProgram(SpatDeferredCommuni
|
||||
program.leaves.push_back(std::move(leaf));
|
||||
return success();
|
||||
}
|
||||
if (value.getType().isIndex() || isa<IntegerType>(value.getType())) {
|
||||
auto folded = evaluateDeferredIndex(value, environment);
|
||||
if (failed(folded))
|
||||
return deferred.emitOpError("deferred index expression is not statically evaluable after specialization");
|
||||
program.staticValues.try_emplace(value, *folded);
|
||||
return success();
|
||||
}
|
||||
Operation *op = value.getDefiningOp();
|
||||
if (!op || op->getBlock() != &body || !isResidual(op))
|
||||
return deferred.emitOpError("deferred residual contains an unsupported operation");
|
||||
if (!op || op->getBlock() != &body)
|
||||
return deferred.emitOpError("deferred residual escapes its verified body");
|
||||
if (auto loop = dyn_cast<scf::ForOp>(op)) {
|
||||
if (failed(specializeDeferredLoopStaticValues(
|
||||
loop, deferred, environment, program)))
|
||||
return failure();
|
||||
for (Value init : loop.getInitArgs())
|
||||
if (failed(visit(init)))
|
||||
return failure();
|
||||
program.residualOps.push_back(op);
|
||||
return success();
|
||||
}
|
||||
for (Value operand : op->getOperands()) if (failed(visit(operand))) return failure();
|
||||
program.residualOps.push_back(op); return success();
|
||||
};
|
||||
if (failed(visit(program.yieldedValue))) return failure();
|
||||
StaticIndexEnvironment assemblyEnvironment = environment;
|
||||
for (auto [value, staticValue] : program.staticValues)
|
||||
assemblyEnvironment.bindings.try_emplace(value, staticValue);
|
||||
auto assembly = analyzeDeferredInsertAssembly(program, assemblyEnvironment);
|
||||
if (failed(assembly))
|
||||
return failure();
|
||||
program.insertAssembly = std::move(*assembly);
|
||||
return std::move(program);
|
||||
}
|
||||
|
||||
|
||||
@@ -35,6 +35,18 @@ struct DeferredProjectionLeaf {
|
||||
mlir::RankedTensorType reconstructedType;
|
||||
};
|
||||
|
||||
struct DeferredInsertAssemblyEntry {
|
||||
unsigned requirementIndex = 0;
|
||||
unsigned leafIndex = 0;
|
||||
StaticSliceGeometry targetGeometry;
|
||||
};
|
||||
|
||||
struct DeferredInsertAssembly {
|
||||
mlir::tensor::EmptyOp initialValue;
|
||||
mlir::RankedTensorType resultType;
|
||||
llvm::SmallVector<DeferredInsertAssemblyEntry, 0> entries;
|
||||
};
|
||||
|
||||
struct SpecializedDeferredProgram {
|
||||
SpatDeferredCommunicationOp deferred;
|
||||
std::optional<unsigned> targetScheduledLane;
|
||||
@@ -42,6 +54,8 @@ struct SpecializedDeferredProgram {
|
||||
mlir::Value yieldedValue;
|
||||
llvm::SmallVector<DeferredProjectionLeaf, 0> leaves;
|
||||
llvm::SmallVector<mlir::Operation *> residualOps;
|
||||
llvm::DenseMap<mlir::Value, int64_t> staticValues;
|
||||
std::optional<DeferredInsertAssembly> insertAssembly;
|
||||
};
|
||||
|
||||
struct ResolvedDeferredSource {
|
||||
@@ -55,6 +69,8 @@ mlir::FailureOr<std::optional<ResolvedDeferredSource>> tryResolveDeferredSource(
|
||||
mlir::FailureOr<ResolvedDeferredSource> requireResolvedDeferredSource(
|
||||
mlir::Value value, SpatDeferredCommunicationOp deferred,
|
||||
const StaticIndexEnvironment &environment);
|
||||
mlir::LogicalResult verifyDeferredProgramContract(
|
||||
SpatDeferredCommunicationOp deferred);
|
||||
mlir::FailureOr<SpecializedDeferredProgram> analyzeDeferredProgram(
|
||||
SpatDeferredCommunicationOp deferred,
|
||||
std::optional<unsigned> targetScheduledLane);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include "ScheduledComputeMaterialization.hpp"
|
||||
#include "ScheduledComputeReport.hpp"
|
||||
#include "ScheduledComputeVerification.hpp"
|
||||
#include "SpatialDataflowCsvExporter.hpp"
|
||||
#include "DeferredCommunicationRealization.hpp"
|
||||
#include "DeferredCommunicationDeadlock.hpp"
|
||||
|
||||
@@ -82,6 +83,13 @@ struct MergeComputeNodesPass final : PassWrapper<MergeComputeNodesPass, Operatio
|
||||
return;
|
||||
}
|
||||
|
||||
SpatialDataflowExportStage exportMode = getSpatialDataflowExportStage();
|
||||
if (shouldExportSpatialDataflowStage(exportMode, SpatialDataflowExportStage::Spatial2)
|
||||
&& failed(exportSpatialDataflowCsvScheduled(funcOp, "spatial2_scheduled_no_comm", "spatial2"))) {
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
dumpScheduledComputeReportAndModule(moduleOp,
|
||||
funcOp,
|
||||
schedule,
|
||||
@@ -92,6 +100,7 @@ struct MergeComputeNodesPass final : PassWrapper<MergeComputeNodesPass, Operatio
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
dumpModule(moduleOp, "spatial3_scheduled", /*assumeVerified=*/true);
|
||||
if (failed(verifyRealizedCommunicationDeadlockFree(funcOp))) {
|
||||
moduleOp.emitError("MergeComputeNodes final communication verification failed");
|
||||
signalPassFailure();
|
||||
@@ -100,6 +109,11 @@ struct MergeComputeNodesPass final : PassWrapper<MergeComputeNodesPass, Operatio
|
||||
if (failed(verifyScheduledSpatialInvariants(funcOp))) {
|
||||
moduleOp.emitError("scheduled Spatial phase 2 verification failed");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
if (shouldExportSpatialDataflowStage(exportMode, SpatialDataflowExportStage::Spatial3)
|
||||
&& failed(exportSpatialDataflowCsvScheduled(funcOp, "spatial3_scheduled", "spatial3"))) {
|
||||
signalPassFailure();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
+31
-29
@@ -204,7 +204,6 @@ static FailureOr<Value> buildSourceLaneStartForScheduledLane(OpBuilder &builder,
|
||||
Value scheduledLane,
|
||||
const SourceLaneSelector &selector,
|
||||
Operation *constantAnchor) {
|
||||
(void)constantAnchor;
|
||||
if (selector.kind == SourceLaneSelector::Kind::Affine) {
|
||||
if (selector.affine.baseLaneStart == 0 && selector.affine.laneCount == 1)
|
||||
return scheduledLane;
|
||||
@@ -214,9 +213,7 @@ static FailureOr<Value> buildSourceLaneStartForScheduledLane(OpBuilder &builder,
|
||||
expr = d0 * selector.affine.laneCount;
|
||||
if (selector.affine.baseLaneStart != 0)
|
||||
expr = expr + selector.affine.baseLaneStart;
|
||||
return affine::AffineApplyOp::create(
|
||||
builder, loc, AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, expr), ValueRange {scheduledLane})
|
||||
.getResult();
|
||||
return createOrFoldAffineApply(builder, loc, expr, ValueRange {scheduledLane}, constantAnchor);
|
||||
}
|
||||
|
||||
if (!selector.table)
|
||||
@@ -569,18 +566,13 @@ static SmallVector<OpFoldResult> buildScheduledOutputInsertOffsets(OpBuilder &bu
|
||||
Location loc,
|
||||
Value scheduledLane,
|
||||
int64_t lanesPerScheduledLane,
|
||||
RankedTensorType localFragmentType) {
|
||||
RankedTensorType localFragmentType,
|
||||
Operation *constantAnchor) {
|
||||
SmallVector<OpFoldResult> offsets;
|
||||
Value scheduledOutputLane = scheduledLane;
|
||||
if (lanesPerScheduledLane != 1) {
|
||||
scheduledOutputLane =
|
||||
affine::AffineApplyOp::create(builder,
|
||||
loc,
|
||||
AffineMap::get(/*dimCount=*/1,
|
||||
/*symbolCount=*/0,
|
||||
builder.getAffineDimExpr(0) * lanesPerScheduledLane),
|
||||
ValueRange {scheduledLane})
|
||||
.getResult();
|
||||
scheduledOutputLane = affineMulConst(
|
||||
builder, loc, scheduledLane, lanesPerScheduledLane, constantAnchor);
|
||||
}
|
||||
offsets.push_back(scheduledOutputLane);
|
||||
offsets.append(localFragmentType.getRank() - 1, OpFoldResult(builder.getIndexAttr(0)));
|
||||
@@ -707,14 +699,12 @@ static LogicalResult materializeMultiCpuPeftClass(
|
||||
[&](OpBuilder &builder, Location bodyLoc, Value innerLane, ValueRange iterArgs, SmallVectorImpl<Value> &yielded) -> LogicalResult {
|
||||
|
||||
IRMapping mapper;
|
||||
Value sourceLane =
|
||||
affine::AffineApplyOp::create(builder,
|
||||
bodyLoc,
|
||||
AffineMap::get(/*dimCount=*/2,
|
||||
/*symbolCount=*/0,
|
||||
builder.getAffineDimExpr(0) + builder.getAffineDimExpr(1)),
|
||||
ValueRange {*sourceLaneStart, innerLane})
|
||||
.getResult();
|
||||
Value sourceLane = createOrFoldAffineApply(
|
||||
builder,
|
||||
bodyLoc,
|
||||
builder.getAffineDimExpr(0) + builder.getAffineDimExpr(1),
|
||||
ValueRange {*sourceLaneStart, innerLane},
|
||||
scheduled.getOperation());
|
||||
mapper.map(*batch.getLaneArgument(), sourceLane);
|
||||
for (auto [index, weight] : llvm::enumerate(batch.getWeights()))
|
||||
mapper.map(*batch.getWeightArgument(index),
|
||||
@@ -789,8 +779,13 @@ static LogicalResult materializeMultiCpuPeftClass(
|
||||
finalLocalFragments.assign(loop->results.begin(), loop->results.end());
|
||||
}
|
||||
|
||||
auto inParallel = SpatInParallelOp::create(rewriter, scheduled.getLoc());
|
||||
rewriter.setInsertionPointToStart(&inParallel.getRegion().front());
|
||||
struct Publication {
|
||||
Value fragment;
|
||||
SmallVector<OpFoldResult> offsets;
|
||||
SmallVector<OpFoldResult> sizes;
|
||||
SmallVector<OpFoldResult> strides;
|
||||
};
|
||||
SmallVector<Publication> publications;
|
||||
for (auto [resultIndex, localFragment] : llvm::enumerate(finalLocalFragments)) {
|
||||
auto localFragmentType = cast<RankedTensorType>(localFragment.getType());
|
||||
int64_t lanesPerScheduledLane = isa<SpatCompute>(representative.op) ? 1 : representative.laneCount;
|
||||
@@ -799,22 +794,29 @@ static LogicalResult materializeMultiCpuPeftClass(
|
||||
scheduled.getLoc(),
|
||||
scheduledLane,
|
||||
lanesPerScheduledLane,
|
||||
localFragmentType);
|
||||
localFragmentType,
|
||||
scheduled.getOperation());
|
||||
SmallVector<OpFoldResult> sizes;
|
||||
SmallVector<OpFoldResult> strides;
|
||||
for (int64_t dim : localFragmentType.getShape()) {
|
||||
sizes.push_back(rewriter.getIndexAttr(dim));
|
||||
strides.push_back(rewriter.getIndexAttr(1));
|
||||
}
|
||||
publications.push_back(
|
||||
{localFragment, std::move(offsets), std::move(sizes),
|
||||
std::move(strides)});
|
||||
}
|
||||
auto inParallel = SpatInParallelOp::create(rewriter, scheduled.getLoc());
|
||||
rewriter.setInsertionPointToStart(&inParallel.getRegion().front());
|
||||
for (auto [resultIndex, publication] : llvm::enumerate(publications))
|
||||
tensor::ParallelInsertSliceOp::create(
|
||||
rewriter,
|
||||
scheduled.getLoc(),
|
||||
localFragment,
|
||||
publication.fragment,
|
||||
block->getArgument(getScheduledBatchResultArgBase(scheduled) + stepPlan.resultOffset + resultIndex),
|
||||
offsets,
|
||||
sizes,
|
||||
strides);
|
||||
}
|
||||
publication.offsets,
|
||||
publication.sizes,
|
||||
publication.strides);
|
||||
}
|
||||
return success();
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
#include "Scheduling/ComputeInstanceUtils.hpp"
|
||||
#include "Scheduling/MergeSchedulingAnalysis.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
|
||||
|
||||
namespace onnx_mlir {
|
||||
namespace spatial {
|
||||
@@ -267,11 +268,9 @@ inline SmallVector<uint32_t> collectSourceLaneStarts(const ComputeStepTuple &ste
|
||||
}
|
||||
|
||||
inline Value createI64LookupTableConstant(OpBuilder &builder, Operation *constantAnchor, ArrayRef<int64_t> values) {
|
||||
OpBuilder::InsertionGuard guard(builder);
|
||||
builder.setInsertionPoint(constantAnchor);
|
||||
RankedTensorType tableType = RankedTensorType::get({static_cast<int64_t>(values.size())}, builder.getI64Type());
|
||||
DenseElementsAttr tableAttr = DenseElementsAttr::get(tableType, values);
|
||||
return arith::ConstantOp::create(builder, constantAnchor->getLoc(), tableType, tableAttr).getResult();
|
||||
return getOrCreateConstant(builder, constantAnchor, tableAttr, tableType);
|
||||
}
|
||||
|
||||
inline FailureOr<Value> createEmptyTensorForType(OpBuilder &builder, Location loc, Type type) {
|
||||
|
||||
+17
-4
@@ -82,24 +82,36 @@ LogicalResult verifyDeferredTransferPhase1Invariants(func::FuncOp funcOp) {
|
||||
pim::CappedDiagnosticReporter diagnostics;
|
||||
GraphBatchPublicationCache publicationCache;
|
||||
funcOp.walk([&](SpatDeferredCommunicationOp transfer) {
|
||||
bool ownershipValid = true;
|
||||
for (Value source : transfer.getSources()) {
|
||||
auto result = dyn_cast<OpResult>(source);
|
||||
if (!result || !isa<SpatGraphCompute, SpatGraphComputeBatch>(result.getOwner())) {
|
||||
ownershipValid = false;
|
||||
diagnostics.report(transfer.getOperation(), [&](Operation *illegalOp) {
|
||||
illegalOp->emitOpError("phase-check deferred communication source operand must be an original graph SSA result");
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!transfer->getParentOfType<SpatScheduledCompute>() &&
|
||||
!transfer->getParentOfType<SpatScheduledComputeBatch>())
|
||||
!transfer->getParentOfType<SpatScheduledComputeBatch>()) {
|
||||
ownershipValid = false;
|
||||
diagnostics.report(transfer.getOperation(), [&](Operation *illegalOp) {
|
||||
illegalOp->emitOpError("phase-check deferred communication must be inside a scheduled compute");
|
||||
});
|
||||
}
|
||||
if (!ownershipValid)
|
||||
return;
|
||||
if (failed(verifyDeferredProgramContract(transfer))) {
|
||||
diagnostics.report(transfer.getOperation(), [&](Operation *) {});
|
||||
return;
|
||||
}
|
||||
if (auto scheduled = transfer->getParentOfType<SpatScheduledComputeBatch>()) {
|
||||
for (unsigned lane = 0; lane < static_cast<unsigned>(scheduled.getLaneCount()); ++lane) {
|
||||
auto program = analyzeDeferredProgram(transfer, lane);
|
||||
if (failed(program))
|
||||
if (failed(program)) {
|
||||
diagnostics.report(transfer.getOperation(), [&](Operation *) {});
|
||||
continue;
|
||||
}
|
||||
for (const DeferredProjectionLeaf &leaf : program->leaves) {
|
||||
if (leaf.kind == DeferredLeafKind::ScalarSource)
|
||||
continue;
|
||||
@@ -110,8 +122,9 @@ LogicalResult verifyDeferredTransferPhase1Invariants(func::FuncOp funcOp) {
|
||||
diagnostics.report(transfer.getOperation(), [&](Operation *) {});
|
||||
}
|
||||
}
|
||||
} else
|
||||
(void)analyzeDeferredProgram(transfer, std::nullopt);
|
||||
} else if (failed(analyzeDeferredProgram(transfer, std::nullopt))) {
|
||||
diagnostics.report(transfer.getOperation(), [&](Operation *) {});
|
||||
}
|
||||
});
|
||||
|
||||
diagnostics.emitSuppressedSummary(funcOp, "scheduled Spatial deferred communication verification failed");
|
||||
|
||||
@@ -0,0 +1,898 @@
|
||||
#include "mlir/Dialect/Affine/IR/AffineOps.h"
|
||||
#include "mlir/Dialect/Arith/IR/Arith.h"
|
||||
#include "mlir/Dialect/Tensor/IR/Tensor.h"
|
||||
#include "mlir/IR/AsmState.h"
|
||||
#include "mlir/IR/BuiltinAttributes.h"
|
||||
#include "mlir/IR/BuiltinOps.h"
|
||||
#include "mlir/IR/BuiltinTypes.h"
|
||||
#include "mlir/IR/Value.h"
|
||||
|
||||
#include "llvm/ADT/DenseMap.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
#include "llvm/Support/Casting.h"
|
||||
#include "llvm/Support/ErrorHandling.h"
|
||||
#include "llvm/Support/raw_ostream.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "SpatialDataflowCsvExporter.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/BatchCoreUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/DebugDump.hpp"
|
||||
#include "src/Accelerators/PIM/Compiler/PimCompilerOptions.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
|
||||
using namespace mlir;
|
||||
|
||||
namespace onnx_mlir {
|
||||
namespace spatial {
|
||||
|
||||
namespace {
|
||||
|
||||
struct TopLevelOpInfo {
|
||||
Operation* op = nullptr;
|
||||
size_t opId = 0;
|
||||
bool isScheduled = false;
|
||||
std::optional<int32_t> scalarCore;
|
||||
};
|
||||
|
||||
struct ExpandedNodeInfo {
|
||||
std::string id;
|
||||
std::optional<int32_t> core;
|
||||
std::optional<uint32_t> lane;
|
||||
};
|
||||
|
||||
struct ChannelSendRecord {
|
||||
std::string sourceId;
|
||||
std::optional<uint32_t> sourceLane;
|
||||
};
|
||||
|
||||
enum class LogicalNodeSelector {
|
||||
Scalar,
|
||||
Lane,
|
||||
RangeRepresentative,
|
||||
};
|
||||
|
||||
struct ResolvedProducer {
|
||||
Operation* op = nullptr;
|
||||
size_t resultIndex = 0;
|
||||
LogicalNodeSelector selector = LogicalNodeSelector::Scalar;
|
||||
uint32_t lane = 0;
|
||||
uint32_t laneStart = 0;
|
||||
uint32_t laneCount = 1;
|
||||
};
|
||||
|
||||
struct EdgeSource {
|
||||
std::string id;
|
||||
std::optional<uint32_t> sourceLane;
|
||||
};
|
||||
|
||||
using ScheduledNodeByGraphLane = DenseMap<std::pair<int64_t, uint32_t>, ExpandedNodeInfo>;
|
||||
|
||||
void emitEdgeRow(std::fstream& edgesFile,
|
||||
StringRef sourceId,
|
||||
StringRef targetId,
|
||||
std::optional<uint64_t> byteSize,
|
||||
Type propagatedType,
|
||||
StringRef stage,
|
||||
std::optional<uint32_t> sourceLane,
|
||||
std::optional<uint32_t> targetLane,
|
||||
std::optional<int64_t> channelId);
|
||||
|
||||
std::string csvEscape(StringRef field) {
|
||||
bool needsQuotes = field.contains(',') || field.contains('"') || field.contains('\n') || field.contains('\r');
|
||||
if (!needsQuotes)
|
||||
return field.str();
|
||||
|
||||
std::string escaped;
|
||||
escaped.reserve(field.size() + 2);
|
||||
escaped.push_back('"');
|
||||
for (char ch : field)
|
||||
if (ch == '"')
|
||||
escaped += "\"\"";
|
||||
else
|
||||
escaped.push_back(ch);
|
||||
escaped.push_back('"');
|
||||
return escaped;
|
||||
}
|
||||
|
||||
void writeCsvRow(std::fstream& file, ArrayRef<std::string> fields) {
|
||||
for (size_t i = 0; i < fields.size(); ++i) {
|
||||
if (i != 0)
|
||||
file << ",";
|
||||
file << csvEscape(fields[i]);
|
||||
}
|
||||
file << "\n";
|
||||
}
|
||||
|
||||
template <typename NumberT>
|
||||
std::string maybeNumber(std::optional<NumberT> value) {
|
||||
if (!value)
|
||||
return "";
|
||||
return std::to_string(*value);
|
||||
}
|
||||
|
||||
std::string stringifyType(Type type) {
|
||||
std::string storage;
|
||||
llvm::raw_string_ostream os(storage);
|
||||
type.print(os);
|
||||
return os.str();
|
||||
}
|
||||
|
||||
std::string stringifyValueAsOperand(Value value, AsmState& asmState) {
|
||||
std::string storage;
|
||||
llvm::raw_string_ostream os(storage);
|
||||
value.printAsOperand(os, asmState);
|
||||
return os.str();
|
||||
}
|
||||
|
||||
std::string stringifyResultSsaNames(Operation* op, AsmState* asmState) {
|
||||
if (!asmState || op->getNumResults() == 0)
|
||||
return "";
|
||||
|
||||
std::string storage;
|
||||
llvm::raw_string_ostream os(storage);
|
||||
llvm::interleave(
|
||||
op->getResults(), [&](Value result) { os << stringifyValueAsOperand(result, *asmState); }, [&]() { os << ";"; });
|
||||
return os.str();
|
||||
}
|
||||
|
||||
std::optional<uint64_t> getTypeSizeBytes(Type type) {
|
||||
if (auto shapedType = dyn_cast<ShapedType>(type)) {
|
||||
if (!shapedType.hasStaticShape() || !hasByteSizedElementType(shapedType.getElementType()))
|
||||
return std::nullopt;
|
||||
return static_cast<uint64_t>(getShapedTypeSizeInBytes(shapedType));
|
||||
}
|
||||
|
||||
if (isa<IndexType>(type))
|
||||
return static_cast<uint64_t>(getElementTypeSizeInBytes(type));
|
||||
if (auto intType = dyn_cast<IntegerType>(type)) {
|
||||
if (intType.getWidth() <= 0 || intType.getWidth() % 8 != 0)
|
||||
return std::nullopt;
|
||||
return static_cast<uint64_t>(getElementTypeSizeInBytes(type));
|
||||
}
|
||||
if (auto floatType = dyn_cast<FloatType>(type)) {
|
||||
if (floatType.getWidth() <= 0 || floatType.getWidth() % 8 != 0)
|
||||
return std::nullopt;
|
||||
return static_cast<uint64_t>(getElementTypeSizeInBytes(type));
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::string getScalarId(bool isScheduled, size_t opId) { return (isScheduled ? "sc:" : "gc:") + std::to_string(opId); }
|
||||
|
||||
std::string getBatchLaneId(bool isScheduled, size_t opId, uint32_t lane) {
|
||||
return (isScheduled ? "scb:" : "gcb:") + std::to_string(opId) + ":" + std::to_string(lane);
|
||||
}
|
||||
|
||||
template <typename ComputeOpTy, typename BatchOpTy>
|
||||
bool isTopLevelRelevantCompute(Operation& op) {
|
||||
return isa<ComputeOpTy, BatchOpTy>(&op);
|
||||
}
|
||||
|
||||
template <typename ComputeOpTy, typename BatchOpTy>
|
||||
FailureOr<TopLevelOpInfo> buildTopLevelOpInfo(Operation& op, bool isScheduled, size_t opId) {
|
||||
TopLevelOpInfo info;
|
||||
info.op = &op;
|
||||
info.opId = opId;
|
||||
info.isScheduled = isScheduled;
|
||||
|
||||
if constexpr (std::is_same_v<ComputeOpTy, SpatScheduledCompute>) {
|
||||
if (auto compute = dyn_cast<ComputeOpTy>(&op)) {
|
||||
auto coreId = getOptionalScheduledCoreId(compute, "spatial dataflow export core id");
|
||||
if (failed(coreId))
|
||||
return failure();
|
||||
if (*coreId)
|
||||
info.scalarCore = **coreId;
|
||||
}
|
||||
}
|
||||
|
||||
return info;
|
||||
}
|
||||
|
||||
template <typename BatchOpTy>
|
||||
FailureOr<SmallVector<int32_t, 8>> getBatchLaneCoreIds(BatchOpTy batch) {
|
||||
if constexpr (std::is_same_v<BatchOpTy, SpatScheduledComputeBatch>) {
|
||||
auto coreIds = getOptionalScheduledBatchCoreIds(batch, "spatial dataflow export core ids");
|
||||
if (failed(coreIds))
|
||||
return failure();
|
||||
if (!*coreIds)
|
||||
return SmallVector<int32_t, 8> {};
|
||||
return SmallVector<int32_t, 8>((**coreIds).begin(), (**coreIds).end());
|
||||
}
|
||||
return SmallVector<int32_t, 8> {};
|
||||
}
|
||||
|
||||
std::string getExpandedNodeId(const DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes,
|
||||
Operation* op,
|
||||
uint32_t lane) {
|
||||
auto it = expandedNodes.find({op, lane});
|
||||
if (it == expandedNodes.end())
|
||||
return "";
|
||||
return it->second.id;
|
||||
}
|
||||
|
||||
void addScalarNodeRow(std::fstream& nodesFile,
|
||||
DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes,
|
||||
const TopLevelOpInfo& info,
|
||||
AsmState* asmState = nullptr) {
|
||||
std::string id = getScalarId(info.isScheduled, info.opId);
|
||||
SmallVector<std::string, 5> row {id, std::to_string(info.opId), "", maybeNumber<int32_t>(info.scalarCore)};
|
||||
if (asmState)
|
||||
row.push_back(stringifyResultSsaNames(info.op, asmState));
|
||||
writeCsvRow(nodesFile, row);
|
||||
expandedNodes[{info.op, 0}] = {id, info.scalarCore, std::nullopt};
|
||||
}
|
||||
|
||||
template <typename BatchOpTy>
|
||||
void addBatchNodeRows(std::fstream& nodesFile,
|
||||
DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes,
|
||||
const TopLevelOpInfo& info,
|
||||
BatchOpTy batch,
|
||||
ArrayRef<std::optional<int32_t>> laneCoreIds,
|
||||
AsmState* asmState = nullptr) {
|
||||
for (uint32_t lane = 0; lane < static_cast<uint32_t>(batch.getLaneCount()); ++lane) {
|
||||
std::string id = getBatchLaneId(info.isScheduled, info.opId, lane);
|
||||
SmallVector<std::string, 5> row {
|
||||
id, std::to_string(info.opId), std::to_string(lane), maybeNumber<int32_t>(laneCoreIds[lane])};
|
||||
if (asmState)
|
||||
row.push_back(stringifyResultSsaNames(info.op, asmState));
|
||||
writeCsvRow(nodesFile, row);
|
||||
expandedNodes[{info.op, lane}] = {id, laneCoreIds[lane], lane};
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<int64_t> evaluateIndexLike(Value value, Value laneArg, uint32_t lane);
|
||||
|
||||
std::optional<int64_t> evaluateIndexLike(Value value, Value laneArg, uint32_t lane) {
|
||||
if (value == laneArg)
|
||||
return static_cast<int64_t>(lane);
|
||||
|
||||
if (std::optional<int64_t> constant = matchConstantIndexValue(value))
|
||||
return *constant;
|
||||
|
||||
if (auto constant = value.getDefiningOp<arith::ConstantOp>()) {
|
||||
if (auto intAttr = dyn_cast<IntegerAttr>(constant.getValue()))
|
||||
return intAttr.getInt();
|
||||
}
|
||||
|
||||
if (auto extract = value.getDefiningOp<tensor::ExtractOp>()) {
|
||||
auto constant = extract.getTensor().getDefiningOp<arith::ConstantOp>();
|
||||
auto elements = constant ? dyn_cast<ElementsAttr>(constant.getValue()) : nullptr;
|
||||
auto shapedType = elements ? dyn_cast<ShapedType>(elements.getType()) : nullptr;
|
||||
if (!elements || !shapedType || shapedType.getRank() != 1 || extract.getIndices().size() != 1)
|
||||
return std::nullopt;
|
||||
|
||||
std::optional<int64_t> index = evaluateIndexLike(extract.getIndices().front(), laneArg, lane);
|
||||
if (!index || *index < 0 || *index >= static_cast<int64_t>(elements.getNumElements()))
|
||||
return std::nullopt;
|
||||
|
||||
if (auto denseInts = dyn_cast<DenseIntElementsAttr>(elements))
|
||||
return (*(denseInts.value_begin<APInt>() + *index)).getSExtValue();
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (auto affineApply = value.getDefiningOp<affine::AffineApplyOp>())
|
||||
if (FailureOr<int64_t> folded = evaluateAffineApply(affineApply,
|
||||
[&](Value operand) -> FailureOr<int64_t> {
|
||||
if (std::optional<int64_t> resolved =
|
||||
evaluateIndexLike(operand, laneArg, lane))
|
||||
return *resolved;
|
||||
return failure();
|
||||
});
|
||||
succeeded(folded)) {
|
||||
return *folded;
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
SmallVector<int64_t, 8> collectPossibleIntValues(Value value, Value laneArg, uint32_t lane) {
|
||||
if (std::optional<int64_t> exact = evaluateIndexLike(value, laneArg, lane))
|
||||
return {*exact};
|
||||
|
||||
auto extract = value.getDefiningOp<tensor::ExtractOp>();
|
||||
auto constant = extract ? extract.getTensor().getDefiningOp<arith::ConstantOp>() : nullptr;
|
||||
auto elements = constant ? dyn_cast<ElementsAttr>(constant.getValue()) : nullptr;
|
||||
if (!elements)
|
||||
return {};
|
||||
|
||||
SmallVector<int64_t, 8> values;
|
||||
if (auto denseInts = dyn_cast<DenseIntElementsAttr>(elements)) {
|
||||
values.reserve(elements.getNumElements());
|
||||
for (APInt element : denseInts.getValues<APInt>())
|
||||
if (!llvm::is_contained(values, element.getSExtValue()))
|
||||
values.push_back(element.getSExtValue());
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
template <typename BatchOpTy>
|
||||
std::optional<Value> getBatchLaneInput(BatchOpTy batch, uint32_t lane, unsigned inputIndex) {
|
||||
if (batch.getNumResults() != 0)
|
||||
return batch.getInputs()[inputIndex];
|
||||
|
||||
size_t laneCount = static_cast<size_t>(batch.getLaneCount());
|
||||
if (laneCount == 0 || batch.getInputs().size() % laneCount != 0)
|
||||
return std::nullopt;
|
||||
|
||||
size_t inputsPerLane = batch.getInputs().size() / laneCount;
|
||||
size_t flatIndex = static_cast<size_t>(lane) * inputsPerLane + inputIndex;
|
||||
if (flatIndex >= batch.getInputs().size())
|
||||
return std::nullopt;
|
||||
return batch.getInputs()[flatIndex];
|
||||
}
|
||||
|
||||
template <typename BatchOpTy>
|
||||
unsigned getBatchLaneInputCount(BatchOpTy batch) {
|
||||
if (batch.getNumResults() != 0)
|
||||
return batch.getInputs().size();
|
||||
|
||||
size_t laneCount = static_cast<size_t>(batch.getLaneCount());
|
||||
if (laneCount == 0 || batch.getInputs().size() % laneCount != 0)
|
||||
return 0;
|
||||
return static_cast<unsigned>(batch.getInputs().size() / laneCount);
|
||||
}
|
||||
|
||||
template <typename ComputeOpTy, typename BatchOpTy>
|
||||
std::optional<ResolvedProducer> resolveProducerForValue(Value value, std::optional<uint32_t> consumerLane) {
|
||||
Operation* op = value.getDefiningOp();
|
||||
if (!op)
|
||||
return std::nullopt;
|
||||
|
||||
while (auto extract = dyn_cast<tensor::ExtractSliceOp>(op)) {
|
||||
Value source = extract.getSource();
|
||||
Operation* sourceOp = source.getDefiningOp();
|
||||
auto sourceBatch = dyn_cast_or_null<BatchOpTy>(sourceOp);
|
||||
if (sourceBatch && sourceBatch.getNumResults() != 0) {
|
||||
auto staticOffsets = extract.getStaticOffsets();
|
||||
if (!staticOffsets.empty() && staticOffsets.front() != ShapedType::kDynamic) {
|
||||
uint32_t lane = static_cast<uint32_t>(staticOffsets.front());
|
||||
return ResolvedProducer {sourceOp, 0, LogicalNodeSelector::Lane, lane, lane, 1};
|
||||
}
|
||||
if (consumerLane)
|
||||
return ResolvedProducer {sourceOp, 0, LogicalNodeSelector::Lane, *consumerLane, *consumerLane, 1};
|
||||
return ResolvedProducer {
|
||||
sourceOp, 0, LogicalNodeSelector::RangeRepresentative, 0, 0, static_cast<uint32_t>(sourceBatch.getLaneCount())};
|
||||
}
|
||||
value = source;
|
||||
op = sourceOp;
|
||||
if (!op)
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
if (auto compute = dyn_cast<ComputeOpTy>(op))
|
||||
return ResolvedProducer {compute.getOperation(),
|
||||
static_cast<size_t>(cast<OpResult>(value).getResultNumber()),
|
||||
LogicalNodeSelector::Scalar,
|
||||
0,
|
||||
0,
|
||||
1};
|
||||
|
||||
if (auto batch = dyn_cast<BatchOpTy>(op)) {
|
||||
if (batch.getNumResults() != 0) {
|
||||
if (consumerLane)
|
||||
return ResolvedProducer {op, 0, LogicalNodeSelector::Lane, *consumerLane, *consumerLane, 1};
|
||||
return ResolvedProducer {
|
||||
op, 0, LogicalNodeSelector::RangeRepresentative, 0, 0, static_cast<uint32_t>(batch.getLaneCount())};
|
||||
}
|
||||
|
||||
uint32_t lane = static_cast<uint32_t>(cast<OpResult>(value).getResultNumber());
|
||||
return ResolvedProducer {op, static_cast<size_t>(lane), LogicalNodeSelector::Lane, lane, lane, 1};
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
SmallVector<EdgeSource, 8>
|
||||
resolveProducerSourcesForCsv(const ResolvedProducer& producer,
|
||||
const DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes) {
|
||||
SmallVector<EdgeSource, 8> sources;
|
||||
|
||||
if (producer.selector == LogicalNodeSelector::Scalar) {
|
||||
std::string id = getExpandedNodeId(expandedNodes, producer.op, 0);
|
||||
if (!id.empty())
|
||||
sources.push_back({id, std::nullopt});
|
||||
return sources;
|
||||
}
|
||||
|
||||
if (producer.selector == LogicalNodeSelector::Lane) {
|
||||
std::string id = getExpandedNodeId(expandedNodes, producer.op, producer.lane);
|
||||
if (!id.empty())
|
||||
sources.push_back({id, producer.lane});
|
||||
return sources;
|
||||
}
|
||||
|
||||
for (uint32_t lane = producer.laneStart; lane < producer.laneStart + producer.laneCount; ++lane) {
|
||||
std::string id = getExpandedNodeId(expandedNodes, producer.op, lane);
|
||||
if (!id.empty())
|
||||
sources.push_back({id, lane});
|
||||
}
|
||||
return sources;
|
||||
}
|
||||
|
||||
FailureOr<SmallVector<int64_t>> getIntegerValues(Operation* op, StringRef name) {
|
||||
Attribute attr = op->getAttr(name);
|
||||
if (auto array = dyn_cast_or_null<DenseI64ArrayAttr>(attr))
|
||||
return SmallVector<int64_t>(array.asArrayRef());
|
||||
if (auto elements = dyn_cast_or_null<DenseIntElementsAttr>(attr))
|
||||
return SmallVector<int64_t>(elements.getValues<int64_t>());
|
||||
return op->emitOpError() << "expected " << name << " integer array for Spatial dataflow report";
|
||||
}
|
||||
|
||||
FailureOr<ScheduledNodeByGraphLane>
|
||||
buildScheduledNodesByGraphLane(const DenseMap<Operation*, TopLevelOpInfo>& topLevelInfo,
|
||||
const DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes,
|
||||
const DenseMap<int64_t, Operation*>& graphOpsById) {
|
||||
ScheduledNodeByGraphLane nodesByGraphLane;
|
||||
for (const auto& entry : topLevelInfo) {
|
||||
Operation* scheduledOp = entry.first;
|
||||
auto sourceIds = getIntegerValues(scheduledOp, "scheduled.step_source_ids");
|
||||
auto sourceStarts = getIntegerValues(scheduledOp, "scheduled.source_lane_starts");
|
||||
auto sourceCounts = getIntegerValues(scheduledOp, "scheduled.source_lane_counts");
|
||||
if (failed(sourceIds) || failed(sourceStarts) || failed(sourceCounts))
|
||||
return failure();
|
||||
|
||||
uint32_t scheduledLaneCount = 1;
|
||||
if (auto batch = dyn_cast<SpatScheduledComputeBatch>(scheduledOp))
|
||||
scheduledLaneCount = static_cast<uint32_t>(batch.getLaneCount());
|
||||
size_t expectedEntries = sourceIds->size() * scheduledLaneCount;
|
||||
if (sourceStarts->size() != expectedEntries || sourceCounts->size() != expectedEntries)
|
||||
return scheduledOp->emitOpError("inconsistent scheduling provenance arrays for Spatial dataflow report");
|
||||
|
||||
for (auto [step, graphId] : llvm::enumerate(*sourceIds)) {
|
||||
auto graphIt = graphOpsById.find(graphId);
|
||||
if (graphIt == graphOpsById.end())
|
||||
return scheduledOp->emitOpError() << "references unknown scheduled graph id " << graphId;
|
||||
bool graphIsBatch = isa<SpatGraphComputeBatch>(graphIt->second);
|
||||
for (uint32_t scheduledLane = 0; scheduledLane < scheduledLaneCount; ++scheduledLane) {
|
||||
auto nodeIt = expandedNodes.find({scheduledOp, scheduledLane});
|
||||
if (nodeIt == expandedNodes.end())
|
||||
continue;
|
||||
size_t index = step * scheduledLaneCount + scheduledLane;
|
||||
int64_t start = graphIsBatch ? (*sourceStarts)[index] : 0;
|
||||
int64_t count = graphIsBatch ? (*sourceCounts)[index] : 1;
|
||||
if (start < 0 || count < 0)
|
||||
return scheduledOp->emitOpError("negative scheduling provenance range for Spatial dataflow report");
|
||||
for (int64_t lane = start; lane < start + count; ++lane)
|
||||
nodesByGraphLane[{graphId, static_cast<uint32_t>(lane)}] = nodeIt->second;
|
||||
}
|
||||
}
|
||||
}
|
||||
return nodesByGraphLane;
|
||||
}
|
||||
|
||||
SmallVector<ExpandedNodeInfo, 8> resolveScheduledProducerNodes(const ResolvedProducer& producer,
|
||||
const ScheduledNodeByGraphLane& nodesByGraphLane) {
|
||||
SmallVector<ExpandedNodeInfo, 8> nodes;
|
||||
auto graphId = producer.op->getAttrOfType<IntegerAttr>("scheduled.graph_id");
|
||||
if (!graphId)
|
||||
return nodes;
|
||||
|
||||
uint32_t laneStart = producer.selector == LogicalNodeSelector::Scalar ? 0 : producer.laneStart;
|
||||
uint32_t laneCount = producer.selector == LogicalNodeSelector::RangeRepresentative ? producer.laneCount : 1;
|
||||
for (uint32_t lane = laneStart; lane < laneStart + laneCount; ++lane)
|
||||
if (auto it = nodesByGraphLane.find({graphId.getInt(), lane}); it != nodesByGraphLane.end())
|
||||
nodes.push_back(it->second);
|
||||
return nodes;
|
||||
}
|
||||
|
||||
LogicalResult
|
||||
emitScheduledPlanningEdges(std::fstream& edgesFile,
|
||||
func::FuncOp func,
|
||||
const DenseMap<Operation*, TopLevelOpInfo>& topLevelInfo,
|
||||
const DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes,
|
||||
StringRef stage) {
|
||||
DenseMap<int64_t, Operation*> graphOpsById;
|
||||
for (Operation& op : func.getBody().front())
|
||||
if (auto graphId = op.getAttrOfType<IntegerAttr>("scheduled.graph_id"))
|
||||
graphOpsById[graphId.getInt()] = &op;
|
||||
|
||||
auto nodesByGraphLane = buildScheduledNodesByGraphLane(topLevelInfo, expandedNodes, graphOpsById);
|
||||
if (failed(nodesByGraphLane))
|
||||
return failure();
|
||||
|
||||
auto emitMappedEdge =
|
||||
[&](const ResolvedProducer& producer, int64_t targetGraphId, uint32_t targetGraphLane, Type type) {
|
||||
auto targetIt = nodesByGraphLane->find({targetGraphId, targetGraphLane});
|
||||
if (targetIt == nodesByGraphLane->end())
|
||||
return;
|
||||
for (const ExpandedNodeInfo& source : resolveScheduledProducerNodes(producer, *nodesByGraphLane)) {
|
||||
if (source.id == targetIt->second.id)
|
||||
continue;
|
||||
emitEdgeRow(edgesFile,
|
||||
source.id,
|
||||
targetIt->second.id,
|
||||
getTypeSizeBytes(type),
|
||||
type,
|
||||
stage,
|
||||
source.lane,
|
||||
targetIt->second.lane,
|
||||
std::nullopt);
|
||||
}
|
||||
};
|
||||
|
||||
for (Operation& op : func.getBody().front()) {
|
||||
auto graphId = op.getAttrOfType<IntegerAttr>("scheduled.graph_id");
|
||||
if (!graphId)
|
||||
continue;
|
||||
if (auto compute = dyn_cast<SpatGraphCompute>(&op)) {
|
||||
for (Value input : compute.getInputs())
|
||||
if (auto producer = resolveProducerForValue<SpatGraphCompute, SpatGraphComputeBatch>(input, std::nullopt))
|
||||
emitMappedEdge(*producer, graphId.getInt(), 0, input.getType());
|
||||
continue;
|
||||
}
|
||||
auto batch = dyn_cast<SpatGraphComputeBatch>(&op);
|
||||
if (!batch)
|
||||
continue;
|
||||
unsigned inputCount = getBatchLaneInputCount(batch);
|
||||
for (uint32_t lane = 0; lane < static_cast<uint32_t>(batch.getLaneCount()); ++lane)
|
||||
for (unsigned inputIndex = 0; inputIndex < inputCount; ++inputIndex)
|
||||
if (std::optional<Value> input = getBatchLaneInput(batch, lane, inputIndex))
|
||||
if (auto producer = resolveProducerForValue<SpatGraphCompute, SpatGraphComputeBatch>(*input, lane))
|
||||
emitMappedEdge(*producer, graphId.getInt(), lane, input->getType());
|
||||
}
|
||||
return success();
|
||||
}
|
||||
|
||||
void emitEdgeRow(std::fstream& edgesFile,
|
||||
StringRef sourceId,
|
||||
StringRef targetId,
|
||||
std::optional<uint64_t> byteSize,
|
||||
Type propagatedType,
|
||||
StringRef stage,
|
||||
std::optional<uint32_t> sourceLane,
|
||||
std::optional<uint32_t> targetLane,
|
||||
std::optional<int64_t> channelId) {
|
||||
writeCsvRow(edgesFile,
|
||||
{sourceId.str(),
|
||||
targetId.str(),
|
||||
maybeNumber<uint64_t>(byteSize),
|
||||
stringifyType(propagatedType),
|
||||
stage.str(),
|
||||
maybeNumber<uint32_t>(sourceLane),
|
||||
maybeNumber<uint32_t>(targetLane),
|
||||
maybeNumber<int64_t>(channelId)});
|
||||
}
|
||||
|
||||
template <typename ComputeOpTy, typename BatchOpTy>
|
||||
LogicalResult emitDataEdges(std::fstream& edgesFile,
|
||||
const DenseMap<Operation*, TopLevelOpInfo>& topLevelInfo,
|
||||
const DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes,
|
||||
StringRef stage) {
|
||||
for (const auto& entry : topLevelInfo) {
|
||||
Operation* op = entry.first;
|
||||
const TopLevelOpInfo& info = entry.second;
|
||||
|
||||
if (auto compute = dyn_cast<ComputeOpTy>(op)) {
|
||||
for (Value input : compute.getInputs()) {
|
||||
if (isa_and_nonnull<SpatChannelReceiveOp>(input.getDefiningOp()))
|
||||
continue;
|
||||
|
||||
auto producer = resolveProducerForValue<ComputeOpTy, BatchOpTy>(input, std::nullopt);
|
||||
if (!producer)
|
||||
continue;
|
||||
|
||||
SmallVector<EdgeSource, 8> sources = resolveProducerSourcesForCsv(*producer, expandedNodes);
|
||||
std::optional<uint64_t> byteSize = getTypeSizeBytes(input.getType());
|
||||
std::string targetId = getScalarId(info.isScheduled, info.opId);
|
||||
for (const EdgeSource& source : sources)
|
||||
emitEdgeRow(edgesFile,
|
||||
source.id,
|
||||
targetId,
|
||||
byteSize,
|
||||
input.getType(),
|
||||
stage,
|
||||
source.sourceLane,
|
||||
std::nullopt,
|
||||
std::nullopt);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
auto batch = dyn_cast<BatchOpTy>(op);
|
||||
if (!batch)
|
||||
continue;
|
||||
|
||||
unsigned inputCount = getBatchLaneInputCount(batch);
|
||||
for (uint32_t lane = 0; lane < static_cast<uint32_t>(batch.getLaneCount()); ++lane) {
|
||||
std::string targetId = getBatchLaneId(info.isScheduled, info.opId, lane);
|
||||
for (unsigned inputIndex = 0; inputIndex < inputCount; ++inputIndex) {
|
||||
std::optional<Value> input = getBatchLaneInput(batch, lane, inputIndex);
|
||||
if (!input || isa_and_nonnull<SpatChannelReceiveOp>((*input).getDefiningOp()))
|
||||
continue;
|
||||
|
||||
auto producer = resolveProducerForValue<ComputeOpTy, BatchOpTy>(*input, lane);
|
||||
if (!producer)
|
||||
continue;
|
||||
|
||||
SmallVector<EdgeSource, 8> sources = resolveProducerSourcesForCsv(*producer, expandedNodes);
|
||||
std::optional<uint64_t> byteSize = getTypeSizeBytes((*input).getType());
|
||||
for (const EdgeSource& source : sources)
|
||||
emitEdgeRow(
|
||||
edgesFile, source.id, targetId, byteSize, (*input).getType(), stage, source.sourceLane, lane, std::nullopt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return success();
|
||||
}
|
||||
|
||||
template <typename BatchOpTy>
|
||||
void collectChannelSends(DenseMap<int64_t, SmallVector<ChannelSendRecord, 4>>& sendsByChannelId,
|
||||
const DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes,
|
||||
BatchOpTy batch) {
|
||||
std::optional<BlockArgument> laneArg = batch.getLaneArgument();
|
||||
if (!laneArg)
|
||||
return;
|
||||
|
||||
for (uint32_t lane = 0; lane < static_cast<uint32_t>(batch.getLaneCount()); ++lane) {
|
||||
std::string sourceId = getExpandedNodeId(expandedNodes, batch.getOperation(), lane);
|
||||
if (sourceId.empty())
|
||||
continue;
|
||||
batch.getBody().walk([&](SpatChannelSendOp send) {
|
||||
std::optional<int64_t> channelId = evaluateIndexLike(send.getChannelId(), *laneArg, lane);
|
||||
if (!channelId)
|
||||
return;
|
||||
sendsByChannelId[*channelId].push_back({sourceId, lane});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void collectChannelSends(DenseMap<int64_t, SmallVector<ChannelSendRecord, 4>>& sendsByChannelId,
|
||||
const DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes,
|
||||
SpatScheduledCompute compute) {
|
||||
std::string sourceId = getExpandedNodeId(expandedNodes, compute.getOperation(), 0);
|
||||
if (sourceId.empty())
|
||||
return;
|
||||
compute.getBody().walk([&](SpatChannelSendOp send) {
|
||||
std::optional<int64_t> channelId = evaluateIndexLike(send.getChannelId(), Value(), 0);
|
||||
if (!channelId)
|
||||
return;
|
||||
sendsByChannelId[*channelId].push_back({sourceId, std::nullopt});
|
||||
});
|
||||
}
|
||||
|
||||
DenseMap<int32_t, SmallVector<ChannelSendRecord, 4>>
|
||||
buildNodesByCore(const DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo>& expandedNodes) {
|
||||
DenseMap<int32_t, SmallVector<ChannelSendRecord, 4>> nodesByCore;
|
||||
for (const auto& entry : expandedNodes) {
|
||||
const ExpandedNodeInfo& node = entry.second;
|
||||
if (!node.core)
|
||||
continue;
|
||||
nodesByCore[*node.core].push_back({node.id, node.lane});
|
||||
}
|
||||
return nodesByCore;
|
||||
}
|
||||
|
||||
template <typename ComputeOpTy, typename BatchOpTy, typename ResolveChannelSourcesFn>
|
||||
LogicalResult emitExplicitChannelEdges(std::fstream& edgesFile,
|
||||
const DenseMap<Operation*, TopLevelOpInfo>& topLevelInfo,
|
||||
ResolveChannelSourcesFn&& resolveChannelSources,
|
||||
StringRef stage) {
|
||||
for (const auto& entry : topLevelInfo) {
|
||||
Operation* op = entry.first;
|
||||
const TopLevelOpInfo& info = entry.second;
|
||||
|
||||
if (auto compute = dyn_cast<ComputeOpTy>(op)) {
|
||||
compute.getBody().walk([&](SpatChannelReceiveOp receive) {
|
||||
SmallVector<ChannelSendRecord, 4> sources = resolveChannelSources(receive, 0);
|
||||
if (sources.empty())
|
||||
return;
|
||||
std::optional<int64_t> channelId = evaluateIndexLike(receive.getChannelId(), Value(), 0);
|
||||
std::string targetId = getScalarId(info.isScheduled, info.opId);
|
||||
std::optional<uint64_t> byteSize = getTypeSizeBytes(receive.getType());
|
||||
for (const ChannelSendRecord& source : sources)
|
||||
emitEdgeRow(edgesFile,
|
||||
source.sourceId,
|
||||
targetId,
|
||||
byteSize,
|
||||
receive.getType(),
|
||||
stage,
|
||||
source.sourceLane,
|
||||
std::nullopt,
|
||||
channelId);
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
auto batch = dyn_cast<BatchOpTy>(op);
|
||||
if (!batch)
|
||||
continue;
|
||||
auto laneArg = batch.getLaneArgument();
|
||||
if (!laneArg)
|
||||
continue;
|
||||
for (uint32_t lane = 0; lane < static_cast<uint32_t>(batch.getLaneCount()); ++lane) {
|
||||
std::string targetId = getBatchLaneId(info.isScheduled, info.opId, lane);
|
||||
batch.getBody().walk([&](SpatChannelReceiveOp receive) {
|
||||
SmallVector<ChannelSendRecord, 4> sources = resolveChannelSources(receive, lane);
|
||||
if (sources.empty())
|
||||
return;
|
||||
std::optional<int64_t> channelId = evaluateIndexLike(receive.getChannelId(), *laneArg, lane);
|
||||
std::optional<uint64_t> byteSize = getTypeSizeBytes(receive.getType());
|
||||
for (const ChannelSendRecord& source : sources)
|
||||
emitEdgeRow(edgesFile,
|
||||
source.sourceId,
|
||||
targetId,
|
||||
byteSize,
|
||||
receive.getType(),
|
||||
stage,
|
||||
source.sourceLane,
|
||||
lane,
|
||||
channelId);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return success();
|
||||
}
|
||||
|
||||
LogicalResult exportGraph(func::FuncOp func, StringRef reportName) {
|
||||
std::fstream nodesFile = openDialectDumpFileWithExtension((reportName + ".nodes").str(), "/reports", "csv");
|
||||
std::fstream edgesFile = openDialectDumpFileWithExtension((reportName + ".edges").str(), "/reports", "csv");
|
||||
if (!nodesFile.is_open() || !edgesFile.is_open())
|
||||
return success();
|
||||
|
||||
writeCsvRow(nodesFile, {"Id", "op_id", "lane", "core", "ssa_name"});
|
||||
writeCsvRow(edgesFile, {"Source", "Target", "Weight", "Type", "stage", "source_lane", "target_lane", "channel_id"});
|
||||
|
||||
Operation* asmRoot = func.getOperation();
|
||||
if (auto moduleOp = func->getParentOfType<ModuleOp>())
|
||||
asmRoot = moduleOp.getOperation();
|
||||
OpPrintingFlags flags;
|
||||
flags.elideLargeElementsAttrs().enableDebugInfo(true, false);
|
||||
AsmState asmState(asmRoot, flags);
|
||||
|
||||
DenseMap<Operation*, TopLevelOpInfo> topLevelInfo;
|
||||
DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo> expandedNodes;
|
||||
|
||||
size_t opId = 0;
|
||||
for (Operation& op : func.getBody().front()) {
|
||||
if (!isTopLevelRelevantCompute<SpatGraphCompute, SpatGraphComputeBatch>(op))
|
||||
continue;
|
||||
FailureOr<TopLevelOpInfo> info = buildTopLevelOpInfo<SpatGraphCompute, SpatGraphComputeBatch>(op, false, opId++);
|
||||
if (failed(info))
|
||||
return failure();
|
||||
topLevelInfo[&op] = *info;
|
||||
|
||||
if (auto compute = dyn_cast<SpatGraphCompute>(&op)) {
|
||||
addScalarNodeRow(nodesFile, expandedNodes, *info, &asmState);
|
||||
continue;
|
||||
}
|
||||
|
||||
auto batch = cast<SpatGraphComputeBatch>(&op);
|
||||
SmallVector<std::optional<int32_t>, 8> laneCoreIds(batch.getLaneCount());
|
||||
addBatchNodeRows(nodesFile, expandedNodes, *info, batch, laneCoreIds, &asmState);
|
||||
}
|
||||
|
||||
return emitDataEdges<SpatGraphCompute, SpatGraphComputeBatch>(edgesFile, topLevelInfo, expandedNodes, "spatial1");
|
||||
}
|
||||
|
||||
LogicalResult exportScheduled(func::FuncOp func, StringRef reportName, StringRef stage) {
|
||||
std::fstream nodesFile = openDialectDumpFileWithExtension((reportName + ".nodes").str(), "/reports", "csv");
|
||||
std::fstream edgesFile = openDialectDumpFileWithExtension((reportName + ".edges").str(), "/reports", "csv");
|
||||
if (!nodesFile.is_open() || !edgesFile.is_open())
|
||||
return success();
|
||||
|
||||
writeCsvRow(nodesFile, {"Id", "op_id", "lane", "core", "ssa_name"});
|
||||
writeCsvRow(edgesFile, {"Source", "Target", "Weight", "Type", "stage", "source_lane", "target_lane", "channel_id"});
|
||||
|
||||
Operation* asmRoot = func.getOperation();
|
||||
if (auto moduleOp = func->getParentOfType<ModuleOp>())
|
||||
asmRoot = moduleOp.getOperation();
|
||||
OpPrintingFlags flags;
|
||||
flags.elideLargeElementsAttrs().enableDebugInfo(true, false);
|
||||
AsmState asmState(asmRoot, flags);
|
||||
|
||||
DenseMap<Operation*, TopLevelOpInfo> topLevelInfo;
|
||||
DenseMap<std::pair<Operation*, uint32_t>, ExpandedNodeInfo> expandedNodes;
|
||||
|
||||
size_t opId = 0;
|
||||
for (Operation& op : func.getBody().front()) {
|
||||
if (!isTopLevelRelevantCompute<SpatScheduledCompute, SpatScheduledComputeBatch>(op))
|
||||
continue;
|
||||
FailureOr<TopLevelOpInfo> info =
|
||||
buildTopLevelOpInfo<SpatScheduledCompute, SpatScheduledComputeBatch>(op, true, opId++);
|
||||
if (failed(info))
|
||||
return failure();
|
||||
topLevelInfo[&op] = *info;
|
||||
|
||||
if (isa<SpatScheduledCompute>(&op)) {
|
||||
addScalarNodeRow(nodesFile, expandedNodes, *info, &asmState);
|
||||
continue;
|
||||
}
|
||||
|
||||
auto batch = cast<SpatScheduledComputeBatch>(&op);
|
||||
auto coreIds = getBatchLaneCoreIds(batch);
|
||||
if (failed(coreIds))
|
||||
return failure();
|
||||
SmallVector<std::optional<int32_t>, 8> laneCoreIds(batch.getLaneCount());
|
||||
for (uint32_t lane = 0; lane < static_cast<uint32_t>(batch.getLaneCount()); ++lane)
|
||||
if (lane < coreIds->size())
|
||||
laneCoreIds[lane] = (*coreIds)[lane];
|
||||
addBatchNodeRows(nodesFile, expandedNodes, *info, batch, laneCoreIds, &asmState);
|
||||
}
|
||||
|
||||
if (stage == "spatial2")
|
||||
return emitScheduledPlanningEdges(edgesFile, func, topLevelInfo, expandedNodes, stage);
|
||||
if (failed(
|
||||
emitDataEdges<SpatScheduledCompute, SpatScheduledComputeBatch>(edgesFile, topLevelInfo, expandedNodes, stage)))
|
||||
return failure();
|
||||
|
||||
DenseMap<int64_t, SmallVector<ChannelSendRecord, 4>> sendsByChannelId;
|
||||
for (const auto& entry : topLevelInfo) {
|
||||
Operation* op = entry.first;
|
||||
if (auto compute = dyn_cast<SpatScheduledCompute>(op))
|
||||
collectChannelSends(sendsByChannelId, expandedNodes, compute);
|
||||
else if (auto batch = dyn_cast<SpatScheduledComputeBatch>(op))
|
||||
collectChannelSends(sendsByChannelId, expandedNodes, batch);
|
||||
}
|
||||
|
||||
DenseMap<int32_t, SmallVector<ChannelSendRecord, 4>> nodesByCore = buildNodesByCore(expandedNodes);
|
||||
auto resolveChannelSources = [&](SpatChannelReceiveOp receive, uint32_t lane) {
|
||||
SmallVector<ChannelSendRecord, 4> sources;
|
||||
|
||||
Value laneArg;
|
||||
if (auto owner = receive->getParentOfType<SpatScheduledComputeBatch>())
|
||||
if (auto maybeLaneArg = owner.getLaneArgument())
|
||||
laneArg = *maybeLaneArg;
|
||||
|
||||
if (std::optional<int64_t> channelId = evaluateIndexLike(receive.getChannelId(), laneArg, lane)) {
|
||||
if (auto it = sendsByChannelId.find(*channelId); it != sendsByChannelId.end())
|
||||
return it->second;
|
||||
}
|
||||
|
||||
for (int64_t sourceCore : collectPossibleIntValues(receive.getSourceCoreId(), laneArg, lane)) {
|
||||
auto it = nodesByCore.find(static_cast<int32_t>(sourceCore));
|
||||
if (it == nodesByCore.end())
|
||||
continue;
|
||||
llvm::append_range(sources, it->second);
|
||||
}
|
||||
return sources;
|
||||
};
|
||||
|
||||
return emitExplicitChannelEdges<SpatScheduledCompute, SpatScheduledComputeBatch>(
|
||||
edgesFile, topLevelInfo, resolveChannelSources, stage);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
SpatialDataflowExportStage getSpatialDataflowExportStage() {
|
||||
switch (pimExportSpatialDataflow.getValue()) {
|
||||
case SpatialDataflowExportNone: return SpatialDataflowExportStage::None;
|
||||
case SpatialDataflowExportSpatial1: return SpatialDataflowExportStage::Spatial1;
|
||||
case SpatialDataflowExportSpatial2: return SpatialDataflowExportStage::Spatial2;
|
||||
case SpatialDataflowExportSpatial3: return SpatialDataflowExportStage::Spatial3;
|
||||
case SpatialDataflowExportAll: return SpatialDataflowExportStage::All;
|
||||
}
|
||||
llvm_unreachable("unknown spatial dataflow export mode");
|
||||
}
|
||||
|
||||
bool shouldExportSpatialDataflowStage(SpatialDataflowExportStage mode, SpatialDataflowExportStage stage) {
|
||||
switch (mode) {
|
||||
case SpatialDataflowExportStage::None: return false;
|
||||
case SpatialDataflowExportStage::Spatial1: return stage == SpatialDataflowExportStage::Spatial1;
|
||||
case SpatialDataflowExportStage::Spatial2: return stage == SpatialDataflowExportStage::Spatial2;
|
||||
case SpatialDataflowExportStage::Spatial3: return stage == SpatialDataflowExportStage::Spatial3;
|
||||
case SpatialDataflowExportStage::All: return stage != SpatialDataflowExportStage::None;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
LogicalResult exportSpatialDataflowCsvGraph(func::FuncOp func, StringRef reportName) {
|
||||
return exportGraph(func, reportName);
|
||||
}
|
||||
|
||||
LogicalResult exportSpatialDataflowCsvScheduled(func::FuncOp func, StringRef reportName, StringRef stage) {
|
||||
return exportScheduled(func, reportName, stage);
|
||||
}
|
||||
|
||||
} // namespace spatial
|
||||
} // namespace onnx_mlir
|
||||
@@ -0,0 +1,28 @@
|
||||
#pragma once
|
||||
|
||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||
#include "mlir/Support/LogicalResult.h"
|
||||
|
||||
#include "llvm/ADT/StringRef.h"
|
||||
|
||||
namespace onnx_mlir {
|
||||
namespace spatial {
|
||||
|
||||
enum class SpatialDataflowExportStage {
|
||||
None,
|
||||
Spatial1,
|
||||
Spatial2,
|
||||
Spatial3,
|
||||
All,
|
||||
};
|
||||
|
||||
SpatialDataflowExportStage getSpatialDataflowExportStage();
|
||||
|
||||
mlir::LogicalResult exportSpatialDataflowCsvGraph(mlir::func::FuncOp func, llvm::StringRef reportName);
|
||||
mlir::LogicalResult
|
||||
exportSpatialDataflowCsvScheduled(mlir::func::FuncOp func, llvm::StringRef reportName, llvm::StringRef stage);
|
||||
|
||||
bool shouldExportSpatialDataflowStage(SpatialDataflowExportStage mode, SpatialDataflowExportStage stage);
|
||||
|
||||
} // namespace spatial
|
||||
} // namespace onnx_mlir
|
||||
Reference in New Issue
Block a user