#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" using namespace mlir; namespace onnx_mlir { namespace spatial { RankedTensorType getGraphBatchPhysicalResultType(int64_t laneCount, RankedTensorType fragmentType) { SmallVector shape {laneCount}; llvm::append_range(shape, fragmentType.getShape()); return RankedTensorType::get(shape, fragmentType.getElementType(), fragmentType.getEncoding()); } FailureOr getGraphBatchFragmentType(RankedTensorType physicalType, int64_t expectedLaneCount) { if (!physicalType || physicalType.getRank() < 1 || physicalType.getDimSize(0) != expectedLaneCount) return failure(); return RankedTensorType::get(physicalType.getShape().drop_front(), physicalType.getElementType(), physicalType.getEncoding()); } namespace { std::optional getBlockArgument(Region& body, unsigned argIdx) { if (body.empty()) return std::nullopt; Block& block = body.front(); if (argIdx >= block.getNumArguments()) return std::nullopt; return block.getArgument(argIdx); } std::optional insertBlockArgument(Region& body, unsigned argIdx, Type type, Location loc) { if (body.empty()) return std::nullopt; return body.insertArgument(argIdx, type, loc); } void setComputeOperandSegmentSizes(Operation* op, int32_t weightCount, int32_t inputCount) { if (auto compute = dyn_cast(op)) { compute.getProperties().setOperandSegmentSizes({weightCount, inputCount}); return; } if (auto compute = dyn_cast(op)) { compute.getProperties().setOperandSegmentSizes({weightCount, inputCount}); return; } if (auto batch = dyn_cast(op)) { batch.getProperties().setOperandSegmentSizes({weightCount, inputCount}); return; } cast(op).getProperties().setOperandSegmentSizes({weightCount, inputCount}); } using CrossbarWeightSet = llvm::SetVector, llvm::SmallDenseSet>; CrossbarWeightSet collectCrossbarWeights(Region& body) { CrossbarWeightSet weights; body.walk([&](SpatVMMOp vmmOp) { Value weight = vmmOp.getWeight(); weights.insert(weight); }); return weights; } template std::optional getComputeWeightArgument(ComputeOpTy compute, unsigned idx) { return getBlockArgument(compute.getBody(), idx); } template std::optional getComputeInputArgument(ComputeOpTy compute, unsigned idx) { return getBlockArgument(compute.getBody(), compute.getWeights().size() + idx); } template std::optional> insertComputeWeight(ComputeOpTy compute, unsigned idx, Value weight, Location loc) { if (auto existing = llvm::find(compute.getWeights(), weight); existing != compute.getWeights().end()) { auto index = std::distance(compute.getWeights().begin(), existing); return {{*existing, *getComputeWeightArgument(compute, index)}}; } unsigned weightCount = compute.getWeights().size(); unsigned inputCount = compute.getInputs().size(); compute.getOperation()->insertOperands(idx, ValueRange {weight}); setComputeOperandSegmentSizes( compute.getOperation(), static_cast(weightCount + 1), static_cast(inputCount)); auto blockArg = insertBlockArgument(compute.getBody(), idx, weight.getType(), loc); if (!blockArg) return std::nullopt; return std::make_tuple(compute.getOperation()->getOperand(idx), *blockArg); } template std::optional> insertComputeBatchWeight(ComputeBatchOpTy batch, unsigned idx, Value weight, Location loc) { if (auto existing = llvm::find(batch.getWeights(), weight); existing != batch.getWeights().end()) { auto index = std::distance(batch.getWeights().begin(), existing); return {{*existing, *batch.getWeightArgument(index)}}; } unsigned weightCount = batch.getWeights().size(); unsigned inputCount = batch.getInputs().size(); batch.getOperation()->insertOperands(idx, ValueRange {weight}); setComputeOperandSegmentSizes( batch.getOperation(), static_cast(weightCount + 1), static_cast(inputCount)); auto blockArg = insertBlockArgument(batch.getBody(), 1 + idx, weight.getType(), loc); if (!blockArg) return std::nullopt; return std::make_tuple(batch.getOperation()->getOperand(idx), *blockArg); } template std::optional> insertComputeInput(ComputeOpTy compute, unsigned idx, Value input, Location loc) { unsigned weightCount = compute.getWeights().size(); unsigned inputCount = compute.getInputs().size(); compute.getOperation()->insertOperands(weightCount + idx, ValueRange {input}); setComputeOperandSegmentSizes( compute.getOperation(), static_cast(weightCount), static_cast(inputCount + 1)); auto blockArg = insertBlockArgument(compute.getBody(), weightCount + idx, input.getType(), loc); if (!blockArg) return std::nullopt; return std::make_tuple(compute.getOperation()->getOperand(weightCount + idx), *blockArg); } template void setComputeAsmBlockArgumentNames(ComputeOpTy compute, Region& region, OpAsmSetValueNameFn setNameFn) { if (region.empty()) return; for (unsigned index = 0; index < compute.getWeights().size(); ++index) if (auto weightArg = compute.getWeightArgument(index)) setNameFn(*weightArg, ("w" + std::to_string(index)).c_str()); for (unsigned index = 0; index < compute.getInputs().size(); ++index) if (auto inputArg = compute.getInputArgument(index)) setNameFn(*inputArg, ("in" + std::to_string(index)).c_str()); } template FailureOr> insertComputeOutput(ComputeOpTy compute, RewriterBase& rewriter, unsigned idx, Type type, Location loc) { if (idx > compute.getNumResults()) return failure(); rewriter.setInsertionPoint(compute.getOperation()); SmallVector resultTypes(compute.getResultTypes().begin(), compute.getResultTypes().end()); resultTypes.insert(resultTypes.begin() + idx, type); auto newCompute = ComputeOpTy::create(rewriter, compute.getLoc(), TypeRange(resultTypes), compute.getWeights(), compute.getInputs()); newCompute->setAttrs(compute->getAttrs()); setComputeOperandSegmentSizes(newCompute.getOperation(), static_cast(newCompute.getWeights().size()), static_cast(newCompute.getInputs().size())); rewriter.inlineRegionBefore(compute.getBody(), newCompute.getBody(), newCompute.getBody().end()); for (unsigned oldResultIdx = 0; oldResultIdx < compute.getNumResults(); ++oldResultIdx) compute.getResult(oldResultIdx) .replaceAllUsesWith(newCompute.getResult(oldResultIdx < idx ? oldResultIdx : oldResultIdx + 1)); rewriter.eraseOp(compute.getOperation()); return std::make_tuple(cast(newCompute.getResult(idx)), newCompute); } template FailureOr> insertComputeBatchOutput(ComputeBatchOpTy batch, RewriterBase& rewriter, unsigned idx, Type type, Location loc) { if (idx > batch.getNumResults()) return failure(); rewriter.setInsertionPoint(batch.getOperation()); SmallVector resultTypes(batch.getResultTypes().begin(), batch.getResultTypes().end()); resultTypes.insert(resultTypes.begin() + idx, type); auto newBatch = ComputeBatchOpTy::create(rewriter, batch.getLoc(), TypeRange(resultTypes), batch.getLaneCountAttr(), batch.getWeights(), batch.getInputs()); newBatch->setAttrs(batch->getAttrs()); setComputeOperandSegmentSizes(newBatch.getOperation(), static_cast(newBatch.getWeights().size()), static_cast(newBatch.getInputs().size())); rewriter.inlineRegionBefore(batch.getBody(), newBatch.getBody(), newBatch.getBody().end()); if (newBatch.getBody().empty()) { rewriter.eraseOp(newBatch); return failure(); } auto blockArg = newBatch.getBody().front().insertArgument( 1 + newBatch.getWeights().size() + newBatch.getInputs().size() + idx, type, loc); for (unsigned oldResultIdx = 0; oldResultIdx < batch.getNumResults(); ++oldResultIdx) batch.getResult(oldResultIdx) .replaceAllUsesWith(newBatch.getResult(oldResultIdx < idx ? oldResultIdx : oldResultIdx + 1)); rewriter.eraseOp(batch.getOperation()); return std::make_tuple(cast(newBatch.getResult(idx)), blockArg, newBatch); } } // namespace bool isGraphComputeLike(Operation* op) { return isa(op); } bool isGraphBatchComputeLike(Operation* op) { return isa(op); } bool isScheduledComputeLike(Operation* op) { return isa(op); } bool isScheduledBatchComputeLike(Operation* op) { return isa(op); } bool isAnySpatialComputeLike(Operation* op) { return isa(op); } bool isAnySpatialComputeBatchLike(Operation* op) { return isa(op); } std::optional SpatGraphCompute::getWeightArgument(unsigned idx) { return getComputeWeightArgument(*this, idx); } std::optional SpatGraphCompute::getInputArgument(unsigned idx) { return getComputeInputArgument(*this, idx); } std::optional> SpatGraphCompute::insertWeight(unsigned idx, Value weight, Location loc) { return insertComputeWeight(*this, idx, weight, loc); } std::optional> SpatGraphCompute::insertInput(unsigned idx, Value input, Location loc) { return insertComputeInput(*this, idx, input, loc); } CrossbarWeightSet SpatGraphCompute::getCrossbarWeights() { return collectCrossbarWeights(getBody()); } FailureOr> SpatGraphCompute::insertOutput(RewriterBase& rewriter, unsigned idx, Type type, Location loc) { return insertComputeOutput(*this, rewriter, idx, type, loc); } void SpatGraphCompute::getAsmBlockArgumentNames(Region& region, OpAsmSetValueNameFn setNameFn) { setComputeAsmBlockArgumentNames(*this, region, setNameFn); } std::optional SpatScheduledCompute::getWeightArgument(unsigned idx) { return getComputeWeightArgument(*this, idx); } std::optional SpatScheduledCompute::getInputArgument(unsigned idx) { return getComputeInputArgument(*this, idx); } std::optional> SpatScheduledCompute::insertWeight(unsigned idx, Value weight, Location loc) { return insertComputeWeight(*this, idx, weight, loc); } std::optional> SpatScheduledCompute::insertInput(unsigned idx, Value input, Location loc) { return insertComputeInput(*this, idx, input, loc); } CrossbarWeightSet SpatScheduledCompute::getCrossbarWeights() { return collectCrossbarWeights(getBody()); } FailureOr> SpatScheduledCompute::insertOutput(RewriterBase& rewriter, unsigned idx, Type type, Location loc) { return insertComputeOutput(*this, rewriter, idx, type, loc); } void SpatScheduledCompute::getAsmBlockArgumentNames(Region& region, OpAsmSetValueNameFn setNameFn) { setComputeAsmBlockArgumentNames(*this, region, setNameFn); } SuccessorOperands SpatBlockYieldOp::getSuccessorOperands(unsigned index) { assert(index == 0 && "invalid successor index"); return SuccessorOperands(getOutputsMutable()); } Block* SpatBlockYieldOp::getSuccessorForOperands(ArrayRef) { return getOperation()->getNumSuccessors() == 0 ? nullptr : getOperation()->getSuccessor(0); } std::optional SpatGraphComputeBatch::getLaneArgument() { return getBlockArgument(getBody(), 0); } std::optional SpatGraphComputeBatch::getWeightArgument(unsigned idx) { return getBlockArgument(getBody(), 1 + idx); } std::optional SpatGraphComputeBatch::getInputArgument(unsigned idx) { return getBlockArgument(getBody(), 1 + getWeights().size() + idx); } std::optional SpatGraphComputeBatch::getOutputArgument(unsigned idx) { return getBlockArgument(getBody(), 1 + getWeights().size() + getInputs().size() + idx); } std::optional> SpatGraphComputeBatch::insertWeight(unsigned idx, Value weight, Location loc) { return insertComputeBatchWeight(*this, idx, weight, loc); } std::optional> SpatGraphComputeBatch::insertInput(unsigned idx, Value input, Location loc) { unsigned weightCount = getWeights().size(); unsigned inputCount = getInputs().size(); getOperation()->insertOperands(weightCount + idx, ValueRange {input}); setComputeOperandSegmentSizes( getOperation(), static_cast(weightCount), static_cast(inputCount + 1)); auto blockArg = insertBlockArgument(getBody(), 1 + weightCount + idx, input.getType(), loc); if (!blockArg) return std::nullopt; return std::make_tuple(getOperation()->getOperand(weightCount + idx), *blockArg); } CrossbarWeightSet SpatGraphComputeBatch::getCrossbarWeights() { return collectCrossbarWeights(getBody()); } FailureOr> SpatGraphComputeBatch::insertOutput(RewriterBase& rewriter, unsigned idx, Type type, Location loc) { return insertComputeBatchOutput(*this, rewriter, idx, type, loc); } void SpatGraphComputeBatch::getAsmBlockArgumentNames(Region& region, OpAsmSetValueNameFn setNameFn) { if (region.empty()) return; if (auto laneArg = getLaneArgument()) setNameFn(*laneArg, "lane"); setComputeAsmBlockArgumentNames(*this, region, setNameFn); for (unsigned index = 0; index < getNumResults(); ++index) { auto outputArg = getOutputArgument(index); if (!outputArg) continue; if (index == 0) { setNameFn(*outputArg, "out"); continue; } setNameFn(*outputArg, ("out" + std::to_string(index)).c_str()); } } std::optional SpatScheduledComputeBatch::getLaneArgument() { return getBlockArgument(getBody(), 0); } std::optional SpatScheduledComputeBatch::getWeightArgument(unsigned idx) { return getBlockArgument(getBody(), 1 + idx); } std::optional SpatScheduledComputeBatch::getInputArgument(unsigned idx) { return getBlockArgument(getBody(), 1 + getWeights().size() + idx); } std::optional SpatScheduledComputeBatch::getOutputArgument(unsigned idx) { return getBlockArgument(getBody(), 1 + getWeights().size() + getInputs().size() + idx); } std::optional> SpatScheduledComputeBatch::insertWeight(unsigned idx, Value weight, Location loc) { return insertComputeBatchWeight(*this, idx, weight, loc); } std::optional> SpatScheduledComputeBatch::insertInput(unsigned idx, Value input, Location loc) { unsigned weightCount = getWeights().size(); unsigned inputCount = getInputs().size(); getOperation()->insertOperands(weightCount + idx, ValueRange {input}); setComputeOperandSegmentSizes( getOperation(), static_cast(weightCount), static_cast(inputCount + 1)); auto blockArg = insertBlockArgument(getBody(), 1 + weightCount + idx, input.getType(), loc); if (!blockArg) return std::nullopt; return std::make_tuple(getOperation()->getOperand(weightCount + idx), *blockArg); } CrossbarWeightSet SpatScheduledComputeBatch::getCrossbarWeights() { return collectCrossbarWeights(getBody()); } FailureOr> SpatScheduledComputeBatch::insertOutput(RewriterBase& rewriter, unsigned idx, Type type, Location loc) { return insertComputeBatchOutput(*this, rewriter, idx, type, loc); } void SpatScheduledComputeBatch::getAsmBlockArgumentNames(Region& region, OpAsmSetValueNameFn setNameFn) { if (region.empty()) return; if (auto laneArg = getLaneArgument()) setNameFn(*laneArg, "lane"); setComputeAsmBlockArgumentNames(*this, region, setNameFn); for (unsigned index = 0; index < getNumResults(); ++index) { auto outputArg = getOutputArgument(index); if (!outputArg) continue; if (index == 0) { setNameFn(*outputArg, "out"); continue; } setNameFn(*outputArg, ("out" + std::to_string(index)).c_str()); } } void SpatInParallelOp::build(OpBuilder& builder, OperationState& result) { OpBuilder::InsertionGuard guard(builder); Region* bodyRegion = result.addRegion(); builder.createBlock(bodyRegion); } OpResult SpatInParallelOp::getParentResult(int64_t idx) { Operation* parent = getOperation()->getParentOp(); assert(isAnySpatialComputeBatchLike(parent) && "expected Spatial compute batch parent"); return parent->getResult(idx); } llvm::iterator_range SpatInParallelOp::getYieldingOps() { return getRegion().front().getOperations(); } void SpatialDialect::initialize() { addTypes< #define GET_TYPEDEF_LIST #include "src/Accelerators/PIM/Dialect/Spatial/SpatialTypes.cpp.inc" >(); addOperations< #define GET_OP_LIST #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.cpp.inc" >(); } } // namespace spatial } // namespace onnx_mlir //===----------------------------------------------------------------------===// // TableGen'd op method definitions //===----------------------------------------------------------------------===// #define GET_OP_CLASSES #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.cpp.inc" #define GET_TYPEDEF_CLASSES #include "src/Accelerators/PIM/Dialect/Spatial/SpatialDialect.cpp.inc" #include "src/Accelerators/PIM/Dialect/Spatial/SpatialTypes.cpp.inc"