#include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Location.h" #include "mlir/IR/Matchers.h" #include "mlir/Support/LogicalResult.h" #include "mlir/Transforms/DialectConversion.h" #include "llvm/ADT/SmallVector.h" #include #include #include "Common/IR/ConstantUtils.hpp" #include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp" #include "src/Accelerators/PIM/Common/IR/LoopUtils.hpp" #include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp" #include "src/Accelerators/PIM/Common/IR/TensorSliceUtils.hpp" #include "src/Accelerators/PIM/Common/PimCommon.hpp" #include "src/Accelerators/PIM/Common/Support/Diagnostics.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/ContractionProblem.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/ContractionPlanning.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns/Math/Gemm.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/CompileTime.hpp" #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" #include "src/Dialect/ONNX/ONNXOps.hpp" using namespace mlir; namespace onnx_mlir { namespace { static FailureOr materializeScaledConstantTensor(Value value, float factor, PatternRewriter& rewriter, Location loc) { if (factor == 1.0f) return value; auto denseAttr = dyn_cast_or_null(getHostConstDenseElementsAttr(value)); if (!denseAttr) return failure(); SmallVector scaledValues; scaledValues.reserve(denseAttr.getNumElements()); APFloat scale(factor); bool hadFailure = false; for (const APFloat& originalValue : denseAttr.getValues()) { APFloat scaledValue(originalValue); if (scaledValue.multiply(scale, APFloat::rmNearestTiesToEven)) hadFailure = true; scaledValues.push_back(std::move(scaledValue)); } if (hadFailure) return failure(); auto scaledAttr = DenseFPElementsAttr::get(cast(denseAttr.getType()), scaledValues); return getOrCreateConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), scaledAttr, denseAttr.getType()); } static Value createGemmBatchKOffset( Value lane, int64_t numOutRows, int64_t numKSlices, int64_t xbarSize, PatternRewriter& rewriter, Location loc) { if (numKSlices == 1) return getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), 0); MLIRContext* context = rewriter.getContext(); AffineExpr d0 = getAffineDimExpr(0, context); return createOrFoldAffineApply(rewriter, loc, (d0.floorDiv(numOutRows) % numKSlices) * xbarSize, ValueRange {lane}, rewriter.getInsertionBlock()->getParentOp()); } static Value createGemmBatchHOffset(Value lane, int64_t numOutRows, int64_t numKSlices, int64_t numOutHSlices, int64_t xbarSize, PatternRewriter& rewriter, Location loc) { if (numOutHSlices == 1) return getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), 0); MLIRContext* context = rewriter.getContext(); AffineExpr d0 = getAffineDimExpr(0, context); return createOrFoldAffineApply(rewriter, loc, d0.floorDiv(numOutRows * numKSlices) * xbarSize, ValueRange {lane}, rewriter.getInsertionBlock()->getParentOp()); } static FailureOr materializePaddedConstantMatrix(Value value, RankedTensorType resultType, PatternRewriter& rewriter, Location loc) { auto sourceType = cast(value.getType()); if (sourceType == resultType) return value; auto denseAttr = getHostConstDenseElementsAttr(value); if (!denseAttr) return failure(); auto denseType = dyn_cast(denseAttr.getType()); if (!denseType || denseType.getRank() != 2 || !denseType.hasStaticShape()) return failure(); ArrayRef sourceShape = denseType.getShape(); ArrayRef resultShape = resultType.getShape(); SmallVector sourceValues(denseAttr.getValues()); Attribute zero = rewriter.getZeroAttr(resultType.getElementType()); SmallVector resultValues(resultType.getNumElements(), zero); for (int64_t row = 0; row < sourceShape[0]; ++row) for (int64_t col = 0; col < sourceShape[1]; ++col) resultValues[row * resultShape[1] + col] = sourceValues[row * sourceShape[1] + col]; auto resultAttr = DenseElementsAttr::get(resultType, resultValues); return getOrCreateConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), resultAttr, resultType); } static FailureOr materializePaddedBroadcastedConstantTensor(Value value, RankedTensorType resultType, int64_t unpaddedColumns, PatternRewriter& rewriter, Location loc) { auto denseAttr = getHostConstDenseElementsAttr(value); if (!denseAttr) return failure(); auto sourceType = dyn_cast(denseAttr.getType()); if (!sourceType || !sourceType.hasStaticShape() || sourceType.getRank() > resultType.getRank()) return failure(); ArrayRef sourceShape = sourceType.getShape(); ArrayRef resultShape = resultType.getShape(); SmallVector unpaddedResultShape(resultShape.begin(), resultShape.end()); unpaddedResultShape.back() = unpaddedColumns; const int64_t rankOffset = static_cast(resultShape.size() - sourceShape.size()); for (int64_t resultIndex = 0; resultIndex < static_cast(resultShape.size()); ++resultIndex) { const int64_t sourceIndex = resultIndex - rankOffset; if (sourceIndex < 0) continue; const int64_t sourceDim = sourceShape[sourceIndex]; const int64_t resultDim = unpaddedResultShape[resultIndex]; if (sourceDim != 1 && sourceDim != resultDim) return failure(); } SmallVector sourceValues(denseAttr.getValues()); SmallVector sourceStrides = computeRowMajorStrides(sourceShape); SmallVector resultStrides = computeRowMajorStrides(resultShape); Attribute zero = rewriter.getZeroAttr(resultType.getElementType()); SmallVector resultValues; resultValues.reserve(resultType.getNumElements()); for (int64_t flatIndex = 0; flatIndex < resultType.getNumElements(); ++flatIndex) { int64_t remaining = flatIndex; SmallVector resultIndices(resultShape.size(), 0); for (int64_t dim = 0; dim < static_cast(resultShape.size()); ++dim) { resultIndices[dim] = resultStrides.empty() ? 0 : remaining / resultStrides[dim]; remaining = resultStrides.empty() ? 0 : remaining % resultStrides[dim]; } if (resultIndices.back() >= unpaddedColumns) { resultValues.push_back(zero); continue; } int64_t sourceFlatIndex = 0; for (int64_t resultIndex = 0; resultIndex < static_cast(resultShape.size()); ++resultIndex) { const int64_t sourceIndex = resultIndex - rankOffset; if (sourceIndex < 0) continue; const int64_t sourceDim = sourceShape[sourceIndex]; const int64_t mappedIndex = sourceDim == 1 ? 0 : resultIndices[resultIndex]; sourceFlatIndex += mappedIndex * sourceStrides[sourceIndex]; } resultValues.push_back(sourceValues[sourceFlatIndex]); } auto resultAttr = DenseElementsAttr::get(resultType, resultValues); return getOrCreateConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), resultAttr, resultType); } static FailureOr prepareBias(Value c, RankedTensorType outType, RankedTensorType paddedOutType, PatternRewriter& rewriter, Location loc) { auto cType = cast(c.getType()); if (!cType.hasStaticShape()) return failure(); if (isCompileTimeComputable(c)) return materializePaddedBroadcastedConstantTensor(c, paddedOutType, outType.getDimSize(1), rewriter, loc); if (cType != outType) return failure(); return c; } static Value extractATile( Value a, Value row, Value kOffset, RankedTensorType aTileType, int64_t xbarSize, PatternRewriter& rewriter, Location loc) { SmallVector offsets {row, kOffset}; SmallVector sizes {rewriter.getIndexAttr(1), rewriter.getIndexAttr(xbarSize)}; SmallVector strides {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}; return tensor::ExtractSliceOp::create(rewriter, loc, aTileType, a, offsets, sizes, strides).getResult(); } static FailureOr createVmmBatch(Value a, Value b, RankedTensorType aType, RankedTensorType paddedBType, RankedTensorType partialPiecesType, int64_t numOutRows, int64_t numKSlices, int64_t numOutHSlices, int64_t xbarSize, PatternRewriter& rewriter, Location loc) { const int64_t laneCount = partialPiecesType.getDimSize(0); auto batchOp = createSpatComputeBatch( rewriter, loc, TypeRange {partialPiecesType}, laneCount, ValueRange {b}, ValueRange {a}, [&](detail::SpatComputeBatchBodyArgs args) { Value row = onnx_mlir::affineModConst(rewriter, loc, args.lane, numOutRows, rewriter.getInsertionBlock()->getParentOp()); Value kOffset = createGemmBatchKOffset(args.lane, numOutRows, numKSlices, xbarSize, rewriter, loc); Value hOffset = createGemmBatchHOffset( args.lane, numOutRows, numKSlices, numOutHSlices, xbarSize, rewriter, loc); auto aTileType = RankedTensorType::get({1, xbarSize}, aType.getElementType()); auto bTileType = RankedTensorType::get( {xbarSize, xbarSize}, paddedBType.getElementType()); auto pieceType = RankedTensorType::get({1, xbarSize}, partialPiecesType.getElementType()); Value aTile = extractATile(args.inputs.front(), row, kOffset, aTileType, xbarSize, rewriter, loc); SmallVector bOffsets {kOffset, hOffset}; SmallVector bSizes {rewriter.getIndexAttr(xbarSize), rewriter.getIndexAttr(xbarSize)}; SmallVector unitStrides = getUnitStrides(rewriter, 2); Value bTile = extractStaticSliceOrIdentity( rewriter, loc, args.weights.front(), bTileType, bOffsets, bSizes, unitStrides); Value piece = spatial::SpatVMMOp::create(rewriter, loc, pieceType, bTile, aTile).getResult(); publishGraphBatchPhysicalFragment(rewriter, loc, piece, args.outputs.front(), args.lane); }); if (failed(batchOp)) return failure(); return *batchOp; } static Value extractDynamicGemmBColumn( Value matrix, Value column, RankedTensorType vectorType, PatternRewriter& rewriter, Location loc) { SmallVector offsets {rewriter.getIndexAttr(0), column}; SmallVector sizes {rewriter.getIndexAttr(vectorType.getDimSize(1)), rewriter.getIndexAttr(1)}; SmallVector strides {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}; auto columnSliceType = RankedTensorType::get({vectorType.getDimSize(1), 1}, vectorType.getElementType()); Value columnSlice = tensor::ExtractSliceOp::create(rewriter, loc, columnSliceType, matrix, offsets, sizes, strides).getResult(); SmallVector collapseReassociation { ReassociationIndices {0, 1} }; auto collapsedType = RankedTensorType::get({vectorType.getDimSize(1)}, vectorType.getElementType()); Value collapsed = tensor::CollapseShapeOp::create(rewriter, loc, collapsedType, columnSlice, collapseReassociation).getResult(); SmallVector expandReassociation { ReassociationIndices {0, 1} }; return tensor::ExpandShapeOp::create(rewriter, loc, vectorType, collapsed, expandReassociation).getResult(); } static Value extractDynamicGemmRowVector( Value matrix, Value row, RankedTensorType vectorType, PatternRewriter& rewriter, Location loc) { SmallVector offsets {row, rewriter.getIndexAttr(0)}; SmallVector sizes {rewriter.getIndexAttr(1), rewriter.getIndexAttr(vectorType.getDimSize(1))}; SmallVector strides {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}; return tensor::ExtractSliceOp::create(rewriter, loc, vectorType, matrix, offsets, sizes, strides).getResult(); } static FailureOr verifyDynamicGemmBiasType(RankedTensorType cType, RankedTensorType outType) { if (!cType.hasStaticShape() || cType.getRank() > 2) return failure(); if (cType.getRank() == 0) return cType; int64_t numOutRows = outType.getDimSize(0); int64_t numOutCols = outType.getDimSize(1); if (cType.getRank() == 1) { int64_t cols = cType.getDimSize(0); if (cols == 1 || cols == numOutCols) return cType; return failure(); } int64_t rows = cType.getDimSize(0); int64_t cols = cType.getDimSize(1); if ((rows == 1 || rows == numOutRows) && (cols == 1 || cols == numOutCols)) return cType; return failure(); } static bool hasGemmBias(Value c) { if (!c) return false; Operation* definingOp = c.getDefiningOp(); return (!definingOp || !isa(definingOp)) && !isZeroSplatHostConstant(c); } static Value createScalarTensorConstant(RankedTensorType scalarType, float value, PatternRewriter& rewriter, Location loc) { auto elementType = scalarType.getElementType(); auto scalarAttr = rewriter.getFloatAttr(elementType, value); auto denseAttr = DenseElementsAttr::get(scalarType, scalarAttr); return getOrCreateConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), denseAttr, scalarType); } static Value createBroadcastedBiasScalar(Value bias, RankedTensorType biasType, Value row, Value column, RankedTensorType scalarType, PatternRewriter& rewriter, Location loc) { SmallVector unitStrides(biasType.getRank(), rewriter.getIndexAttr(1)); if (biasType.getRank() == 1) { SmallVector offsets {biasType.getDimSize(0) == 1 ? OpFoldResult(rewriter.getIndexAttr(0)) : OpFoldResult(column)}; SmallVector sizes {rewriter.getIndexAttr(1)}; auto vectorType = RankedTensorType::get({1}, scalarType.getElementType()); Value vector = tensor::ExtractSliceOp::create(rewriter, loc, vectorType, bias, offsets, sizes, unitStrides).getResult(); SmallVector reassociation { ReassociationIndices {0, 1} }; return tensor::ExpandShapeOp::create(rewriter, loc, scalarType, vector, reassociation).getResult(); } if (biasType.getRank() == 2) { SmallVector offsets { biasType.getDimSize(0) == 1 ? OpFoldResult(rewriter.getIndexAttr(0)) : OpFoldResult(row), biasType.getDimSize(1) == 1 ? OpFoldResult(rewriter.getIndexAttr(0)) : OpFoldResult(column)}; SmallVector sizes {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}; return tensor::ExtractSliceOp::create(rewriter, loc, scalarType, bias, offsets, sizes, unitStrides).getResult(); } Value scalar = tensor::ExtractOp::create(rewriter, loc, bias, ValueRange {}).getResult(); return tensor::SplatOp::create(rewriter, loc, scalarType, scalar).getResult(); } static FailureOr createVvdmulBatch(Value a, Value b, RankedTensorType aType, RankedTensorType bType, RankedTensorType columnPiecesType, RankedTensorType outType, bool transposeB, PatternRewriter& rewriter, Location loc) { const int64_t numOutRows = outType.getDimSize(0); const int64_t numOutCols = outType.getDimSize(1); const int64_t reductionSize = aType.getDimSize(1); auto vectorType = RankedTensorType::get({1, reductionSize}, aType.getElementType()); auto scalarType = RankedTensorType::get({1, 1}, outType.getElementType()); auto columnType = RankedTensorType::get({numOutRows, 1}, outType.getElementType()); auto batchOp = createSpatComputeBatch( rewriter, loc, TypeRange {columnPiecesType}, numOutCols, ValueRange {}, ValueRange {a, b}, [&](detail::SpatComputeBatchBodyArgs args) { Value bVector = transposeB ? extractDynamicGemmRowVector(args.inputs[1], args.lane, vectorType, rewriter, loc) : extractDynamicGemmBColumn(args.inputs[1], args.lane, vectorType, rewriter, loc); Value columnInit = tensor::EmptyOp::create(rewriter, loc, columnType.getShape(), columnType.getElementType()); Value c0 = getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), 0); Value c1 = getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), 1); Value cNumOutRows = getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), numOutRows); auto loop = buildNormalizedScfFor( rewriter, loc, c0, cNumOutRows, c1, ValueRange {columnInit}, [&](OpBuilder&, Location nestedLoc, Value row, ValueRange iterArgs, SmallVectorImpl& yielded) { Value aVector = extractDynamicGemmRowVector(args.inputs[0], row, vectorType, rewriter, nestedLoc); Value scalar = spatial::SpatVVDMulOp::create(rewriter, nestedLoc, scalarType, aVector, bVector).getResult(); Value next = tensor::InsertSliceOp::create(rewriter, nestedLoc, scalar, iterArgs.front(), SmallVector {row, rewriter.getIndexAttr(0)}, SmallVector {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}, getUnitStrides(rewriter, 2)); yielded.push_back(next); return success(); }); assert(succeeded(loop) && "dynamic Gemm row loop construction must succeed"); publishGraphBatchPhysicalFragment(rewriter, loc, loop->results.front(), args.outputs.front(), args.lane); }); if (failed(batchOp)) return failure(); return *batchOp; } static FailureOr createDynamicGemmOutputCompute(Value scalarPieces, Value bias, RankedTensorType scalarPiecesType, RankedTensorType biasType, RankedTensorType outType, float alpha, float beta, PatternRewriter& rewriter, Location loc) { const int64_t numOutRows = outType.getDimSize(0); const int64_t numOutCols = outType.getDimSize(1); SmallVector inputs {scalarPieces}; if (bias) inputs.push_back(bias); return createSpatCompute(rewriter, loc, TypeRange {outType}, {}, inputs, [&](ValueRange blockArgs) -> LogicalResult { Value pieces = blockArgs[0]; Value biasArg = bias ? blockArgs[1] : Value(); auto scalarType = RankedTensorType::get({1, 1}, outType.getElementType()); Value outputInit = tensor::EmptyOp::create(rewriter, loc, outType.getShape(), outType.getElementType()).getResult(); Value c0 = getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), 0); Value c1 = getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), 1); Value cNumOutCols = getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), numOutCols); auto columnType = RankedTensorType::get({numOutRows, 1}, outType.getElementType()); auto loop = buildNormalizedScfFor( rewriter, loc, c0, cNumOutCols, c1, ValueRange {outputInit}, [&](OpBuilder&, Location nestedLoc, Value column, ValueRange iterArgs, SmallVectorImpl& yielded) { Value outputAcc = iterArgs.front(); FailureOr columnPiece = extractGraphBatchPhysicalFragment(rewriter, nestedLoc, pieces, column, columnType); if (failed(columnPiece)) return failure(); Value cNumOutRows = getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), numOutRows); auto rowLoop = buildNormalizedScfFor( rewriter, nestedLoc, c0, cNumOutRows, c1, ValueRange {outputAcc}, [&](OpBuilder&, Location rowLoc, Value row, ValueRange rowIterArgs, SmallVectorImpl& rowYielded) { SmallVector scalarOffsets {row, rewriter.getIndexAttr(0)}; SmallVector scalarSizes {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}; SmallVector unitStrides {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}; Value scalar = tensor::ExtractSliceOp::create( rewriter, rowLoc, scalarType, *columnPiece, scalarOffsets, scalarSizes, unitStrides); if (alpha != 1.0f) { Value alphaTensor = createScalarTensorConstant(scalarType, alpha, rewriter, rowLoc); scalar = spatial::SpatVMulOp::create(rewriter, rowLoc, scalarType, scalar, alphaTensor).getResult(); } if (biasArg) { Value biasScalar = createBroadcastedBiasScalar(biasArg, biasType, row, column, scalarType, rewriter, rowLoc); if (beta != 1.0f) { Value betaTensor = createScalarTensorConstant(scalarType, beta, rewriter, rowLoc); biasScalar = spatial::SpatVMulOp::create(rewriter, rowLoc, scalarType, biasScalar, betaTensor).getResult(); } scalar = spatial::SpatVAddOp::create(rewriter, rowLoc, scalarType, scalar, biasScalar).getResult(); } Value next = tensor::InsertSliceOp::create(rewriter, rowLoc, scalar, rowIterArgs.front(), SmallVector {row, column}, scalarSizes, unitStrides); rowYielded.push_back(next); return success(); }); if (failed(rowLoop)) return failure(); yielded.push_back(rowLoop->results.front()); return success(); }); if (failed(loop)) return failure(); spatial::SpatYieldOp::create(rewriter, loc, loop->results.front()); return success(); }); } static Value createPartialGroupOffset(Value hSlice, int64_t kSlice, int64_t numKSlices, int64_t numOutRows, PatternRewriter& rewriter, Location loc) { MLIRContext* context = rewriter.getContext(); AffineExpr d0 = getAffineDimExpr(0, context); return createOrFoldAffineApply(rewriter, loc, d0 * (numKSlices * numOutRows) + kSlice * numOutRows, ValueRange {hSlice}, rewriter.getInsertionBlock()->getParentOp()); } static Value extractReductionPiece(Value partialPiecesArg, Value hSlice, int64_t kSlice, RankedTensorType pieceType, int64_t numKSlices, int64_t numOutRows, int64_t xbarSize, PatternRewriter& rewriter, Location loc) { SmallVector unitStrides {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}; SmallVector pieceSizes { rewriter.getIndexAttr(numOutRows), rewriter.getIndexAttr(1), rewriter.getIndexAttr(xbarSize)}; SmallVector pieceOffsets { createPartialGroupOffset(hSlice, kSlice, numKSlices, numOutRows, rewriter, loc), rewriter.getIndexAttr(0), rewriter.getIndexAttr(0)}; return extractMixedSliceOrIdentity( rewriter, loc, partialPiecesArg, pieceType, {pieceOffsets, pieceSizes, unitStrides}); } static Value reducePartialPiecesForHSlice(Value partialPiecesArg, Value hSlice, RankedTensorType pieceType, int64_t numKSlices, int64_t numOutRows, int64_t xbarSize, PatternRewriter& rewriter, Location loc) { SmallVector activePieces; activePieces.reserve(numKSlices); for (int64_t kSlice = 0; kSlice < numKSlices; ++kSlice) activePieces.push_back( extractReductionPiece( partialPiecesArg, hSlice, kSlice, pieceType, numKSlices, numOutRows, xbarSize, rewriter, loc)); while (activePieces.size() > 1) { SmallVector nextPieces; nextPieces.reserve((activePieces.size() + 1) / 2); for (size_t pieceIndex = 0; pieceIndex + 1 < activePieces.size(); pieceIndex += 2) nextPieces.push_back( spatial::SpatVAddOp::create(rewriter, loc, pieceType, activePieces[pieceIndex], activePieces[pieceIndex + 1]) .getResult()); if (activePieces.size() % 2 != 0) nextPieces.push_back(activePieces.back()); activePieces = std::move(nextPieces); } return activePieces.front(); } static FailureOr createReductionOutput(Value partialPieces, Value bias, RankedTensorType partialPiecesType, RankedTensorType outType, RankedTensorType paddedOutType, int64_t numKSlices, int64_t xbarSize, PatternRewriter& rewriter, Location loc) { const int64_t numOutRows = outType.getDimSize(0); const int64_t numOutHSlices = ceilIntegerDivide(outType.getDimSize(1), xbarSize); auto pieceType = RankedTensorType::get({numOutRows, xbarSize}, partialPiecesType.getElementType()); if (bias && cast(bias.getType()) != paddedOutType) bias = createZeroPaddedTensor(bias, paddedOutType, rewriter, loc); SmallVector inputs {partialPieces}; if (bias) inputs.push_back(bias); SmallVector outputSlices; outputSlices.reserve(numOutHSlices); for (int64_t hSlice = 0; hSlice < numOutHSlices; ++hSlice) { const int64_t columnOffset = hSlice * xbarSize; const int64_t columns = std::min(xbarSize, outType.getDimSize(1) - columnOffset); auto outputSliceType = RankedTensorType::get({numOutRows, columns}, outType.getElementType()); auto computeOp = createSpatCompute( rewriter, loc, TypeRange {outputSliceType}, {}, inputs, [&](ValueRange blockArgs) -> LogicalResult { Value hSliceValue = getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), hSlice); Value reduced = reducePartialPiecesForHSlice( blockArgs[0], hSliceValue, pieceType, numKSlices, numOutRows, xbarSize, rewriter, loc); if (bias) { SmallVector biasOffsets {rewriter.getIndexAttr(0), rewriter.getIndexAttr(columnOffset)}; SmallVector pieceSizes {rewriter.getIndexAttr(numOutRows), rewriter.getIndexAttr(xbarSize)}; SmallVector unitStrides {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1)}; Value biasSlice = tensor::ExtractSliceOp::create(rewriter, loc, pieceType, blockArgs[1], biasOffsets, pieceSizes, unitStrides) .getResult(); reduced = spatial::SpatVAddOp::create(rewriter, loc, pieceType, reduced, biasSlice).getResult(); } if (outputSliceType != pieceType) { SmallVector offsets {rewriter.getIndexAttr(0), rewriter.getIndexAttr(0)}; SmallVector sizes {rewriter.getIndexAttr(numOutRows), rewriter.getIndexAttr(columns)}; reduced = tensor::ExtractSliceOp::create( rewriter, loc, outputSliceType, reduced, offsets, sizes, getUnitStrides(rewriter, 2)); } spatial::SpatYieldOp::create(rewriter, loc, reduced); return success(); }); if (failed(computeOp)) return failure(); outputSlices.push_back(computeOp->getResult(0)); } if (outputSlices.size() == 1) return outputSlices.front(); auto concatCompute = createSpatCompute(rewriter, loc, TypeRange {outType}, {}, outputSlices, [&](ValueRange blockArgs) { Value result = spatial::SpatConcatOp::create(rewriter, loc, outType, rewriter.getI64IntegerAttr(1), blockArgs).getOutput(); spatial::SpatYieldOp::create(rewriter, loc, result); }); return concatCompute.getResult(0); } struct GemmToSpatialComputes : OpConversionPattern { explicit GemmToSpatialComputes(MLIRContext* ctx, const spatial::SpatialTargetResources& target) : OpConversionPattern(ctx), target(target) {} LogicalResult matchAndRewrite(ONNXGemmOp gemmOp, ONNXGemmOpAdaptor gemmOpAdaptor, ConversionPatternRewriter& rewriter) const override; const spatial::SpatialTargetResources& target; }; } // namespace FailureOr lowerGemmToSpatial( Operation* diagnosticAnchor, Value a, Value b, Value c, RankedTensorType outType, bool transA, bool transB, float alpha, float beta, const spatial::SpatialTargetResources& target, PatternRewriter& rewriter, Location loc) { auto aType = dyn_cast(a.getType()); auto bType = dyn_cast(b.getType()); if (!diagnosticAnchor || !aType || !bType || !outType) return failure(); if (!aType.hasStaticShape()) { pim::emitUnsupportedStaticShapeDiagnostic(diagnosticAnchor, "Gemm input A"); return failure(); } if (!bType.hasStaticShape()) { pim::emitUnsupportedStaticShapeDiagnostic(diagnosticAnchor, "Gemm input B"); return failure(); } if (!outType.hasStaticShape()) { pim::emitUnsupportedStaticShapeDiagnostic(diagnosticAnchor, "Gemm result"); return failure(); } if (aType.getRank() != 2) { pim::emitUnsupportedRankDiagnostic(diagnosticAnchor, "Gemm input A", aType.getRank(), {2}); return failure(); } if (bType.getRank() != 2) { pim::emitUnsupportedRankDiagnostic(diagnosticAnchor, "Gemm input B", bType.getRank(), {2}); return failure(); } if (outType.getRank() != 2) { pim::emitUnsupportedRankDiagnostic(diagnosticAnchor, "Gemm result", outType.getRank(), {2}); return failure(); } if (transA) { auto aShape = aType.getShape(); auto transposedType = RankedTensorType::get({aShape[1], aShape[0]}, aType.getElementType(), aType.getEncoding()); a = createLinalgTranspose(a, transposedType, {1, 0}, rewriter, loc); aType = transposedType; } ContractionProblem problem; problem.lhsBatchShape = {}; problem.rhsBatchShape = {}; problem.outputBatchShape = {}; problem.lhsBatch = 1; problem.rhsBatch = 1; problem.batch = 1; problem.m = outType.getDimSize(0); problem.k = aType.getDimSize(1); problem.n = outType.getDimSize(1); problem.lhsElementType = aType.getElementType(); problem.rhsElementType = bType.getElementType(); problem.resultElementType = outType.getElementType(); const bool transposeB = transB; if (!isCompileTimeComputable(b)) { ContractionPlan plan = makeContractionPlan( problem, target, ContractionPlanKind::BatchedDynamicVVD); bool hasC = hasGemmBias(c); RankedTensorType biasType; if (hasC) { auto cType = dyn_cast(c.getType()); if (!cType || !cType.hasStaticShape()) { pim::emitUnsupportedStaticShapeDiagnostic(diagnosticAnchor, "Gemm bias"); return failure(); } auto verifiedBiasType = verifyDynamicGemmBiasType(cType, outType); if (failed(verifiedBiasType)) { diagnosticAnchor->emitOpError("requires Gemm bias C to be broadcastable to the output shape"); return failure(); } biasType = *verifiedBiasType; } const int64_t bReductionSize = bType.getDimSize(transposeB ? 1 : 0); const int64_t bOutputColumns = bType.getDimSize(transposeB ? 0 : 1); if (aType.getDimSize(0) != problem.m || bReductionSize != problem.k || bOutputColumns != problem.n) { diagnosticAnchor->emitOpError("has inconsistent A, B, and output shapes"); return failure(); } const int64_t laneCount64 = plan.laneCount; if (laneCount64 > std::numeric_limits::max()) { diagnosticAnchor->emitOpError("requires Gemm dynamic batch lane count to fit in i32"); return failure(); } auto columnType = RankedTensorType::get({problem.m, 1}, outType.getElementType()); auto scalarPiecesType = spatial::getGraphBatchPhysicalResultType(problem.n, columnType); auto batchOp = createVvdmulBatch(a, b, aType, bType, scalarPiecesType, outType, transposeB, rewriter, loc); if (failed(batchOp)) return failure(); auto outputCompute = createDynamicGemmOutputCompute( batchOp->getResult(0), hasC ? c : Value(), scalarPiecesType, biasType, outType, alpha, beta, rewriter, loc); if (failed(outputCompute)) return failure(); return outputCompute->getResult(0); } if (transposeB) { auto bShape = bType.getShape(); auto transposedType = RankedTensorType::get({bShape[1], bShape[0]}, bType.getElementType(), bType.getEncoding()); if (isCompileTimeComputable(b)) { auto denseAttr = getHostConstDenseElementsAttr(b); auto inputType = denseAttr ? dyn_cast(denseAttr.getType()) : nullptr; auto transposedAttr = inputType && inputType.hasStaticShape() && transposedType.hasStaticShape() ? transposeDenseElementsAttr(denseAttr, {1, 0}) : FailureOr(failure()); if (failed(transposedAttr) || transposedAttr->getType() != transposedType) { diagnosticAnchor->emitOpError("requires Gemm input B transpose to remain statically materializable"); return failure(); } b = getOrCreateConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), *transposedAttr, transposedType); } else { b = createLinalgTranspose(b, transposedType, {1, 0}, rewriter, loc); } bType = transposedType; } auto scaledB = materializeScaledConstantTensor(b, alpha, rewriter, loc); if (failed(scaledB)) { diagnosticAnchor->emitOpError("requires constant Gemm input B when alpha is not 1.0"); return failure(); } b = *scaledB; bType = cast(b.getType()); if (aType.getDimSize(0) != problem.m || bType.getDimSize(0) != problem.k || bType.getDimSize(1) != problem.n) { diagnosticAnchor->emitOpError("has inconsistent A, B, and output shapes after transpose handling"); return failure(); } ContractionPlan plan = makeContractionPlan( problem, target, ContractionPlanKind::StaticTiled); const int64_t xbarSize = plan.tileK; const int64_t numKSlices = plan.reductionSlices; const int64_t numOutHSlices = plan.outputTiles; const int64_t paddedReductionSize = numKSlices * plan.tileK; const int64_t paddedOutCols = numOutHSlices * plan.tileN; auto paddedBType = RankedTensorType::get({paddedReductionSize, paddedOutCols}, bType.getElementType()); auto paddedB = materializePaddedConstantMatrix(b, paddedBType, rewriter, loc); if (failed(paddedB)) { diagnosticAnchor->emitOpError("requires constant Gemm input B so tiled weights can be padded statically"); return failure(); } b = *paddedB; auto paddedAType = RankedTensorType::get({problem.m, paddedReductionSize}, aType.getElementType()); a = createPaddedInputCompute(a, paddedAType, rewriter, loc); aType = paddedAType; Value bias; bool hasC = hasGemmBias(c); auto paddedOutType = RankedTensorType::get({problem.m, paddedOutCols}, outType.getElementType()); if (hasC) { auto cType = dyn_cast(c.getType()); if (!cType || !cType.hasStaticShape()) { pim::emitUnsupportedStaticShapeDiagnostic(diagnosticAnchor, "Gemm bias"); return failure(); } auto scaledC = materializeScaledConstantTensor(c, beta, rewriter, loc); if (failed(scaledC)) { diagnosticAnchor->emitOpError("requires constant Gemm bias C when beta is not 1.0"); return failure(); } c = *scaledC; auto preparedBias = prepareBias(c, outType, paddedOutType, rewriter, loc); if (failed(preparedBias)) { diagnosticAnchor->emitOpError("requires Gemm bias C to be broadcastable to the output shape"); return failure(); } bias = *preparedBias; } const int64_t laneCount64 = plan.laneCount; if (laneCount64 > std::numeric_limits::max()) { diagnosticAnchor->emitOpError("requires Gemm tiled batch lane count to fit in i32"); return failure(); } auto partialPiecesType = spatial::getGraphBatchPhysicalResultType( laneCount64, RankedTensorType::get({1, xbarSize}, outType.getElementType())); auto batchOp = createVmmBatch( a, b, aType, paddedBType, partialPiecesType, problem.m, numKSlices, numOutHSlices, xbarSize, rewriter, loc); if (failed(batchOp)) return failure(); auto reductionOutput = createReductionOutput( batchOp->getResult(0), bias, partialPiecesType, outType, paddedOutType, numKSlices, xbarSize, rewriter, loc); if (failed(reductionOutput)) return failure(); return *reductionOutput; } LogicalResult GemmToSpatialComputes::matchAndRewrite(ONNXGemmOp gemmOp, ONNXGemmOpAdaptor gemmOpAdaptor, ConversionPatternRewriter& rewriter) const { FailureOr result = lowerGemmToSpatial( gemmOp.getOperation(), gemmOpAdaptor.getA(), gemmOpAdaptor.getB(), gemmOpAdaptor.getC(), cast(gemmOp.getY().getType()), gemmOpAdaptor.getTransA(), gemmOpAdaptor.getTransB(), gemmOpAdaptor.getAlpha().convertToFloat(), gemmOpAdaptor.getBeta().convertToFloat(), target, rewriter, gemmOp.getLoc()); if (failed(result)) return failure(); rewriter.replaceOp(gemmOp, *result); return success(); } void populateGemmPatterns(RewritePatternSet& patterns, MLIRContext* ctx, const spatial::SpatialTargetResources& target) { patterns.insert(ctx, target); } } // namespace onnx_mlir