#include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/Transforms/DialectConversion.h" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ConversionPatterns.hpp" #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" #include "src/Dialect/ONNX/ONNXOps.hpp" using namespace mlir; namespace onnx_mlir { namespace { static int64_t normalizeAxis(int64_t axis, int64_t rank) { return axis >= 0 ? axis : rank + axis; } static SmallVector permuteShape(ArrayRef shape, ArrayRef permutation) { SmallVector permutedShape; permutedShape.reserve(permutation.size()); for (int64_t axis : permutation) permutedShape.push_back(shape[axis]); return permutedShape; } static Value buildLoopSoftmaxSlice(Value input, Value accumulator, RankedTensorType inputType, ArrayRef outerIndices, ConversionPatternRewriter& rewriter, Location loc) { int64_t rank = inputType.getRank(); SmallVector sliceShape(static_cast(rank - 1), 1); sliceShape.push_back(inputType.getDimSize(rank - 1)); auto sliceType = RankedTensorType::get(sliceShape, inputType.getElementType(), inputType.getEncoding()); SmallVector offsets; SmallVector sizes; SmallVector strides(rank, rewriter.getIndexAttr(1)); offsets.reserve(rank); sizes.reserve(rank); for (Value outerIndex : outerIndices) { offsets.push_back(outerIndex); sizes.push_back(rewriter.getIndexAttr(1)); } offsets.push_back(rewriter.getIndexAttr(0)); sizes.push_back(rewriter.getIndexAttr(inputType.getDimSize(rank - 1))); Value inputSlice = tensor::ExtractSliceOp::create(rewriter, loc, sliceType, input, offsets, sizes, strides); Value softmaxSlice = spatial::SpatSoftmaxOp::create(rewriter, loc, sliceType, inputSlice).getResult(); return tensor::InsertSliceOp::create(rewriter, loc, softmaxSlice, accumulator, offsets, sizes, strides); } static Value buildLoopSoftmaxNest(Value input, Value accumulator, RankedTensorType inputType, int64_t axis, SmallVectorImpl& outerIndices, ConversionPatternRewriter& rewriter, Location loc) { if (axis == inputType.getRank() - 1) return buildLoopSoftmaxSlice(input, accumulator, inputType, outerIndices, rewriter, loc); Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0); Value c1 = arith::ConstantIndexOp::create(rewriter, loc, 1); Value cUpper = arith::ConstantIndexOp::create(rewriter, loc, inputType.getDimSize(axis)); auto loop = scf::ForOp::create(rewriter, loc, c0, cUpper, c1, ValueRange {accumulator}); rewriter.setInsertionPointToStart(loop.getBody()); Value loopIndex = loop.getInductionVar(); Value loopAccumulator = loop.getRegionIterArgs().front(); outerIndices.push_back(loopIndex); Value updatedAccumulator = buildLoopSoftmaxNest(input, loopAccumulator, inputType, axis + 1, outerIndices, rewriter, loc); outerIndices.pop_back(); scf::YieldOp::create(rewriter, loc, updatedAccumulator); rewriter.setInsertionPointAfter(loop); return loop.getResult(0); } static Value createLoopSoftmaxCompute(Value input, ConversionPatternRewriter& rewriter, Location loc) { auto inputType = cast(input.getType()); constexpr size_t numInputs = 1; auto computeOp = createSpatCompute(rewriter, loc, TypeRange {inputType}, {}, ValueRange {input}, [&](Value x) { if (inputType.getRank() == 1) { Value softmax = spatial::SpatSoftmaxOp::create(rewriter, loc, inputType, x).getResult(); spatial::SpatYieldOp::create(rewriter, loc, softmax); return; } Value outputInit = tensor::EmptyOp::create(rewriter, loc, inputType.getShape(), inputType.getElementType()); SmallVector outerIndices; Value result = buildLoopSoftmaxNest(x, outputInit, inputType, /*axis=*/0, outerIndices, rewriter, loc); spatial::SpatYieldOp::create(rewriter, loc, result); }); return computeOp.getResult(0); } struct SoftmaxToSpatialCompute : OpConversionPattern { using OpConversionPattern::OpConversionPattern; LogicalResult matchAndRewrite(ONNXSoftmaxOp softmaxOp, ONNXSoftmaxOpAdaptor adaptor, ConversionPatternRewriter& rewriter) const override { auto inputType = dyn_cast(adaptor.getInput().getType()); if (!inputType || !inputType.hasStaticShape()) return failure(); int64_t axis = normalizeAxis(softmaxOp.getAxis(), inputType.getRank()); if (axis < 0 || axis >= inputType.getRank()) return failure(); Value input = adaptor.getInput(); Value result; if (axis == inputType.getRank() - 1) { result = createLoopSoftmaxCompute(input, rewriter, softmaxOp.getLoc()); } else { SmallVector permutation; permutation.reserve(inputType.getRank()); for (int64_t dim = 0; dim < inputType.getRank(); ++dim) if (dim != axis) permutation.push_back(dim); permutation.push_back(axis); SmallVector inversePermutation(inputType.getRank()); for (auto [newIndex, oldIndex] : llvm::enumerate(permutation)) inversePermutation[oldIndex] = static_cast(newIndex); auto transposedType = RankedTensorType::get( permuteShape(inputType.getShape(), permutation), inputType.getElementType(), inputType.getEncoding()); auto preTransposeCompute = createSpatCompute<1>(rewriter, softmaxOp.getLoc(), TypeRange {transposedType}, {}, input, [&](Value x) { Value transposed = ONNXTransposeOp::create( rewriter, softmaxOp.getLoc(), transposedType, x, rewriter.getI64ArrayAttr(permutation)); spatial::SpatYieldOp::create(rewriter, softmaxOp.getLoc(), transposed); }); Value transposedInput = preTransposeCompute.getResult(0); Value transposedResult = createLoopSoftmaxCompute(transposedInput, rewriter, softmaxOp.getLoc()); auto postTransposeCompute = createSpatCompute<1>(rewriter, softmaxOp.getLoc(), TypeRange {inputType}, {}, transposedResult, [&](Value x) { Value transposed = ONNXTransposeOp::create( rewriter, softmaxOp.getLoc(), inputType, x, rewriter.getI64ArrayAttr(inversePermutation)); spatial::SpatYieldOp::create(rewriter, softmaxOp.getLoc(), transposed); }); result = postTransposeCompute.getResult(0); } rewriter.replaceOp(softmaxOp, result); return success(); } }; } // namespace void populateSoftmaxPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.add(ctx); } } // namespace onnx_mlir