#include "mlir/Dialect/Affine/IR/AffineOps.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/Transforms/DialectConversion.h" #include "llvm/ADT/SmallVector.h" #include #include #include #include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/CompileTime.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns.hpp" #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" #include "src/Dialect/ONNX/ONNXOps.hpp" using namespace mlir; namespace onnx_mlir { namespace { struct ReduceMeanSemantics { SmallVector axes; int64_t keepdims = 1; bool isIdentity = false; }; static bool isNoneValueLike(Value value) { return isa_and_nonnull(value.getDefiningOp()); } static FailureOr> getConstantIntValues(Value value) { auto denseAttr = dyn_cast_or_null(getHostConstDenseElementsAttr(value)); if (!denseAttr) return failure(); return SmallVector(denseAttr.getValues().begin(), denseAttr.getValues().end()); } static FailureOr> normalizeAxesChecked(ArrayRef axes, int64_t rank) { SmallVector normalizedAxes; normalizedAxes.reserve(axes.size()); for (int64_t axis : axes) { auto normalizedAxis = normalizeAxisChecked(axis, rank); if (failed(normalizedAxis)) return failure(); normalizedAxes.push_back(*normalizedAxis); } llvm::sort(normalizedAxes); normalizedAxes.erase(std::unique(normalizedAxes.begin(), normalizedAxes.end()), normalizedAxes.end()); return normalizedAxes; } template static FailureOr getReduceMeanSemantics(ReduceMeanOp reduceMeanOp, ReduceMeanOpAdaptor adaptor, int64_t inputRank) { ReduceMeanSemantics semantics; semantics.keepdims = reduceMeanOp.getKeepdims(); if constexpr (std::is_same_v) { auto axes = onnx_mlir::normalizeAxesChecked(std::optional(reduceMeanOp.getAxesAttr()), inputRank); if (failed(axes)) return failure(); semantics.axes = std::move(*axes); return semantics; } else { if (isNoneValueLike(adaptor.getAxes())) { if (reduceMeanOp.getNoopWithEmptyAxes() != 0) { semantics.isIdentity = true; return semantics; } semantics.axes.reserve(inputRank); for (int64_t axis = 0; axis < inputRank; ++axis) semantics.axes.push_back(axis); return semantics; } auto axes = getConstantIntValues(adaptor.getAxes()); if (failed(axes)) return failure(); if (axes->empty()) { if (reduceMeanOp.getNoopWithEmptyAxes() != 0) { semantics.isIdentity = true; return semantics; } semantics.axes.reserve(inputRank); for (int64_t axis = 0; axis < inputRank; ++axis) semantics.axes.push_back(axis); return semantics; } auto normalizedAxes = normalizeAxesChecked(*axes, inputRank); if (failed(normalizedAxes)) return failure(); semantics.axes = std::move(*normalizedAxes); return semantics; } } static SmallVector buildReducedAxesMask(ArrayRef axes, int64_t rank) { SmallVector reducedAxes(rank, false); for (int64_t axis : axes) { if (axis < 0 || axis >= rank) return {}; reducedAxes[axis] = true; } return reducedAxes; } static RankedTensorType getAllOnesType(RankedTensorType inputType, Type elementType) { return RankedTensorType::get(SmallVector(inputType.getRank(), 1), elementType); } static RankedTensorType getKeepdimsType(RankedTensorType inputType, Type elementType, ArrayRef reducedAxes) { SmallVector shape; shape.reserve(inputType.getRank()); for (auto [dim, isReduced] : llvm::zip_equal(inputType.getShape(), reducedAxes)) shape.push_back(isReduced ? 1 : dim); return RankedTensorType::get(shape, elementType, inputType.getEncoding()); } static RankedTensorType getReducedSliceType(RankedTensorType inputType, ArrayRef reducedAxes) { SmallVector shape; shape.reserve(inputType.getRank()); for (auto [dim, isReduced] : llvm::zip_equal(inputType.getShape(), reducedAxes)) shape.push_back(isReduced ? dim : 1); return RankedTensorType::get(shape, inputType.getElementType(), inputType.getEncoding()); } static RankedTensorType getLanePackedKeepdimsType(int64_t laneCount, RankedTensorType leafType) { return spatial::getGraphBatchPhysicalResultType(laneCount, leafType); } static SmallVector getKeptAxes(ArrayRef reducedAxes) { SmallVector keptAxes; for (auto [axis, isReduced] : llvm::enumerate(reducedAxes)) if (!isReduced) keptAxes.push_back(static_cast(axis)); return keptAxes; } static Value computeLaneIndex(Value lane, int64_t stride, int64_t dimSize, ConversionPatternRewriter& rewriter, Location loc) { if (dimSize == 1) return getOrCreateIndexConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), 0); MLIRContext* context = rewriter.getContext(); AffineExpr d0 = getAffineDimExpr(0, context); AffineExpr expr = d0; if (stride != 1) expr = expr.floorDiv(stride); if (dimSize != 1) expr = expr % dimSize; return createOrFoldAffineApply(rewriter, loc, expr, ValueRange {lane}, rewriter.getInsertionBlock()->getParentOp()); } static FailureOr buildReduceMeanKeepdimsBatch(Value input, ArrayRef reducedAxes, RankedTensorType batchType, RankedTensorType leafType, ConversionPatternRewriter& rewriter, Location loc) { auto inputType = cast(input.getType()); auto sliceType = getReducedSliceType(inputType, reducedAxes); SmallVector keptAxes = getKeptAxes(reducedAxes); int64_t laneCount = 1; SmallVector keptAxisStrides(keptAxes.size(), 1); for (int64_t index = static_cast(keptAxes.size()) - 1; index >= 0; --index) { keptAxisStrides[index] = laneCount; int64_t dimSize = inputType.getDimSize(keptAxes[index]); if (dimSize <= 0) return failure(); if (laneCount > std::numeric_limits::max() / dimSize) return failure(); laneCount *= dimSize; } SmallVector sliceOffsets; SmallVector sliceSizes; SmallVector unitStrides = getUnitStrides(rewriter, inputType.getRank()); sliceOffsets.reserve(inputType.getRank()); sliceSizes.reserve(inputType.getRank()); auto batchOp = createSpatComputeBatch(rewriter, loc, TypeRange {batchType}, laneCount, {}, ValueRange {input}, [&](detail::SpatComputeBatchBodyArgs args) { size_t keptAxisIndex = 0; sliceOffsets.clear(); sliceSizes.clear(); for (auto [axis, isReduced] : llvm::enumerate(reducedAxes)) { if (isReduced) { sliceOffsets.push_back(rewriter.getIndexAttr(0)); sliceSizes.push_back(rewriter.getIndexAttr(inputType.getDimSize(axis))); continue; } Value axisIndex = computeLaneIndex( args.lane, keptAxisStrides[keptAxisIndex], inputType.getDimSize(axis), rewriter, loc); ++keptAxisIndex; sliceOffsets.push_back(axisIndex); sliceSizes.push_back(rewriter.getIndexAttr(1)); } Value slice = tensor::ExtractSliceOp::create( rewriter, loc, sliceType, args.inputs.front(), sliceOffsets, sliceSizes, unitStrides); Value reduced = spatial::SpatVAvgOp::create(rewriter, loc, leafType, slice).getResult(); publishGraphBatchPhysicalFragment(rewriter, loc, reduced, args.outputs.front(), args.lane); }); if (failed(batchOp)) return failure(); return (*batchOp).getResult(0); } static FailureOr buildReduceMeanKeepdimsBlueprint( Value batchValue, RankedTensorType keepdimsType, ArrayRef reducedAxes, ConversionPatternRewriter& rewriter, Location loc) { auto batchType = dyn_cast(batchValue.getType()); int64_t rank = keepdimsType.getRank(); if (!batchType || !batchType.hasStaticShape() || !keepdimsType.hasStaticShape() || static_cast(reducedAxes.size()) != rank || batchType.getRank() != rank + 1 || batchType.getElementType() != keepdimsType.getElementType()) return failure(); int64_t laneCount = 1; SmallVector keptAxes; SmallVector keptAxisStrides; for (auto [axis, isReduced] : llvm::enumerate(reducedAxes)) { int64_t dim = keepdimsType.getDimSize(axis); if (dim <= 0 || (isReduced && dim != 1)) return failure(); if (!isReduced) keptAxes.push_back(axis); } keptAxisStrides.resize(keptAxes.size(), 1); for (int64_t index = static_cast(keptAxes.size()) - 1; index >= 0; --index) { keptAxisStrides[index] = laneCount; int64_t dim = keepdimsType.getDimSize(keptAxes[index]); if (laneCount > std::numeric_limits::max() / dim) return failure(); laneCount *= dim; } if (batchType.getDimSize(0) != laneCount || llvm::any_of(batchType.getShape().drop_front(), [](int64_t dim) { return dim != 1; })) return failure(); SmallVector operandIndices(laneCount, 0); SmallVector sourceSlots; SmallVector sourceOffsets(laneCount, 0); SmallVector fragmentOffsets; sourceSlots.reserve(laneCount); fragmentOffsets.reserve(laneCount * rank); for (int64_t lane = 0; lane < laneCount; ++lane) { sourceSlots.push_back(lane); size_t keptAxisIndex = 0; for (auto [axis, isReduced] : llvm::enumerate(reducedAxes)) { if (isReduced) { fragmentOffsets.push_back(0); continue; } int64_t dim = keepdimsType.getDimSize(axis); fragmentOffsets.push_back( (lane / keptAxisStrides[keptAxisIndex]) % dim); ++keptAxisIndex; } } SmallVector fragmentSizes(fragmentOffsets.size(), 1); SmallVector fragmentStrides(fragmentOffsets.size(), 1); return spatial::SpatBlueprintOp::create( rewriter, loc, keepdimsType, batchValue, ValueRange {}, rewriter.getStringAttr("nchw"), rewriter.getStringAttr("fragmented"), rewriter.getDenseI64ArrayAttr(fragmentOffsets), rewriter.getDenseI64ArrayAttr(fragmentSizes), rewriter.getStringAttr("reduce_mean_keepdims_fragments"), rewriter.getStringAttr("fragment_assembly"), rewriter.getDenseI64ArrayAttr(operandIndices), rewriter.getDenseI64ArrayAttr(sourceSlots), rewriter.getDenseI64ArrayAttr(sourceOffsets), rewriter.getDenseI64ArrayAttr(fragmentStrides), rewriter.getStringAttr("disjoint"), rewriter.getStringAttr("complete")) .getOutput(); } static SmallVector buildCollapseReassociation(ArrayRef reducedAxes) { SmallVector reassociation; ReassociationIndices currentGroup; for (auto [axis, isReduced] : llvm::enumerate(reducedAxes)) { currentGroup.push_back(axis); if (!isReduced) { reassociation.push_back(currentGroup); currentGroup.clear(); } } if (!currentGroup.empty()) { if (reassociation.empty()) reassociation.push_back(std::move(currentGroup)); else reassociation.back().append(currentGroup.begin(), currentGroup.end()); } return reassociation; } static Value squeezeReducedAxes(Value keepdimsValue, RankedTensorType resultType, ArrayRef reducedAxes, ConversionPatternRewriter& rewriter, Location loc) { SmallVector reassociation = resultType.getRank() == 0 ? SmallVector {} : buildCollapseReassociation(reducedAxes); if (isCompileTimeComputable(keepdimsValue)) return tensor::CollapseShapeOp::create(rewriter, loc, resultType, keepdimsValue, reassociation).getResult(); auto squeezeCompute = createSpatCompute<1>(rewriter, loc, TypeRange {resultType}, {}, ValueRange {keepdimsValue}, [&](Value input) { Value collapsed = tensor::CollapseShapeOp::create(rewriter, loc, resultType, input, reassociation); spatial::SpatYieldOp::create(rewriter, loc, collapsed); }); return squeezeCompute.getResult(0); } template struct ReduceMeanToSpatialCompute : OpConversionPattern { using OpConversionPattern::OpConversionPattern; using Adaptor = typename ReduceMeanOp::Adaptor; LogicalResult matchAndRewrite(ReduceMeanOp reduceMeanOp, Adaptor adaptor, ConversionPatternRewriter& rewriter) const override { auto inputType = dyn_cast(adaptor.getData().getType()); auto resultType = dyn_cast(reduceMeanOp.getReduced().getType()); if (!inputType || !resultType || !inputType.hasStaticShape() || !resultType.hasStaticShape()) return failure(); if (inputType.getRank() == 0) { rewriter.replaceOp(reduceMeanOp, adaptor.getData()); return success(); } auto semantics = getReduceMeanSemantics(reduceMeanOp, adaptor, inputType.getRank()); if (failed(semantics)) return rewriter.notifyMatchFailure(reduceMeanOp, "requires compile-time constant, in-range ReduceMean axes"); if (semantics->isIdentity) { if (inputType != resultType) return rewriter.notifyMatchFailure( reduceMeanOp, "noop_with_empty_axes identity requires the result type to match the input type"); rewriter.replaceOp(reduceMeanOp, adaptor.getData()); return success(); } SmallVector reducedAxes = buildReducedAxesMask(semantics->axes, inputType.getRank()); if (reducedAxes.empty() && inputType.getRank() != 0) return failure(); Location loc = reduceMeanOp.getLoc(); RankedTensorType leafType = getAllOnesType(inputType, resultType.getElementType()); RankedTensorType keepdimsType = getKeepdimsType(inputType, resultType.getElementType(), reducedAxes); int64_t laneCount = 1; for (auto [dim, isReduced] : llvm::zip_equal(keepdimsType.getShape(), reducedAxes)) { if (isReduced) continue; if (dim <= 0 || laneCount > std::numeric_limits::max() / dim) return rewriter.notifyMatchFailure( reduceMeanOp, "ReduceMean physical lane count is not representable"); laneCount *= dim; } RankedTensorType batchType = getLanePackedKeepdimsType(laneCount, leafType); auto lanePackedKeepdims = buildReduceMeanKeepdimsBatch(adaptor.getData(), reducedAxes, batchType, leafType, rewriter, loc); if (failed(lanePackedKeepdims)) return failure(); auto reducedKeepdims = buildReduceMeanKeepdimsBlueprint( *lanePackedKeepdims, keepdimsType, reducedAxes, rewriter, loc); if (failed(reducedKeepdims)) return rewriter.notifyMatchFailure( reduceMeanOp, "cannot build physical-fragment ReduceMean keepdims reconstruction"); if (semantics->keepdims != 0) { rewriter.replaceOp(reduceMeanOp, *reducedKeepdims); return success(); } Value reduced = squeezeReducedAxes( *reducedKeepdims, resultType, reducedAxes, rewriter, loc); rewriter.replaceOp(reduceMeanOp, reduced); return success(); } }; } // namespace void populateReduceMeanPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.add, ReduceMeanToSpatialCompute>(ctx); } } // namespace onnx_mlir