#include "llvm/ADT/SmallVector.h" #include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp" #include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/BiasAddUtils.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/ComputeRegionBuilder.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/MatrixProductLowering.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/RowStripLayoutUtils.hpp" #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" #include using namespace mlir; namespace onnx_mlir { FailureOr describeRowStripPhysicalValue(Value storage, RankedTensorType logicalType) { auto storageType = dyn_cast(storage.getType()); if (!storageType || !storageType.hasStaticShape() || !logicalType || !logicalType.hasStaticShape() || storageType.getRank() != 5 || logicalType.getRank() != 4 || logicalType.getDimSize(0) != 1 || storageType.getElementType() != logicalType.getElementType() || storageType.getDimSize(1) != 1 || storageType.getDimSize(2) != 1 || storageType.getDimSize(3) != logicalType.getDimSize(3) || storageType.getDimSize(4) <= 0) return failure(); const int64_t tilesPerRow = ceilIntegerDivide(logicalType.getDimSize(1), storageType.getDimSize(4)); if (storageType.getDimSize(0) != logicalType.getDimSize(2) * tilesPerRow) return failure(); return RowStripPhysicalValue {storage, logicalType, RankedTensorType::get(storageType.getShape().drop_front(), storageType.getElementType(), storageType.getEncoding()), tilesPerRow}; } FailureOr getRowStripPhysicalValue(Value value) { auto blueprint = value.getDefiningOp(); auto logicalType = dyn_cast(value.getType()); if (!blueprint || !logicalType || blueprint.getOutput() != value || blueprint.getPhysicalLayout() != spatial::PhysicalLayout::NHWCRowStrip || !spatial::isPhysicalView(blueprint.getMode())) return failure(); return describeRowStripPhysicalValue(blueprint.getInput(), logicalType); } RankedTensorType getRowStripFragmentType(RankedTensorType logicalType) { return RankedTensorType::get({logicalType.getDimSize(0), 1, logicalType.getDimSize(3), logicalType.getDimSize(1)}, logicalType.getElementType(), logicalType.getEncoding()); } RankedTensorType getRowStripStorageType(RankedTensorType logicalType) { return spatial::getGraphBatchPhysicalResultType(logicalType.getDimSize(2), getRowStripFragmentType(logicalType)); } std::pair, SmallVector> buildRowStripMetadata(RankedTensorType type) { SmallVector offsets; SmallVector sizes; const int64_t channels = type.getDimSize(1); const int64_t height = type.getDimSize(2); const int64_t width = type.getDimSize(3); offsets.reserve(height * 4); sizes.reserve(height * 4); for (int64_t row = 0; row < height; ++row) { offsets.append({0, 0, row, 0}); sizes.append({1, channels, 1, width}); } return {offsets, sizes}; } Value extractRowStripFragment(Value storage, RankedTensorType logicalType, OpFoldResult row, PatternRewriter& rewriter, Location loc) { return *extractGraphBatchPhysicalFragment(rewriter, loc, storage, row, getRowStripFragmentType(logicalType)); } void insertRowStripFragment(Value fragment, Value output, RankedTensorType logicalType, OpFoldResult row, PatternRewriter& rewriter, Location loc) { assert(fragment.getType() == getRowStripFragmentType(logicalType)); assert(output.getType() == getRowStripStorageType(logicalType)); auto slot = dyn_cast(row); assert(slot && "row-strip graph publication requires a dynamic physical slot"); publishGraphBatchPhysicalFragment(rewriter, loc, fragment, output, slot); } FailureOr createPerChannelConstantFragment(DenseElementsAttr denseAttr, RankedTensorType fragmentType, PatternRewriter& rewriter) { auto logicalType = RankedTensorType::get( {1, fragmentType.getDimSize(3), 1, 1}, fragmentType.getElementType()); FailureOr> channelValues = getBiasChannelValues(denseAttr, logicalType); if (failed(channelValues)) return failure(); SmallVector values; values.reserve(fragmentType.getNumElements()); for (int64_t n = 0; n < fragmentType.getDimSize(0); ++n) for (int64_t h = 0; h < fragmentType.getDimSize(1); ++h) for (int64_t w = 0; w < fragmentType.getDimSize(2); ++w) for (int64_t channel = 0; channel < fragmentType.getDimSize(3); ++channel) values.push_back((*channelValues)[channel]); auto attr = DenseElementsAttr::get(fragmentType, values); return getOrCreateConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), attr, fragmentType); } FailureOr createRowStripStorageFromRows(Value rows, RankedTensorType logicalType, PatternRewriter& rewriter, Location loc) { auto rowsType = dyn_cast(rows.getType()); if (!rowsType || !rowsType.hasStaticShape() || rowsType.getRank() != 2) return failure(); if (!logicalType || !logicalType.hasStaticShape() || logicalType.getRank() != 4) return failure(); if (logicalType.getDimSize(0) != 1) return failure(); if (rowsType.getElementType() != logicalType.getElementType()) return failure(); const int64_t channels = logicalType.getDimSize(1); const int64_t height = logicalType.getDimSize(2); const int64_t width = logicalType.getDimSize(3); if (rowsType.getDimSize(0) != height * width) return failure(); if (rowsType.getDimSize(1) != channels) return failure(); auto rowSliceType = RankedTensorType::get({width, channels}, logicalType.getElementType(), rowsType.getEncoding()); auto fragmentType = getRowStripFragmentType(logicalType); auto storageType = getRowStripStorageType(logicalType); auto batchOp = createSpatComputeBatch( rewriter, loc, TypeRange {storageType}, height, {}, ValueRange {rows}, [&](detail::SpatComputeBatchBodyArgs args) { Operation* anchorOp = rewriter.getInsertionBlock()->getParentOp(); Value rowStart = affineMulConst(rewriter, loc, args.lane, width, anchorOp); SmallVector rowOffsets {rowStart, rewriter.getIndexAttr(0)}; SmallVector rowSizes {rewriter.getIndexAttr(width), rewriter.getIndexAttr(channels)}; Value rowSlice = tensor::ExtractSliceOp::create( rewriter, loc, rowSliceType, args.inputs.front(), rowOffsets, rowSizes, getUnitStrides(rewriter, 2)); Value fragment = tensor::ExpandShapeOp::create( rewriter, loc, fragmentType, rowSlice, SmallVector {{0, 1, 2}, {3}}); insertRowStripFragment(fragment, args.outputs.front(), logicalType, args.lane, rewriter, loc); return success(); }); if (failed(batchOp)) return failure(); return batchOp->getResult(0); } FailureOr createRowStripStorageBlueprint(Value storage, RankedTensorType logicalType, PatternRewriter& rewriter, Location loc) { FailureOr value = describeRowStripPhysicalValue(storage, logicalType); if (failed(value)) return failure(); auto blueprint = spatial::SpatBlueprintOp::create( rewriter, loc, logicalType, storage, ValueRange {}, spatial::getNCHWLayout(rewriter.getContext()), spatial::getNHWCRowStripLayout(rewriter.getContext()), rewriter.getDenseI64ArrayAttr({}), rewriter.getDenseI64ArrayAttr({}), rewriter.getStringAttr(kRowStripIndexMap), spatial::getPhysicalViewMode(rewriter.getContext()), nullptr, nullptr, nullptr, nullptr, nullptr, nullptr); return blueprint.getOutput(); } FailureOr createRowStripAssemblyBlueprint(const RowStripPhysicalValue& value, PatternRewriter& rewriter, Location loc) { const int64_t laneCount = cast(value.storage.getType()).getDimSize(0); const int64_t tileChannels = value.fragmentType.getDimSize(3); auto nchwFragmentType = RankedTensorType::get( {1, tileChannels, 1, value.logicalType.getDimSize(3)}, value.fragmentType.getElementType(), value.fragmentType.getEncoding()); auto nchwStorageType = spatial::getGraphBatchPhysicalResultType(laneCount, nchwFragmentType); auto transposed = createSpatComputeBatch( rewriter, loc, TypeRange {nchwStorageType}, laneCount, {}, ValueRange {value.storage}, [&](detail::SpatComputeBatchBodyArgs args) { FailureOr fragment = extractGraphBatchPhysicalFragment( rewriter, loc, args.inputs.front(), args.lane, value.fragmentType); if (failed(fragment)) return failure(); Value nchw = createLinalgTranspose( *fragment, nchwFragmentType, {0, 3, 1, 2}, rewriter, loc); publishGraphBatchPhysicalFragment(rewriter, loc, nchw, args.outputs.front(), args.lane); return success(); }); if (failed(transposed)) return failure(); SmallVector entries; for (int64_t row = 0; row < value.logicalType.getDimSize(2); ++row) for (int64_t tile = 0; tile < value.tilesPerRow; ++tile) { const int64_t channelOffset = tile * tileChannels; entries.push_back({row * value.tilesPerRow + tile, 0, {0, channelOffset, row, 0}, {1, std::min(tileChannels, value.logicalType.getDimSize(1) - channelOffset), 1, value.logicalType.getDimSize(3)}}); } return createFragmentAssemblyBlueprint(transposed->getResult(0), value.logicalType, entries, "dense_nchw", kRowStripIndexMap, rewriter, loc); } template static FailureOr applyRowStripActivation(const RowStripPhysicalValue& value, PatternRewriter& rewriter, Location loc, BuildActivation buildActivation) { auto storageType = cast(value.storage.getType()); return mapGraphBatchFragments(value.storage, storageType, rewriter, loc, [&](Value fragment, RankedTensorType) { return FailureOr(buildActivation(fragment)); }); } FailureOr applyRowStripRelu(const RowStripPhysicalValue& value, PatternRewriter& rewriter, Location loc) { return applyRowStripActivation(value, rewriter, loc, [&](Value fragment) { return spatial::SpatReluOp::create(rewriter, loc, value.fragmentType, fragment).getResult(); }); } FailureOr applyRowStripSilu(const RowStripPhysicalValue& value, PatternRewriter& rewriter, Location loc) { return applyRowStripActivation(value, rewriter, loc, [&](Value fragment) { Value sigmoid = spatial::SpatSigmoidOp::create(rewriter, loc, value.fragmentType, fragment).getResult(); return spatial::SpatVMulOp::create(rewriter, loc, value.fragmentType, fragment, sigmoid).getResult(); }); } FailureOr applyRowStripBiasAdd(const RowStripPhysicalValue& value, Value bias, PatternRewriter& rewriter, Location loc) { DenseElementsAttr denseAttr; if (!isSupportedBiasAddValue(bias, value.logicalType, &denseAttr)) return failure(); FailureOr> channelValues = getBiasChannelValues(denseAttr, value.logicalType); if (failed(channelValues)) return failure(); auto storageType = cast(value.storage.getType()); auto biasStorageType = spatial::getGraphBatchPhysicalResultType(value.tilesPerRow, value.fragmentType); SmallVector biasValues( biasStorageType.getNumElements(), cast(rewriter.getZeroAttr(value.fragmentType.getElementType()))); const int64_t tileChannels = value.fragmentType.getDimSize(3); const int64_t width = value.fragmentType.getDimSize(2); for (int64_t channel = 0; channel < value.logicalType.getDimSize(1); ++channel) for (int64_t w = 0; w < width; ++w) biasValues[(channel / tileChannels) * width * tileChannels + w * tileChannels + channel % tileChannels] = (*channelValues)[channel]; Value biasStorage = getOrCreateConstant(rewriter, rewriter.getInsertionBlock()->getParentOp(), DenseElementsAttr::get(biasStorageType, biasValues), biasStorageType); const int64_t laneCount = storageType.getDimSize(0); auto batchOp = createSpatComputeBatch(rewriter, loc, TypeRange {storageType}, laneCount, {}, ValueRange {value.storage, biasStorage}, [&](detail::SpatComputeBatchBodyArgs args) { Operation* anchorOp = rewriter.getInsertionBlock()->getParentOp(); FailureOr fragment = extractGraphBatchPhysicalFragment( rewriter, loc, args.inputs[0], args.lane, value.fragmentType); Value tile = affineModConst(rewriter, loc, args.lane, value.tilesPerRow, anchorOp); FailureOr constant = extractGraphBatchPhysicalFragment( rewriter, loc, args.inputs[1], tile, value.fragmentType); if (failed(fragment) || failed(constant)) return failure(); Value added = spatial::SpatVAddOp::create( rewriter, loc, value.fragmentType, *fragment, *constant).getResult(); publishGraphBatchPhysicalFragment( rewriter, loc, added, args.outputs.front(), args.lane); return success(); }); if (failed(batchOp)) return failure(); return batchOp->getResult(0); } FailureOr applyRowStripAdd(const RowStripPhysicalValue& lhs, const RowStripPhysicalValue& rhs, PatternRewriter& rewriter, Location loc) { if (lhs.logicalType != rhs.logicalType) return failure(); const int64_t fragmentChannels = std::gcd(lhs.fragmentType.getDimSize(3), rhs.fragmentType.getDimSize(3)); auto fragmentType = RankedTensorType::get( {1, 1, lhs.logicalType.getDimSize(3), fragmentChannels}, lhs.logicalType.getElementType(), lhs.logicalType.getEncoding()); const int64_t tilesPerRow = ceilIntegerDivide(lhs.logicalType.getDimSize(1), fragmentChannels); auto storageType = spatial::getGraphBatchPhysicalResultType(lhs.logicalType.getDimSize(2) * tilesPerRow, fragmentType); const int64_t laneCount = storageType.getDimSize(0); auto batch = createSpatComputeBatch( rewriter, loc, TypeRange {storageType}, laneCount, {}, ValueRange {lhs.storage, rhs.storage}, [&](detail::SpatComputeBatchBodyArgs args) { auto extractFragment = [&](Value storage, const RowStripPhysicalValue& input) -> FailureOr { Operation* anchor = rewriter.getInsertionBlock()->getParentOp(); MLIRContext* context = rewriter.getContext(); AffineExpr lane = getAffineDimExpr(0, context); AffineExpr outputTile = lane % tilesPerRow; AffineExpr channelOffset = outputTile * fragmentChannels; Value sourceSlot = createOrFoldAffineApply( rewriter, loc, lane.floorDiv(tilesPerRow) * input.tilesPerRow + channelOffset.floorDiv(input.fragmentType.getDimSize(3)), ValueRange {args.lane}, anchor); FailureOr source = extractGraphBatchPhysicalFragment(rewriter, loc, storage, sourceSlot, input.fragmentType); if (failed(source) || input.fragmentType == fragmentType) return source; Value sourceOffset = createOrFoldAffineApply( rewriter, loc, channelOffset % input.fragmentType.getDimSize(3), ValueRange {args.lane}, anchor); return tensor::ExtractSliceOp::create( rewriter, loc, fragmentType, *source, SmallVector { rewriter.getIndexAttr(0), rewriter.getIndexAttr(0), rewriter.getIndexAttr(0), sourceOffset}, SmallVector {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1), rewriter.getIndexAttr(lhs.logicalType.getDimSize(3)), rewriter.getIndexAttr(fragmentChannels)}, getUnitStrides(rewriter, 4)).getResult(); }; FailureOr lhsFragment = extractFragment(args.inputs[0], lhs); FailureOr rhsFragment = extractFragment(args.inputs[1], rhs); if (failed(lhsFragment) || failed(rhsFragment)) return failure(); Value added = spatial::SpatVAddOp::create(rewriter, loc, fragmentType, *lhsFragment, *rhsFragment); publishGraphBatchPhysicalFragment(rewriter, loc, added, args.outputs.front(), args.lane); return success(); }); if (failed(batch)) return failure(); return batch->getResult(0); } FailureOr applyRowStripConcat(ArrayRef inputs, RankedTensorType outputType, PatternRewriter& rewriter, Location loc) { if (inputs.empty() || !outputType || !outputType.hasStaticShape() || outputType.getRank() != 4 || outputType.getDimSize(0) != 1) return failure(); int64_t channels = 0; for (const RowStripPhysicalValue& input : inputs) { if (input.logicalType.getElementType() != outputType.getElementType() || input.logicalType.getDimSize(0) != outputType.getDimSize(0) || input.logicalType.getDimSize(2) != outputType.getDimSize(2) || input.logicalType.getDimSize(3) != outputType.getDimSize(3)) return failure(); channels += input.logicalType.getDimSize(1); } if (channels != outputType.getDimSize(1)) return failure(); SmallVector storages; llvm::transform( inputs, std::back_inserter(storages), [](const RowStripPhysicalValue& input) { return input.storage; }); const int64_t tileWidth = outputType.getDimSize(3); auto fragmentType = getRowStripFragmentType(outputType); auto storageType = getRowStripStorageType(outputType); auto batch = createSpatComputeBatch( rewriter, loc, TypeRange {storageType}, outputType.getDimSize(2), {}, storages, [&](detail::SpatComputeBatchBodyArgs args) { Operation* anchor = rewriter.getInsertionBlock()->getParentOp(); SmallVector fragments; for (auto [inputIndex, input] : llvm::enumerate(inputs)) { Value tileStart = affineMulConst( rewriter, loc, args.lane, input.tilesPerRow, anchor); for (int64_t tile = 0; tile < input.tilesPerRow; ++tile) { Value slot = affineAddConst(rewriter, loc, tileStart, tile, anchor); FailureOr fragment = extractGraphBatchPhysicalFragment(rewriter, loc, args.inputs[inputIndex], slot, input.fragmentType); if (failed(fragment)) return failure(); int64_t channelOffset = tile * input.fragmentType.getDimSize(3); int64_t validChannels = std::min(input.fragmentType.getDimSize(3), input.logicalType.getDimSize(1) - channelOffset); auto validType = RankedTensorType::get( {1, 1, tileWidth, validChannels}, outputType.getElementType()); MixedSliceGeometry slice; slice.offsets.assign(4, rewriter.getIndexAttr(0)); slice.sizes = {rewriter.getIndexAttr(1), rewriter.getIndexAttr(1), rewriter.getIndexAttr(tileWidth), rewriter.getIndexAttr(validChannels)}; slice.strides.assign(4, rewriter.getIndexAttr(1)); Value valid = extractMixedSliceOrIdentity(rewriter, loc, *fragment, validType, slice); if (!valid) return failure(); fragments.push_back(valid); } } Value concatenated = spatial::SpatConcatOp::create(rewriter, loc, fragmentType, rewriter.getI64IntegerAttr(3), fragments); publishGraphBatchPhysicalFragment(rewriter, loc, concatenated, args.outputs.front(), args.lane); return success(); }); if (failed(batch)) return failure(); return batch->getResult(0); } } // namespace onnx_mlir