blazingly faster

src/PIM/Conversion/SpatialToPim/Patterns/ChannelLowering.cpp

@@ -1,7 +1,8 @@
+#include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
-
 #include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp"
 #include "src/Accelerators/PIM/Conversion/SpatialToPim/Patterns.hpp"
+#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
 #include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
 #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
 
@@ -18,6 +19,30 @@ static void copyRaptorDebugAttrs(Operation* source, Operation* target) {
   }
 }
 
+static Value createDestinationByteOffset(PatternRewriter& rewriter,
+                                         tensor::InsertSliceOp insert) {
+  auto destinationType = cast<RankedTensorType>(insert.getDestType());
+  SmallVector<int64_t> strides = computeRowMajorStrides(destinationType.getShape());
+  int64_t elementBytes = getElementTypeSizeInBytes(destinationType.getElementType());
+  Value total = arith::ConstantIndexOp::create(rewriter, insert.getLoc(), 0);
+  for (auto [dimension, offset] : llvm::enumerate(insert.getMixedOffsets())) {
+    int64_t scale = strides[dimension] * elementBytes;
+    Value component;
+    if (auto attribute = dyn_cast<Attribute>(offset)) {
+      component = arith::ConstantIndexOp::create(
+        rewriter, insert.getLoc(), cast<IntegerAttr>(attribute).getInt() * scale);
+    } else {
+      component = cast<Value>(offset);
+      if (scale != 1)
+        component = arith::MulIOp::create(
+          rewriter, insert.getLoc(), component,
+          arith::ConstantIndexOp::create(rewriter, insert.getLoc(), scale));
+    }
+    total = arith::AddIOp::create(rewriter, insert.getLoc(), total, component);
+  }
+  return total;
+}
+
 struct ChannelSendLowering : OpRewritePattern<spatial::SpatChannelSendOp> {
   using OpRewritePattern::OpRewritePattern;
 
@@ -40,7 +65,21 @@ struct ChannelReceiveLowering : OpRewritePattern<spatial::SpatChannelReceiveOp>
       rewriter.eraseOp(op);
       return success();
     }
-    auto outputType = cast<ShapedType>(op.getResult().getType());
+    auto outputType = cast<RankedTensorType>(op.getResult().getType());
+    tensor::InsertSliceOp destinationInsert;
+    if (op->hasOneUse()) {
+      auto insert = dyn_cast<tensor::InsertSliceOp>(*op->getUsers().begin());
+      auto destinationType = insert
+        ? dyn_cast<RankedTensorType>(insert.getDestType()) : RankedTensorType();
+      if (insert && insert.getSource() == op.getOutput()
+          && insert.getSourceType() == outputType
+          && insert->getBlock() == op->getBlock() && destinationType
+          && destinationType.hasStaticShape()
+          && isContiguousSubviewWithDynamicOffsets(
+            destinationType.getShape(), insert.getMixedOffsets(),
+            insert.getStaticSizes(), insert.getStaticStrides()))
+        destinationInsert = insert;
+    }
     Value outputBuffer =
       tensor::EmptyOp::create(rewriter, op.getLoc(), outputType.getShape(), outputType.getElementType()).getResult();
     auto sizeAttr = getTensorSizeInBytesAttr(rewriter, op.getOperation(), op.getResult());
@@ -50,7 +89,19 @@ struct ChannelReceiveLowering : OpRewritePattern<spatial::SpatChannelReceiveOp>
       rewriter, op.getLoc(), op.getResult().getType(), outputBuffer, *sizeAttr, op.getSourceCoreId());
     copyRaptorDebugAttrs(op.getOperation(), receive.getOperation());
     Value received = receive.getOutput();
-    rewriter.replaceOp(op, received);
+    if (!destinationInsert) {
+      rewriter.replaceOp(op, received);
+      return success();
+    }
+
+    rewriter.setInsertionPoint(destinationInsert);
+    Value targetOffset = createDestinationByteOffset(rewriter, destinationInsert);
+    Value zero = arith::ConstantIndexOp::create(rewriter, op.getLoc(), 0);
+    auto copy = pim::PimMemCopyOp::create(
+      rewriter, op.getLoc(), destinationInsert.getDestType(), targetOffset, zero,
+      destinationInsert.getDest(), received, *sizeAttr);
+    rewriter.replaceOp(destinationInsert, copy.getOutput());
+    rewriter.eraseOp(op);
     return success();
   }
 };

src/PIM/Conversion/SpatialToPim/SpatialToPimPass.cpp

@@ -1,4 +1,3 @@
-#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
@@ -175,11 +174,11 @@ void onnx_mlir::raptor::SpatialToPimPass::runOnOperation() {
 
   RewritePatternSet coreBodyPatterns(ctx);
   populateCoreBodyPatterns(coreBodyPatterns);
-  populateAffineToStdConversionPatterns(coreBodyPatterns);
   FrozenRewritePatternSet frozenCoreBodyPatterns(std::move(coreBodyPatterns));
 
   ConversionTarget coreBodyTarget(*ctx);
-  coreBodyTarget.addLegalDialect<PimDialect,
+  coreBodyTarget.addLegalDialect<affine::AffineDialect,
+                                 PimDialect,
                                  tensor::TensorDialect,
                                  arith::ArithDialect,
                                  bufferization::BufferizationDialect,
@@ -226,7 +225,8 @@ void onnx_mlir::raptor::SpatialToPimPass::runOnOperation() {
   eraseUnusedTensorPackingOps(funcOp, rewriter);
 
   ConversionTarget communicationTarget(*ctx);
-  communicationTarget.addLegalDialect<PimDialect,
+  communicationTarget.addLegalDialect<affine::AffineDialect,
+                                      PimDialect,
                                       tensor::TensorDialect,
                                       arith::ArithDialect,
                                       bufferization::BufferizationDialect,