Cose belle

src/PIM/Conversion/ONNXToSpatial/CompileTime.cpp

@@ -9,10 +9,12 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 
+#include <cstring>
 #include <utility>
 
 #include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
 #include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
+#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp"
 #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/CompileTime.hpp"
 #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
 #include "src/Dialect/ONNX/ONNXOps.hpp"
@@ -21,24 +23,7 @@ using namespace mlir;
 
 namespace onnx_mlir {
 
-namespace {
-
-static bool hasStaticUnitStrides(tensor::ExtractSliceOp extractSliceOp) {
-  return llvm::all_of(extractSliceOp.getStaticStrides(), [](int64_t stride) { return stride == 1; });
-}
-
-static bool hasConstantIndices(tensor::ExtractOp extractOp) {
-  return llvm::all_of(extractOp.getIndices(), [](Value index) { return matchConstantIndexValue(index).has_value(); });
-}
-
-static bool isStaticTensorResult(Operation* op) {
-  return llvm::all_of(op->getResultTypes(), [](Type type) {
-    auto shapedType = dyn_cast<ShapedType>(type);
-    return shapedType && shapedType.hasStaticShape();
-  });
-}
-
-static FailureOr<DenseElementsAttr> transposeDenseElements(DenseElementsAttr denseAttr, ArrayRef<int64_t> perms) {
+FailureOr<DenseElementsAttr> transposeDenseElementsAttr(DenseElementsAttr denseAttr, ArrayRef<int64_t> perms) {
   auto tensorType = dyn_cast<RankedTensorType>(denseAttr.getType());
   if (!tensorType)
     return failure();
@@ -59,7 +44,45 @@ static FailureOr<DenseElementsAttr> transposeDenseElements(DenseElementsAttr den
 
   auto transposedType = RankedTensorType::get(transposedShape, tensorType.getElementType(), tensorType.getEncoding());
   if (denseAttr.isSplat())
-    return DenseElementsAttr::get(transposedType, denseAttr.getSplatValue<Attribute>());
+    return DenseElementsAttr::getFromRawBuffer(transposedType, denseAttr.getRawData());
+
+  const unsigned elementBitWidth = tensorType.getElementTypeBitWidth();
+  const ArrayRef<char> inputData = denseAttr.getRawData();
+  if (elementBitWidth % 8 == 0) {
+    const size_t elementBytes = elementBitWidth / 8;
+    const size_t expectedBytes = denseAttr.getNumElements() * elementBytes;
+    if (inputData.size() == expectedBytes) {
+      SmallVector<char> transposedData(expectedBytes);
+      if (rank == 2 && perms[0] == 1 && perms[1] == 0) {
+        const int64_t rows = tensorType.getDimSize(0);
+        const int64_t columns = tensorType.getDimSize(1);
+        for (int64_t row = 0; row < rows; ++row)
+          for (int64_t column = 0; column < columns; ++column)
+            std::memcpy(transposedData.data() + (column * rows + row) * elementBytes,
+                        inputData.data() + (row * columns + column) * elementBytes,
+                        elementBytes);
+        return DenseElementsAttr::getFromRawBuffer(transposedType, transposedData);
+      }
+
+      SmallVector<int64_t> originalStrides = computeRowMajorStrides(tensorType.getShape());
+      SmallVector<int64_t> transposedStrides = computeRowMajorStrides(transposedShape);
+      SmallVector<int64_t> originalIndices(rank);
+      for (int64_t linearIndex = 0; linearIndex < tensorType.getNumElements(); ++linearIndex) {
+        int64_t remaining = linearIndex;
+        for (int64_t dim = 0; dim < rank; ++dim) {
+          originalIndices[dim] = originalStrides.empty() ? 0 : remaining / originalStrides[dim];
+          remaining = originalStrides.empty() ? 0 : remaining % originalStrides[dim];
+        }
+        int64_t transposedLinearIndex = 0;
+        for (int64_t dim = 0; dim < rank; ++dim)
+          transposedLinearIndex += originalIndices[perms[dim]] * transposedStrides[dim];
+        std::memcpy(transposedData.data() + transposedLinearIndex * elementBytes,
+                    inputData.data() + linearIndex * elementBytes,
+                    elementBytes);
+      }
+      return DenseElementsAttr::getFromRawBuffer(transposedType, transposedData);
+    }
+  }
 
   SmallVector<Attribute> originalValues(denseAttr.getValues<Attribute>());
   SmallVector<Attribute> transposedValues(originalValues.size());
@@ -84,16 +107,30 @@ static FailureOr<DenseElementsAttr> transposeDenseElements(DenseElementsAttr den
   return DenseElementsAttr::get(transposedType, transposedValues);
 }
 
+namespace {
+
+static bool hasStaticUnitStrides(tensor::ExtractSliceOp extractSliceOp) {
+  return llvm::all_of(extractSliceOp.getStaticStrides(), [](int64_t stride) { return stride == 1; });
+}
+
+static bool hasConstantIndices(tensor::ExtractOp extractOp) {
+  return llvm::all_of(extractOp.getIndices(), [](Value index) { return matchConstantIndexValue(index).has_value(); });
+}
+
+static bool isStaticTensorResult(Operation* op) {
+  return llvm::all_of(op->getResultTypes(), [](Type type) {
+    auto shapedType = dyn_cast<ShapedType>(type);
+    return shapedType && shapedType.hasStaticShape();
+  });
+}
+
 static FailureOr<DenseElementsAttr> reshapeDenseElements(DenseElementsAttr denseAttr, RankedTensorType resultType) {
   auto sourceType = dyn_cast<RankedTensorType>(denseAttr.getType());
-  if (!sourceType || !resultType || sourceType.getNumElements() != resultType.getNumElements())
+  if (!sourceType || !resultType || sourceType.getNumElements() != resultType.getNumElements()
+      || sourceType.getElementType() != resultType.getElementType())
     return failure();
 
-  if (denseAttr.isSplat())
-    return DenseElementsAttr::get(resultType, denseAttr.getSplatValue<Attribute>());
-
-  SmallVector<Attribute> values(denseAttr.getValues<Attribute>());
-  return DenseElementsAttr::get(resultType, values);
+  return DenseElementsAttr::getFromRawBuffer(resultType, denseAttr.getRawData());
 }
 
 static FailureOr<DenseElementsAttr> extractSliceDenseElements(DenseElementsAttr denseAttr,
@@ -161,7 +198,7 @@ static DenseElementsAttr getHostConstantDenseElementsAttrImpl(Value value, llvm:
     perm.reserve(transposeOp.getPermAttr().size());
     for (IntegerAttr attr : transposeOp.getPermAttr().getAsRange<IntegerAttr>())
       perm.push_back(attr.getInt());
-    auto transposedAttr = transposeDenseElements(inputAttr, perm);
+    auto transposedAttr = transposeDenseElementsAttr(inputAttr, perm);
     return succeeded(transposedAttr) ? *transposedAttr : nullptr;
   }
 
@@ -171,7 +208,7 @@ static DenseElementsAttr getHostConstantDenseElementsAttrImpl(Value value, llvm:
       return nullptr;
 
     SmallVector<int64_t> perm(transposeOp.getPermutation().begin(), transposeOp.getPermutation().end());
-    auto transposedAttr = transposeDenseElements(inputAttr, perm);
+    auto transposedAttr = transposeDenseElementsAttr(inputAttr, perm);
     return succeeded(transposedAttr) ? *transposedAttr : nullptr;
   }
 
@@ -219,6 +256,9 @@ getCompileTimeSourceImpl(Operation* op, llvm::SmallPtrSetImpl<Operation*>& visit
 
   chainLength += 1;
 
+  if (!isShapingOnlyOp(op))
+    return std::nullopt;
+
   if (auto extractOp = dyn_cast<tensor::ExtractOp>(op))
     return hasConstantIndices(extractOp)
            ? getCompileTimeSourceImpl(extractOp.getTensor().getDefiningOp(), visited, chainLength)

src/PIM/Conversion/ONNXToSpatial/CompileTime.hpp

@@ -4,6 +4,8 @@
 #include "mlir/IR/Operation.h"
 #include "mlir/IR/Value.h"
 
+#include "llvm/ADT/ArrayRef.h"
+
 namespace onnx_mlir {
 
 struct CompileTimeSource {
@@ -19,4 +21,7 @@ bool isCompileTimeOp(mlir::Operation* op);
 
 mlir::DenseElementsAttr getHostConstDenseElementsAttr(mlir::Value value);
 
+mlir::FailureOr<mlir::DenseElementsAttr> transposeDenseElementsAttr(
+  mlir::DenseElementsAttr denseAttr, llvm::ArrayRef<int64_t> permutation);
+
 } // namespace onnx_mlir

src/PIM/Conversion/ONNXToSpatial/LowerSpatialPlansPass.cpp

@@ -20,6 +20,7 @@
 #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns.hpp"
 #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/PlanLowering.hpp"
 #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
+#include "src/Accelerators/PIM/Dialect/Spatial/Transforms/MergeComputeNodes/SpatialDataflowCsvExporter.hpp"
 #include "src/Accelerators/PIM/Pass/PIMPasses.h"
 #include "src/Dialect/ONNX/ONNXOps.hpp"
 
@@ -330,17 +331,25 @@ struct LowerSpatialPlansPass final : PassWrapper<LowerSpatialPlansPass, Operatio
     RewritePatternSet helperPatterns(ctx);
     populateGemmPatterns(helperPatterns, ctx);
     populateTransposePatterns(helperPatterns, ctx);
-    if (failed(applyPartialConversion(moduleOp, helperTarget, std::move(helperPatterns)))) {
-      moduleOp.emitError("failed to lower helper ONNX ops emitted by selected Spatial plan lowering");
-      signalPassFailure();
-      return;
+    FrozenRewritePatternSet frozenHelperPatterns(
+      std::move(helperPatterns));
+    SmallVector<Operation*> topLevelHelperOps;
+    funcOp.walk([&](Operation* op) {
+      if (isa<spatial::SpatGraphCompute,
+              spatial::SpatGraphComputeBatch>(op))
+        return WalkResult::skip();
+      if (isa<ONNXGemmOp, ONNXTransposeOp>(op))
+        topLevelHelperOps.push_back(op);
+      return WalkResult::advance();
+    });
+    for (Operation *helper : topLevelHelperOps) {
+      if (failed(applyPartialConversion(
+            helper, helperTarget, frozenHelperPatterns))) {
+        moduleOp.emitError("failed to lower helper ONNX ops emitted by selected Spatial plan lowering");
+        signalPassFailure();
+        return;
+      }
     }
-    FrozenRewritePatternSet nestedHelperPatterns([&] {
-      RewritePatternSet patterns(ctx);
-      populateGemmPatterns(patterns, ctx);
-      populateTransposePatterns(patterns, ctx);
-      return patterns;
-    }());
     ConversionTarget nestedHelperTarget(*ctx);
     nestedHelperTarget.addLegalDialect<spatial::SpatialDialect,
                                        tensor::TensorDialect,
@@ -356,7 +365,8 @@ struct LowerSpatialPlansPass final : PassWrapper<LowerSpatialPlansPass, Operatio
         computeLikeOps.push_back(op);
     });
     for (Operation* op : computeLikeOps) {
-      if (failed(applyFullConversion(op, nestedHelperTarget, nestedHelperPatterns))) {
+      if (failed(applyFullConversion(
+            op, nestedHelperTarget, frozenHelperPatterns))) {
         op->emitOpError("failed to lower nested helper ONNX ops emitted by selected Spatial plan lowering");
         signalPassFailure();
         return;
@@ -392,6 +402,12 @@ struct LowerSpatialPlansPass final : PassWrapper<LowerSpatialPlansPass, Operatio
       signalPassFailure();
     } else {
       dumpModule(moduleOp, "spatial1_graph");
+      spatial::SpatialDataflowExportStage exportMode = spatial::getSpatialDataflowExportStage();
+      if (spatial::shouldExportSpatialDataflowStage(exportMode, spatial::SpatialDataflowExportStage::Spatial1)
+          && failed(spatial::exportSpatialDataflowCsvGraph(funcOp, "spatial1_graph"))) {
+        signalPassFailure();
+        return;
+      }
     }
 
     if (!verifyLogicalPhase("at the end of LowerSpatialPlans"))

src/PIM/Conversion/ONNXToSpatial/Patterns/Math/ReduceMean.cpp

@@ -5,7 +5,6 @@
 #include "llvm/ADT/SmallVector.h"
 
 #include <algorithm>
-#include <numeric>
 #include <optional>
 #include <type_traits>
 
@@ -122,14 +121,6 @@ static RankedTensorType getKeepdimsType(RankedTensorType inputType, Type element
   return RankedTensorType::get(shape, elementType, inputType.getEncoding());
 }
 
-static RankedTensorType getCompactKeptType(RankedTensorType inputType, Type elementType, ArrayRef<bool> reducedAxes) {
-  SmallVector<int64_t> shape;
-  for (auto [dim, isReduced] : llvm::zip_equal(inputType.getShape(), reducedAxes))
-    if (!isReduced)
-      shape.push_back(dim);
-  return RankedTensorType::get(shape, elementType, inputType.getEncoding());
-}
-
 static RankedTensorType getReducedSliceType(RankedTensorType inputType, ArrayRef<bool> reducedAxes) {
   SmallVector<int64_t> shape;
   shape.reserve(inputType.getRank());
@@ -228,59 +219,80 @@ static FailureOr<Value> buildReduceMeanKeepdimsBatch(Value input,
   return (*batchOp).getResult(0);
 }
 
-static Value buildKeepdimsFromLanePackedBatch(Value batchValue,
-                                              RankedTensorType keepdimsType,
-                                              RankedTensorType compactKeptType,
-                                              ArrayRef<bool> reducedAxes,
-                                              ConversionPatternRewriter& rewriter,
-                                              Location loc) {
-  auto batchType = cast<RankedTensorType>(batchValue.getType());
-  if (batchType == keepdimsType)
-    return batchValue;
+static FailureOr<Value> buildReduceMeanKeepdimsBlueprint(
+    Value batchValue, RankedTensorType keepdimsType,
+    ArrayRef<bool> reducedAxes, ConversionPatternRewriter& rewriter,
+    Location loc) {
+  auto batchType = dyn_cast<RankedTensorType>(batchValue.getType());
+  int64_t rank = keepdimsType.getRank();
+  if (!batchType || !batchType.hasStaticShape()
+      || !keepdimsType.hasStaticShape()
+      || static_cast<int64_t>(reducedAxes.size()) != rank
+      || batchType.getRank() != rank + 1
+      || batchType.getElementType() != keepdimsType.getElementType())
+    return failure();
 
-  SmallVector<ReassociationIndices> collapseToFlat {{}};
-  for (int64_t axis = 0; axis < batchType.getRank(); ++axis)
-    collapseToFlat.front().push_back(axis);
-
-  SmallVector<ReassociationIndices> expandFlatToCompact(1);
-  for (int64_t axis = 0; axis < compactKeptType.getRank(); ++axis)
-    expandFlatToCompact.front().push_back(axis);
-
-  SmallVector<ReassociationIndices> expandCompactToKeepdims;
-  ReassociationIndices pendingLeadingReducedAxes;
+  int64_t laneCount = 1;
+  SmallVector<int64_t> keptAxes;
+  SmallVector<int64_t> keptAxisStrides;
   for (auto [axis, isReduced] : llvm::enumerate(reducedAxes)) {
-    if (isReduced) {
-      if (expandCompactToKeepdims.empty())
-        pendingLeadingReducedAxes.push_back(axis);
-      else
-        expandCompactToKeepdims.back().push_back(axis);
-      continue;
-    }
-
-    expandCompactToKeepdims.emplace_back();
-    auto& group = expandCompactToKeepdims.back();
-    group.append(pendingLeadingReducedAxes.begin(), pendingLeadingReducedAxes.end());
-    pendingLeadingReducedAxes.clear();
-    group.push_back(axis);
+    int64_t dim = keepdimsType.getDimSize(axis);
+    if (dim <= 0 || (isReduced && dim != 1))
+      return failure();
+    if (!isReduced)
+      keptAxes.push_back(axis);
   }
-  if (!pendingLeadingReducedAxes.empty())
-    expandCompactToKeepdims.back().append(pendingLeadingReducedAxes.begin(), pendingLeadingReducedAxes.end());
+  keptAxisStrides.resize(keptAxes.size(), 1);
+  for (int64_t index = static_cast<int64_t>(keptAxes.size()) - 1;
+       index >= 0; --index) {
+    keptAxisStrides[index] = laneCount;
+    int64_t dim = keepdimsType.getDimSize(keptAxes[index]);
+    if (laneCount > std::numeric_limits<int64_t>::max() / dim)
+      return failure();
+    laneCount *= dim;
+  }
+  if (batchType.getDimSize(0) != laneCount
+      || llvm::any_of(batchType.getShape().drop_front(),
+                      [](int64_t dim) { return dim != 1; }))
+    return failure();
 
-  if (batchType.getNumElements() != batchType.getDimSize(0))
-    return {};
-  auto reshapeCompute =
-    createSpatCompute<1>(rewriter, loc, TypeRange {keepdimsType}, {}, ValueRange {batchValue}, [&](Value input) {
-      auto flatType = RankedTensorType::get({batchType.getNumElements()}, batchType.getElementType(), batchType.getEncoding());
-      Value flat = tensor::CollapseShapeOp::create(rewriter, loc, flatType, input, collapseToFlat);
-      Value compact = flat;
-      if (compactKeptType != flatType)
-        compact = tensor::ExpandShapeOp::create(rewriter, loc, compactKeptType, flat, expandFlatToCompact);
-      Value keepdims = compact;
-      if (keepdimsType != compactKeptType)
-        keepdims = tensor::ExpandShapeOp::create(rewriter, loc, keepdimsType, compact, expandCompactToKeepdims);
-      spatial::SpatYieldOp::create(rewriter, loc, keepdims);
-    });
-  return reshapeCompute.getResult(0);
+  SmallVector<int64_t> operandIndices(laneCount, 0);
+  SmallVector<int64_t> sourceSlots;
+  SmallVector<int64_t> sourceOffsets(laneCount, 0);
+  SmallVector<int64_t> fragmentOffsets;
+  sourceSlots.reserve(laneCount);
+  fragmentOffsets.reserve(laneCount * rank);
+  for (int64_t lane = 0; lane < laneCount; ++lane) {
+    sourceSlots.push_back(lane);
+    size_t keptAxisIndex = 0;
+    for (auto [axis, isReduced] : llvm::enumerate(reducedAxes)) {
+      if (isReduced) {
+        fragmentOffsets.push_back(0);
+        continue;
+      }
+      int64_t dim = keepdimsType.getDimSize(axis);
+      fragmentOffsets.push_back(
+        (lane / keptAxisStrides[keptAxisIndex]) % dim);
+      ++keptAxisIndex;
+    }
+  }
+  SmallVector<int64_t> fragmentSizes(fragmentOffsets.size(), 1);
+  SmallVector<int64_t> fragmentStrides(fragmentOffsets.size(), 1);
+  return spatial::SpatBlueprintOp::create(
+           rewriter, loc, keepdimsType, batchValue, ValueRange {},
+           rewriter.getStringAttr("nchw"),
+           rewriter.getStringAttr("fragmented"),
+           rewriter.getDenseI64ArrayAttr(fragmentOffsets),
+           rewriter.getDenseI64ArrayAttr(fragmentSizes),
+           rewriter.getStringAttr("reduce_mean_keepdims_fragments"),
+           rewriter.getStringAttr("fragment_assembly"),
+           rewriter.getDenseI64ArrayAttr(operandIndices),
+           rewriter.getDenseI64ArrayAttr(sourceSlots),
+           rewriter.getDenseI64ArrayAttr(sourceOffsets),
+           rewriter.getDenseI64ArrayAttr(fragmentStrides),
+           rewriter.getStringAttr("disjoint"),
+           rewriter.getStringAttr("complete"))
+    .getOutput();
 }
 
 static SmallVector<ReassociationIndices> buildCollapseReassociation(ArrayRef<bool> reducedAxes) {
@@ -357,26 +369,36 @@ struct ReduceMeanToSpatialCompute : OpConversionPattern<ReduceMeanOp> {
 
     Location loc = reduceMeanOp.getLoc();
     RankedTensorType leafType = getAllOnesType(inputType, resultType.getElementType());
-    RankedTensorType compactKeptType = getCompactKeptType(inputType, resultType.getElementType(), reducedAxes);
     RankedTensorType keepdimsType = getKeepdimsType(inputType, resultType.getElementType(), reducedAxes);
     int64_t laneCount = 1;
-    for (int64_t dim : compactKeptType.getShape())
+    for (auto [dim, isReduced] : llvm::zip_equal(keepdimsType.getShape(), reducedAxes)) {
+      if (isReduced)
+        continue;
+      if (dim <= 0 || laneCount > std::numeric_limits<int32_t>::max() / dim)
+        return rewriter.notifyMatchFailure(
+          reduceMeanOp, "ReduceMean physical lane count is not representable");
       laneCount *= dim;
+    }
     RankedTensorType batchType = getLanePackedKeepdimsType(laneCount, leafType);
 
     auto lanePackedKeepdims =
       buildReduceMeanKeepdimsBatch(adaptor.getData(), reducedAxes, batchType, leafType, rewriter, loc);
     if (failed(lanePackedKeepdims))
       return failure();
-    Value reducedKeepdims =
-      buildKeepdimsFromLanePackedBatch(*lanePackedKeepdims, keepdimsType, compactKeptType, reducedAxes, rewriter, loc);
+    auto reducedKeepdims = buildReduceMeanKeepdimsBlueprint(
+      *lanePackedKeepdims, keepdimsType, reducedAxes, rewriter, loc);
+    if (failed(reducedKeepdims))
+      return rewriter.notifyMatchFailure(
+        reduceMeanOp,
+        "cannot build physical-fragment ReduceMean keepdims reconstruction");
 
     if (semantics->keepdims != 0) {
-      rewriter.replaceOp(reduceMeanOp, reducedKeepdims);
+      rewriter.replaceOp(reduceMeanOp, *reducedKeepdims);
       return success();
     }
 
-    Value reduced = squeezeReducedAxes(reducedKeepdims, resultType, reducedAxes, rewriter, loc);
+    Value reduced = squeezeReducedAxes(
+      *reducedKeepdims, resultType, reducedAxes, rewriter, loc);
     rewriter.replaceOp(reduceMeanOp, reduced);
     return success();
   }

src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Transpose.cpp

@@ -5,7 +5,7 @@
 
 #include "llvm/ADT/SmallVector.h"
 
-#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp"
+#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/CompileTime.hpp"
 #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp"
 #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns.hpp"
 #include "src/Dialect/ONNX/ONNXOps.hpp"
@@ -52,35 +52,12 @@ static FailureOr<Value> materializeTransposedConstant(Value input,
     return failure();
   }
 
-  if (denseAttr.isSplat())
-    return getOrCreateConstant(rewriter,
-                               rewriter.getInsertionBlock()->getParentOp(),
-                               DenseElementsAttr::get(resultType, denseAttr.getSplatValue<Attribute>()),
-                               resultType);
-
-  SmallVector<Attribute> inputValues(denseAttr.getValues<Attribute>());
-  SmallVector<Attribute> resultValues(inputValues.size());
-  SmallVector<int64_t> inputStrides = computeRowMajorStrides(inputType.getShape());
-  SmallVector<int64_t> resultStrides = computeRowMajorStrides(resultType.getShape());
-  SmallVector<int64_t> inputIndices(inputType.getRank(), 0);
-
-  for (auto [linearIndex, value] : llvm::enumerate(inputValues)) {
-    int64_t remaining = static_cast<int64_t>(linearIndex);
-    for (int64_t dim = 0; dim < inputType.getRank(); ++dim) {
-      inputIndices[dim] = inputStrides.empty() ? 0 : remaining / inputStrides[dim];
-      remaining = inputStrides.empty() ? 0 : remaining % inputStrides[dim];
-    }
-
-    int64_t resultLinearIndex = 0;
-    for (int64_t dim = 0; dim < resultType.getRank(); ++dim)
-      resultLinearIndex += inputIndices[permutation[dim]] * resultStrides[dim];
-
-    resultValues[resultLinearIndex] = value;
-  }
-
+  auto transposedAttr = transposeDenseElementsAttr(denseAttr, permutation);
+  if (failed(transposedAttr) || transposedAttr->getType() != resultType)
+    return failure();
   return getOrCreateConstant(rewriter,
                              rewriter.getInsertionBlock()->getParentOp(),
-                             DenseElementsAttr::get(resultType, resultValues),
+                             *transposedAttr,
                              resultType);
 }
 

src/PIM/Conversion/SpatialToPim/BatchCoreLoweringPatterns.cpp

@@ -400,6 +400,11 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul
     if (isa<spatial::SpatYieldOp>(op))
       continue;
 
+    // Cloning a region-bearing operation may leave the rewriter inside that
+    // region. Every old-block operation is lowered at the core-batch body
+    // boundary.
+    rewriter.setInsertionPointToEnd(newBlock);
+
     if (auto blueprint = dyn_cast<spatial::SpatBlueprintOp>(op)) {
       std::optional<StringRef> modeAttr = blueprint.getMode();
       if (modeAttr && *modeAttr == "fragment_assembly") {

src/PIM/Conversion/SpatialToPim/Common.cpp

@@ -8,6 +8,8 @@
 #include <limits>
 
 #include "Common.hpp"
+#include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp"
+#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
 #include "src/Accelerators/PIM/Common/IR/LoopUtils.hpp"
 #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
 #include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp"
@@ -192,21 +194,23 @@ forEachContiguousDestinationChunk(ArrayRef<int64_t> destShape,
 }
 
 static mlir::Value
-createSteppedOffset(OpBuilder& builder, Location loc, mlir::Value start, mlir::Value index, int64_t stepBytes) {
+createSteppedOffset(OpBuilder& builder, Location loc, mlir::Value start, mlir::Value index,
+                    int64_t stepBytes, Operation *constantAnchor) {
   if (stepBytes == 0)
     return start;
-  mlir::Value step = arith::ConstantIndexOp::create(builder, loc, stepBytes);
-  mlir::Value scaled = arith::MulIOp::create(builder, loc, index, step).getResult();
-  return arith::AddIOp::create(builder, loc, start, scaled).getResult();
+  return createOrFoldAffineApply(
+    builder, loc, builder.getAffineDimExpr(0) + builder.getAffineDimExpr(1) * stepBytes,
+    ValueRange {start, index}, constantAnchor);
 }
 
 static mlir::Value createIndexedOffset(OpBuilder& builder,
                                        Location loc,
                                        mlir::Value indexArg,
-                                       ArrayRef<int64_t> values) {
+                                       ArrayRef<int64_t> values,
+                                       Operation *constantAnchor) {
   assert(!values.empty() && "expected lane-indexed values");
   if (llvm::all_of(values.drop_front(), [&](int64_t value) { return value == values.front(); }))
-    return arith::ConstantIndexOp::create(builder, loc, values.front());
+    return getOrCreateIndexConstant(builder, constantAnchor, values.front());
 
   if (values.size() >= 2) {
     int64_t step = values[1] - values[0];
@@ -214,21 +218,18 @@ static mlir::Value createIndexedOffset(OpBuilder& builder,
       return values[index] == values.front() + static_cast<int64_t>(index) * step;
     });
     if (arithmetic) {
-      mlir::Value base = arith::ConstantIndexOp::create(builder, loc, values.front());
-      mlir::Value stepValue = arith::ConstantIndexOp::create(builder, loc, step);
-      mlir::Value scaledIndex = arith::MulIOp::create(builder, loc, indexArg, stepValue).getResult();
-      return arith::AddIOp::create(builder, loc, base, scaledIndex).getResult();
+      return createOrFoldAffineApply(
+        builder, loc, builder.getAffineDimExpr(0) * step + values.front(),
+        ValueRange {indexArg}, constantAnchor);
     }
   }
 
-  mlir::Value selected = arith::ConstantIndexOp::create(builder, loc, values.front());
-  for (auto [lane, value] : llvm::enumerate(values.drop_front())) {
-    mlir::Value indexValue = arith::ConstantIndexOp::create(builder, loc, static_cast<int64_t>(lane + 1));
-    mlir::Value cmp = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, indexArg, indexValue);
-    mlir::Value candidate = arith::ConstantIndexOp::create(builder, loc, value);
-    selected = arith::SelectOp::create(builder, loc, cmp, candidate, selected);
-  }
-  return selected;
+  RankedTensorType tableType = RankedTensorType::get(
+    {static_cast<int64_t>(values.size())}, builder.getI64Type());
+  DenseElementsAttr tableAttr = DenseElementsAttr::get(tableType, values);
+  mlir::Value table = getOrCreateConstant(builder, constantAnchor, tableAttr, tableType);
+  mlir::Value selected = tensor::ExtractOp::create(builder, loc, table, ValueRange {indexArg});
+  return arith::IndexCastOp::create(builder, loc, builder.getIndexType(), selected).getResult();
 }
 
 struct FragmentAssemblyCopyRunFamily {
@@ -433,11 +434,11 @@ static FailureOr<mlir::Value> emitFragmentAssemblyCopyRun(OpBuilder& builder,
   mlir::Value hostStart;
   mlir::Value sourceStart;
   if (laneArg) {
-    hostStart = createIndexedOffset(builder, loc, *laneArg, run.hostStartBytesByLane);
-    sourceStart = createIndexedOffset(builder, loc, *laneArg, run.sourceStartBytesByLane);
+    hostStart = createIndexedOffset(builder, loc, *laneArg, run.hostStartBytesByLane, anchor);
+    sourceStart = createIndexedOffset(builder, loc, *laneArg, run.sourceStartBytesByLane, anchor);
   } else {
-    hostStart = arith::ConstantIndexOp::create(builder, loc, run.hostStartBytesByLane.front());
-    sourceStart = arith::ConstantIndexOp::create(builder, loc, run.sourceStartBytesByLane.front());
+    hostStart = getOrCreateIndexConstant(builder, anchor, run.hostStartBytesByLane.front());
+    sourceStart = getOrCreateIndexConstant(builder, anchor, run.sourceStartBytesByLane.front());
   }
 
   if (hostRunStartDelta)
@@ -459,9 +460,9 @@ static FailureOr<mlir::Value> emitFragmentAssemblyCopyRun(OpBuilder& builder,
       .getOutput();
   }
 
-  mlir::Value lowerBound = arith::ConstantIndexOp::create(builder, loc, 0);
-  mlir::Value upperBound = arith::ConstantIndexOp::create(builder, loc, run.count);
-  mlir::Value step = arith::ConstantIndexOp::create(builder, loc, 1);
+  mlir::Value lowerBound = getOrCreateIndexConstant(builder, anchor, 0);
+  mlir::Value upperBound = getOrCreateIndexConstant(builder, anchor, run.count);
+  mlir::Value step = getOrCreateIndexConstant(builder, anchor, 1);
   FailureOr<NormalizedLoopResult> loop = buildNormalizedScfFor(
     builder,
     loc,
@@ -474,9 +475,10 @@ static FailureOr<mlir::Value> emitFragmentAssemblyCopyRun(OpBuilder& builder,
         mlir::Value flatIndex,
         ValueRange iterArgs,
         SmallVectorImpl<mlir::Value>& yielded) {
-      mlir::Value hostOffset = createSteppedOffset(loopBuilder, bodyLoc, hostStart, flatIndex, run.hostStepBytes);
+      mlir::Value hostOffset = createSteppedOffset(
+        loopBuilder, bodyLoc, hostStart, flatIndex, run.hostStepBytes, anchor);
       mlir::Value sourceOffset =
-        createSteppedOffset(loopBuilder, bodyLoc, sourceStart, flatIndex, run.sourceStepBytes);
+        createSteppedOffset(loopBuilder, bodyLoc, sourceStart, flatIndex, run.sourceStepBytes, anchor);
       mlir::Value copied =
         pim::PimMemCopyDevToHostOp::create(loopBuilder,
                                            bodyLoc,
@@ -506,9 +508,9 @@ static FailureOr<mlir::Value> emitFragmentAssemblyCopyRunFamily(OpBuilder& build
     return emitFragmentAssemblyCopyRun(
       builder, loc, family.prototype, hostTarget, anchor, laneArg, baseHostOffset);
 
-  mlir::Value lowerBound = arith::ConstantIndexOp::create(builder, loc, 0);
-  mlir::Value upperBound = arith::ConstantIndexOp::create(builder, loc, family.sourceRunStartDeltas.size());
-  mlir::Value step = arith::ConstantIndexOp::create(builder, loc, 1);
+  mlir::Value lowerBound = getOrCreateIndexConstant(builder, anchor, 0);
+  mlir::Value upperBound = getOrCreateIndexConstant(builder, anchor, family.sourceRunStartDeltas.size());
+  mlir::Value step = getOrCreateIndexConstant(builder, anchor, 1);
   FailureOr<NormalizedLoopResult> outerLoop = buildNormalizedScfFor(
     builder,
     loc,
@@ -522,9 +524,9 @@ static FailureOr<mlir::Value> emitFragmentAssemblyCopyRunFamily(OpBuilder& build
         ValueRange iterArgs,
         SmallVectorImpl<mlir::Value>& yielded) {
       mlir::Value sourceRunStartDelta =
-        createIndexedOffset(loopBuilder, bodyLoc, runIndex, family.sourceRunStartDeltas);
+        createIndexedOffset(loopBuilder, bodyLoc, runIndex, family.sourceRunStartDeltas, anchor);
       mlir::Value hostRunStartDelta =
-        createIndexedOffset(loopBuilder, bodyLoc, runIndex, family.hostRunStartDeltas);
+        createIndexedOffset(loopBuilder, bodyLoc, runIndex, family.hostRunStartDeltas, anchor);
       FailureOr<mlir::Value> copied = emitFragmentAssemblyCopyRun(loopBuilder,
                                                                   bodyLoc,
                                                                   family.prototype,

src/PIM/Conversion/SpatialToPim/CoreLoweringPatterns.cpp

@@ -10,7 +10,9 @@
 #include "Conversion/ONNXToSpatial/Common/Common.hpp"
 #include "Conversion/SpatialToPim/SpatialToPimPass.hpp"
 #include "src/Accelerators/PIM/Common/IR/BatchCoreUtils.hpp"
+#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp"
 #include "src/Accelerators/PIM/Common/IR/LoopUtils.hpp"
+#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp"
 #include "src/Accelerators/PIM/Common/PimCommon.hpp"
 #include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp"
 #include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp"
@@ -180,16 +182,79 @@ static LogicalResult collectHelperComputeChain(spatial::SpatScheduledCompute com
   return success();
 }
 
+static bool isHostMaterializableHelperOp(Operation* op) {
+  if (isa<spatial::SpatYieldOp>(op))
+    return true;
+  if (isa<arith::ConstantOp>(op) || op->hasTrait<OpTrait::ConstantLike>())
+    return true;
+  if (auto blueprint = dyn_cast<spatial::SpatBlueprintOp>(op)) {
+    std::optional<StringRef> mode = blueprint.getMode();
+    return mode && *mode == "fragment_assembly";
+  }
+  return isShapingOnlyOp(op) || isPureIndexComputationOp(op);
+}
+
+static FailureOr<DenseMap<Value, Attribute>>
+analyzeHostMaterializableHelper(spatial::SpatScheduledCompute computeOp) {
+  DenseMap<Value, Attribute> folded;
+  for (auto [weightIndex, weight] : llvm::enumerate(computeOp.getWeights())) {
+    auto argument = computeOp.getWeightArgument(weightIndex);
+    if (!argument)
+      return failure();
+    Attribute constant;
+    if (matchPattern(weight, m_Constant(&constant)))
+      folded[*argument] = constant;
+  }
+  Block& block = computeOp.getBody().front();
+  for (Operation& op : block) {
+    if (!isHostMaterializableHelperOp(&op))
+      return failure();
+    if (isa<spatial::SpatYieldOp, spatial::SpatBlueprintOp>(op)
+        || (isShapingOnlyOp(&op) && !isPureIndexComputationOp(&op)))
+      continue;
+    if (isa<arith::ConstantOp>(op) || op.hasTrait<OpTrait::ConstantLike>()) {
+      for (Value result : op.getResults()) {
+        Attribute constant;
+        if (!matchPattern(result, m_Constant(&constant)))
+          return failure();
+        folded[result] = constant;
+      }
+      continue;
+    }
+    if (!isPureIndexComputationOp(&op) || op.getNumRegions() != 0)
+      return failure();
+    SmallVector<Attribute> operands;
+    for (Value operand : op.getOperands()) {
+      auto it = folded.find(operand);
+      if (it == folded.end())
+        return failure();
+      operands.push_back(it->second);
+    }
+    SmallVector<OpFoldResult> results;
+    if (failed(op.fold(operands, results))
+        || results.size() != op.getNumResults())
+      return failure();
+    for (auto [result, foldResult] : llvm::zip(op.getResults(), results)) {
+      auto attribute = dyn_cast<Attribute>(foldResult);
+      if (!attribute)
+        return failure();
+      folded[result] = attribute;
+    }
+  }
+  return folded;
+}
+
 static bool inlineInputlessHelperComputeForWeightLikeUsers(spatial::SpatScheduledCompute computeOp,
                                                            IRRewriter& rewriter,
                                                            OperationFolder& constantFolder) {
   if (!computeOp.getInputs().empty() || computeOp.getNumResults() != 1)
     return false;
+  if (computeOp.getResult(0).use_empty())
+    return false;
   if (!llvm::all_of(computeOp.getResult(0).getUsers(), [](Operation* user) {
         return isa<spatial::SpatScheduledCompute, spatial::SpatScheduledComputeBatch, pim::PimCoreOp, pim::PimCoreBatchOp>(user);
       }))
     return false;
-
   Block& block = computeOp.getBody().front();
   if (block.getNumArguments() != computeOp.getWeights().size())
     return false;
@@ -197,6 +262,9 @@ static bool inlineInputlessHelperComputeForWeightLikeUsers(spatial::SpatSchedule
   auto yieldOp = dyn_cast<spatial::SpatYieldOp>(block.getTerminator());
   if (!yieldOp || yieldOp.getNumOperands() != 1)
     return false;
+  auto folded = analyzeHostMaterializableHelper(computeOp);
+  if (failed(folded))
+    return false;
 
   rewriter.setInsertionPoint(computeOp);
   IRMapping mapping;
@@ -218,6 +286,20 @@ static bool inlineInputlessHelperComputeForWeightLikeUsers(spatial::SpatSchedule
       }
     }
 
+    if (isa<arith::ConstantOp>(op) || op.hasTrait<OpTrait::ConstantLike>()
+        || isPureIndexComputationOp(&op)) {
+      for (Value result : op.getResults()) {
+        auto it = folded->find(result);
+        if (it == folded->end())
+          return false;
+        mapping.map(
+          result,
+          getOrCreateConstant(constantFolder, computeOp, it->second,
+                              result.getType()));
+      }
+      continue;
+    }
+
     cloneMappedHelperOperands(&op, mapping, rewriter, constantFolder);
     Operation* clonedOp = rewriter.clone(op, mapping);
     for (auto [originalResult, newResult] : llvm::zip(op.getResults(), clonedOp->getResults()))