meno diamantini

2026-07-06 10:12:20 +02:00
parent cc9b025a35
commit 83a54e28e4
26 changed files with 3756 additions and 1278 deletions
@@ -13,7 +13,9 @@
 #include "Conversion/ONNXToSpatial/ONNXToSpatialVerifier.hpp"
 #include "src/Accelerators/PIM/Common/PimCommon.hpp"
 #include "src/Accelerators/PIM/Common/Support/DebugDump.hpp"
+#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/BiasAddUtils.hpp"
 #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp"
+#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/RowStripLayoutUtils.hpp"
 #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns.hpp"
 #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/PlanLowering.hpp"
 #include "src/Accelerators/PIM/Dialect/Spatial/Transforms/MergeComputeNodes/SpatialDataflowCsvExporter.hpp"
@@ -29,14 +31,6 @@ namespace {
 static constexpr StringLiteral kDenseLayout = "dense_nchw";
 static constexpr StringLiteral kRowStripLayout = "nchw_row_strip";

-struct RowStripPhysicalValue {
-  Value physicalValue;
-  RankedTensorType logicalType;
-  SmallVector<int64_t, 16> fragmentOffsets;
-  SmallVector<int64_t, 16> fragmentSizes;
-  std::string indexMap;
-};
-
 static FailureOr<RowStripPhysicalValue> getRowStripValue(llvm::DenseMap<Value, RowStripPhysicalValue>& rowStripValues,
                                                         Value value) {
  auto it = rowStripValues.find(value);
@@ -46,112 +40,42 @@ static FailureOr<RowStripPhysicalValue> getRowStripValue(llvm::DenseMap<Value, R
 }

 static FailureOr<RowStripPhysicalValue> buildRowStripValue(spatial::SpatBlueprintOp blueprint,
-                                                           Value physicalValue) {
+                                                           Value storage) {
  auto logicalType = dyn_cast<RankedTensorType>(blueprint.getOutput().getType());
  if (!logicalType)
    return blueprint.emitOpError("requires ranked logical output type"), failure();
  RowStripPhysicalValue value;
-  value.physicalValue = physicalValue;
+  value.storage = storage;
  value.logicalType = logicalType;
  value.fragmentOffsets.append(blueprint.getFragmentOffsets().begin(), blueprint.getFragmentOffsets().end());
  value.fragmentSizes.append(blueprint.getFragmentSizes().begin(), blueprint.getFragmentSizes().end());
-  value.indexMap = blueprint.getIndexMap().str();
+  if (blueprint.getIndexMap() != kRowStripIndexMap)
+    return blueprint.emitOpError("requires the canonical row-strip index map"), failure();
+  auto storageType = dyn_cast<RankedTensorType>(storage.getType());
+  if (!storageType || storageType != getRowStripStorageType(logicalType))
+    return blueprint.emitOpError("requires physical row-strip fragment storage"), failure();
  return value;
 }

 static FailureOr<Value>
 lowerRowStripRelu(const RowStripPhysicalValue& input, spatial::SpatReluPlanOp planOp, PatternRewriter& rewriter) {
-  auto packedType = cast<RankedTensorType>(input.physicalValue.getType());
-  auto computeOp =
-    createSpatCompute<1>(rewriter, planOp.getLoc(), TypeRange {packedType}, {}, input.physicalValue, [&](Value x) {
-      auto relu = spatial::SpatReluOp::create(rewriter, planOp.getLoc(), packedType, x);
-      spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), relu.getResult());
-    });
-  return computeOp.getResult(0);
+  return applyRowStripRelu(input.storage, input.logicalType, rewriter, planOp.getLoc());
+}
+
+static FailureOr<Value> lowerRowStripBiasAdd(const RowStripPhysicalValue& input,
+                                             spatial::SpatBiasAddPlanOp planOp,
+                                             PatternRewriter& rewriter) {
+  return applyRowStripBiasAdd(input.storage, input.logicalType, planOp.getBias(), rewriter, planOp.getLoc());
 }

 static FailureOr<Value>
 materializeRowStripToDense(const RowStripPhysicalValue& rowStripValue, Location loc, PatternRewriter& rewriter) {
-  auto packedType = dyn_cast<RankedTensorType>(rowStripValue.physicalValue.getType());
-  if (!packedType || packedType.getRank() != 3 || !packedType.hasStaticShape())
-    return failure();
  if (rowStripValue.logicalType.getRank() != 4 || !rowStripValue.logicalType.hasStaticShape())
    return failure();
-  if (rowStripValue.indexMap != "packed_hwc_rows_to_nchw")
+  auto [expectedOffsets, expectedSizes] = buildRowStripMetadata(rowStripValue.logicalType);
+  if (!llvm::equal(rowStripValue.fragmentOffsets, expectedOffsets) || !llvm::equal(rowStripValue.fragmentSizes, expectedSizes))
    return failure();
-
-  const int64_t rank = rowStripValue.logicalType.getRank();
-  const int64_t fragmentCount = rowStripValue.fragmentOffsets.size() / rank;
-  const int64_t packedWidth = packedType.getDimSize(1);
-  const int64_t packedChannels = packedType.getDimSize(2);
-  if (fragmentCount != packedType.getDimSize(0))
-    return failure();
-  for (int64_t fragmentIndex = 0; fragmentIndex < fragmentCount; ++fragmentIndex) {
-    if (rowStripValue.fragmentOffsets[fragmentIndex * rank + 0] != 0
-        || rowStripValue.fragmentOffsets[fragmentIndex * rank + 1] != 0
-        || rowStripValue.fragmentOffsets[fragmentIndex * rank + 2] != fragmentIndex
-        || rowStripValue.fragmentOffsets[fragmentIndex * rank + 3] != 0)
-      return failure();
-    if (rowStripValue.fragmentSizes[fragmentIndex * rank + 0] != 1
-        || rowStripValue.fragmentSizes[fragmentIndex * rank + 1] != packedChannels
-        || rowStripValue.fragmentSizes[fragmentIndex * rank + 2] != 1
-        || rowStripValue.fragmentSizes[fragmentIndex * rank + 3] != packedWidth)
-      return failure();
-  }
-
-  auto packedSliceType =
-    RankedTensorType::get({1, packedWidth, packedChannels}, packedType.getElementType(), packedType.getEncoding());
-  auto expandedType =
-    RankedTensorType::get({1, 1, packedWidth, packedChannels}, packedType.getElementType(), packedType.getEncoding());
-  auto logicalFragmentType =
-    RankedTensorType::get({1, packedChannels, 1, packedWidth}, packedType.getElementType(), packedType.getEncoding());
-  auto batchOp = createSpatComputeBatch(
-    rewriter,
-    loc,
-    TypeRange {rowStripValue.logicalType},
-    fragmentCount,
-    {},
-    ValueRange {rowStripValue.physicalValue},
-    [&](detail::SpatComputeBatchBodyArgs args) {
-      SmallVector<OpFoldResult> packedOffsets {args.lane, rewriter.getIndexAttr(0), rewriter.getIndexAttr(0)};
-      SmallVector<OpFoldResult> packedSizes {
-        rewriter.getIndexAttr(1), rewriter.getIndexAttr(packedWidth), rewriter.getIndexAttr(packedChannels)};
-      Value packedSlice = tensor::ExtractSliceOp::create(
-        rewriter, loc, packedSliceType, args.inputs.front(), packedOffsets, packedSizes, getUnitStrides(rewriter, 3));
-
-      Value expanded = tensor::ExpandShapeOp::create(rewriter,
-                                                     loc,
-                                                     expandedType,
-                                                     packedSlice,
-                                                     SmallVector<ReassociationIndices> {
-                                                       {0, 1},
-                                                       {2},
-                                                       {3}
-      });
-      Value transposeInit =
-        tensor::EmptyOp::create(rewriter, loc, logicalFragmentType.getShape(), logicalFragmentType.getElementType());
-      Value logicalFragment =
-        linalg::TransposeOp::create(rewriter, loc, expanded, transposeInit, SmallVector<int64_t> {0, 3, 1, 2})
-          .getResult()[0];
-
-      SmallVector<OpFoldResult> logicalOffsets {
-        rewriter.getIndexAttr(0), rewriter.getIndexAttr(0), args.lane, rewriter.getIndexAttr(0)};
-      SmallVector<OpFoldResult> logicalSizes {rewriter.getIndexAttr(1),
-                                              rewriter.getIndexAttr(packedChannels),
-                                              rewriter.getIndexAttr(1),
-                                              rewriter.getIndexAttr(packedWidth)};
-      createParallelInsertSliceIntoBatchOutput(rewriter,
-                                               loc,
-                                               logicalFragment,
-                                               args.outputs.front(),
-                                               logicalOffsets,
-                                               logicalSizes,
-                                               getUnitStrides(rewriter, 4));
-      return success();
-    });
-  if (failed(batchOp))
-    return failure();
-  return batchOp->getResult(0);
+  return materializeRowStripStorageToDense(rowStripValue.storage, rowStripValue.logicalType, rewriter, loc);
 }

 struct LowerSpatialPlansPass final : PassWrapper<LowerSpatialPlansPass, OperationPass<ModuleOp>> {
@@ -194,7 +118,7 @@ struct LowerSpatialPlansPass final : PassWrapper<LowerSpatialPlansPass, Operatio
          rewriter.setInsertionPoint(planOp);
          FailureOr<Value> lowered = lowerSelectedConv2DPlan(
            planOp,
-            succeeded(rowStripInput) ? std::optional<Value> {rowStripInput->physicalValue} : std::nullopt,
+            succeeded(rowStripInput) ? std::optional<Value> {rowStripInput->storage} : std::nullopt,
            /*emitRowStripLayout=*/true,
            rewriter);
          if (failed(lowered)) {
@@ -266,6 +190,64 @@ struct LowerSpatialPlansPass final : PassWrapper<LowerSpatialPlansPass, Operatio
        rewriter.replaceOp(planOp, computeOp.getResults());
        continue;
      }
+      if (auto planOp = dyn_cast<spatial::SpatBiasAddPlanOp>(&op)) {
+        if (succeeded(getRowStripValue(rowStripValues, planOp.getInput()))) {
+          auto outputBlueprint = llvm::find_if(planOp.getResult().getUsers(), [](Operation* user) {
+            auto blueprint = dyn_cast<spatial::SpatBlueprintOp>(user);
+            return blueprint && blueprint.getPhysicalLayout() == kRowStripLayout;
+          });
+          if (outputBlueprint == planOp.getResult().getUsers().end()) {
+            planOp.emitOpError("row-strip bias_add plan requires a row-strip blueprint result");
+            signalPassFailure();
+            return;
+          }
+
+          FailureOr<RowStripPhysicalValue> input = getRowStripValue(rowStripValues, planOp.getInput());
+          rewriter.setInsertionPoint(planOp);
+          FailureOr<Value> lowered = lowerRowStripBiasAdd(*input, planOp, rewriter);
+          if (failed(lowered)) {
+            planOp.emitOpError("failed to lower selected row-strip Spatial bias_add plan");
+            signalPassFailure();
+            return;
+          }
+          auto blueprint = cast<spatial::SpatBlueprintOp>(*outputBlueprint);
+          FailureOr<RowStripPhysicalValue> output = buildRowStripValue(blueprint, *lowered);
+          if (failed(output)) {
+            signalPassFailure();
+            return;
+          }
+          rowStripValues[blueprint.getResult()] = *output;
+          eraseAfterLowering.insert(planOp);
+          eraseAfterLowering.insert(blueprint);
+          continue;
+        }
+
+        auto resultType = dyn_cast<RankedTensorType>(planOp.getOutput().getType());
+        if (!resultType) {
+          planOp.emitOpError("requires ranked output type");
+          signalPassFailure();
+          return;
+        }
+        rewriter.setInsertionPoint(planOp);
+        FailureOr<Value> denseBias = materializeDenseBiasAddTensor(planOp.getBias(), resultType, rewriter, planOp.getLoc());
+        if (failed(denseBias)) {
+          planOp.emitOpError("failed to materialize dense Conv-style bias");
+          signalPassFailure();
+          return;
+        }
+        auto computeOp = createSpatCompute<2>(rewriter,
+                                              planOp.getLoc(),
+                                              planOp.getOutput().getType(),
+                                              {},
+                                              ValueRange {planOp.getInput(), *denseBias},
+                                              [&](Value x, Value y) {
+                                                auto added = spatial::SpatVAddOp::create(
+                                                  rewriter, planOp.getLoc(), planOp.getOutput().getType(), x, y);
+                                                spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), added.getResult());
+                                              });
+        rewriter.replaceOp(planOp, computeOp.getResults());
+        continue;
+      }
      if (auto materializeOp = dyn_cast<spatial::SpatMaterializeLayoutOp>(&op)) {
        if (materializeOp.getSourcePhysicalLayout() == kDenseLayout
            && materializeOp.getTargetPhysicalLayout() == kDenseLayout) {
@@ -385,6 +367,7 @@ struct LowerSpatialPlansPass final : PassWrapper<LowerSpatialPlansPass, Operatio
      if (isa<ONNXEntryPointOp>(op))
        return;
      if (isa<spatial::SpatConv2DPlanOp,
+              spatial::SpatBiasAddPlanOp,
              spatial::SpatReluPlanOp,
              spatial::SpatBlueprintOp,
              spatial::SpatMaterializeLayoutOp>(op)