robba

src/PIM/Conversion/SpatialToPim/Common.cpp

@@ -1,12 +1,17 @@
 #include "mlir/IR/ValueRange.h"
 
+#include "mlir/Dialect/Arith/IR/Arith.h"
+
 #include "llvm/ADT/STLExtras.h"
 
 #include <cassert>
+#include <limits>
 
 #include "Common.hpp"
+#include "src/Accelerators/PIM/Common/IR/LoopUtils.hpp"
 #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
 #include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp"
+#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp"
 
 using namespace llvm;
 using namespace mlir;
@@ -186,4 +191,375 @@ forEachContiguousDestinationChunk(ArrayRef<int64_t> destShape,
   return visit(visit, 0);
 }
 
+static mlir::Value
+createSteppedOffset(OpBuilder& builder, Location loc, mlir::Value start, mlir::Value index, int64_t stepBytes) {
+  if (stepBytes == 0)
+    return start;
+  mlir::Value step = arith::ConstantIndexOp::create(builder, loc, stepBytes);
+  mlir::Value scaled = arith::MulIOp::create(builder, loc, index, step).getResult();
+  return arith::AddIOp::create(builder, loc, start, scaled).getResult();
+}
+
+static mlir::Value createIndexedOffset(OpBuilder& builder,
+                                       Location loc,
+                                       mlir::Value indexArg,
+                                       ArrayRef<int64_t> values) {
+  assert(!values.empty() && "expected lane-indexed values");
+  if (llvm::all_of(values.drop_front(), [&](int64_t value) { return value == values.front(); }))
+    return arith::ConstantIndexOp::create(builder, loc, values.front());
+
+  if (values.size() >= 2) {
+    int64_t step = values[1] - values[0];
+    bool arithmetic = llvm::all_of(llvm::seq<size_t>(2, values.size()), [&](size_t index) {
+      return values[index] == values.front() + static_cast<int64_t>(index) * step;
+    });
+    if (arithmetic) {
+      mlir::Value base = arith::ConstantIndexOp::create(builder, loc, values.front());
+      mlir::Value stepValue = arith::ConstantIndexOp::create(builder, loc, step);
+      mlir::Value scaledIndex = arith::MulIOp::create(builder, loc, indexArg, stepValue).getResult();
+      return arith::AddIOp::create(builder, loc, base, scaledIndex).getResult();
+    }
+  }
+
+  mlir::Value selected = arith::ConstantIndexOp::create(builder, loc, values.front());
+  for (auto [lane, value] : llvm::enumerate(values.drop_front())) {
+    mlir::Value indexValue = arith::ConstantIndexOp::create(builder, loc, static_cast<int64_t>(lane + 1));
+    mlir::Value cmp = arith::CmpIOp::create(builder, loc, arith::CmpIPredicate::eq, indexArg, indexValue);
+    mlir::Value candidate = arith::ConstantIndexOp::create(builder, loc, value);
+    selected = arith::SelectOp::create(builder, loc, cmp, candidate, selected);
+  }
+  return selected;
+}
+
+struct FragmentAssemblyCopyRunFamily {
+  FragmentAssemblyCopyRun prototype;
+  SmallVector<int64_t, 8> sourceRunStartDeltas;
+  SmallVector<int64_t, 8> hostRunStartDeltas;
+};
+
+static bool computeUniformRunStartDelta(ArrayRef<int64_t> prototypeStarts,
+                                        ArrayRef<int64_t> runStarts,
+                                        int64_t& delta) {
+  if (prototypeStarts.size() != runStarts.size() || prototypeStarts.empty())
+    return false;
+
+  delta = runStarts.front() - prototypeStarts.front();
+  return llvm::all_of(llvm::zip_equal(prototypeStarts, runStarts), [&](auto pair) {
+    auto [prototypeStart, runStart] = pair;
+    return runStart - prototypeStart == delta;
+  });
+}
+
+static bool canMergeFragmentAssemblyCopyRunIntoFamily(const FragmentAssemblyCopyRunFamily& family,
+                                                      const FragmentAssemblyCopyRun& run,
+                                                      int64_t& sourceRunStartDelta,
+                                                      int64_t& hostRunStartDelta) {
+  const FragmentAssemblyCopyRun& prototype = family.prototype;
+  if (prototype.source != run.source || prototype.sourceType != run.sourceType
+      || prototype.hostTargetIndex != run.hostTargetIndex || prototype.count != run.count
+      || prototype.sourceStepBytes != run.sourceStepBytes || prototype.hostStepBytes != run.hostStepBytes
+      || prototype.byteSize != run.byteSize)
+    return false;
+
+  if (!computeUniformRunStartDelta(prototype.sourceStartBytesByLane, run.sourceStartBytesByLane, sourceRunStartDelta))
+    return false;
+  return computeUniformRunStartDelta(prototype.hostStartBytesByLane, run.hostStartBytesByLane, hostRunStartDelta);
+}
+
+static SmallVector<FragmentAssemblyCopyRunFamily, 8>
+groupFragmentAssemblyCopyRunFamilies(ArrayRef<FragmentAssemblyCopyRun> runs) {
+  auto compareRunStarts = [](ArrayRef<int64_t> lhs, ArrayRef<int64_t> rhs) {
+    return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), rhs.end());
+  };
+
+  SmallVector<FragmentAssemblyCopyRun, 8> sortedRuns(runs.begin(), runs.end());
+  llvm::sort(sortedRuns, [&](const FragmentAssemblyCopyRun& lhs, const FragmentAssemblyCopyRun& rhs) {
+    if (lhs.hostTargetIndex != rhs.hostTargetIndex)
+      return lhs.hostTargetIndex < rhs.hostTargetIndex;
+    if (lhs.source != rhs.source)
+      return lhs.source.getAsOpaquePointer() < rhs.source.getAsOpaquePointer();
+    if (lhs.byteSize != rhs.byteSize)
+      return lhs.byteSize < rhs.byteSize;
+    if (lhs.count != rhs.count)
+      return lhs.count < rhs.count;
+    if (lhs.sourceStepBytes != rhs.sourceStepBytes)
+      return lhs.sourceStepBytes < rhs.sourceStepBytes;
+    if (lhs.hostStepBytes != rhs.hostStepBytes)
+      return lhs.hostStepBytes < rhs.hostStepBytes;
+    if (compareRunStarts(lhs.sourceStartBytesByLane, rhs.sourceStartBytesByLane))
+      return true;
+    if (compareRunStarts(rhs.sourceStartBytesByLane, lhs.sourceStartBytesByLane))
+      return false;
+    return compareRunStarts(lhs.hostStartBytesByLane, rhs.hostStartBytesByLane);
+  });
+
+  SmallVector<FragmentAssemblyCopyRunFamily, 8> families;
+  for (const FragmentAssemblyCopyRun& run : sortedRuns) {
+    int64_t sourceRunStartDelta = 0;
+    int64_t hostRunStartDelta = 0;
+    if (!families.empty()
+        && canMergeFragmentAssemblyCopyRunIntoFamily(
+          families.back(), run, sourceRunStartDelta, hostRunStartDelta)) {
+      families.back().sourceRunStartDeltas.push_back(sourceRunStartDelta);
+      families.back().hostRunStartDeltas.push_back(hostRunStartDelta);
+      continue;
+    }
+
+    FragmentAssemblyCopyRunFamily family;
+    family.prototype = run;
+    family.sourceRunStartDeltas.push_back(0);
+    family.hostRunStartDeltas.push_back(0);
+    families.push_back(std::move(family));
+  }
+
+  return families;
+}
+
+FailureOr<SmallVector<FragmentAssemblyCopyRun, 8>>
+groupFragmentAssemblyCopyRuns(ArrayRef<FragmentAssemblyCopy> copies, uint32_t laneCount) {
+  if (laneCount == 0)
+    return failure();
+
+  struct LaneLocalCopyRun {
+    FragmentAssemblyCopyRun run;
+    int64_t lane = 0;
+  };
+
+  SmallVector<FragmentAssemblyCopy, 8> sortedCopies(copies.begin(), copies.end());
+  llvm::sort(sortedCopies, [](const FragmentAssemblyCopy& lhs, const FragmentAssemblyCopy& rhs) {
+    if (lhs.hostTargetIndex != rhs.hostTargetIndex)
+      return lhs.hostTargetIndex < rhs.hostTargetIndex;
+    if (lhs.source != rhs.source)
+      return lhs.source.getAsOpaquePointer() < rhs.source.getAsOpaquePointer();
+    if (lhs.lane != rhs.lane)
+      return lhs.lane < rhs.lane;
+    if (lhs.byteSize != rhs.byteSize)
+      return lhs.byteSize < rhs.byteSize;
+    if (lhs.sourceByteOffset != rhs.sourceByteOffset)
+      return lhs.sourceByteOffset < rhs.sourceByteOffset;
+    return lhs.hostByteOffset < rhs.hostByteOffset;
+  });
+
+  SmallVector<LaneLocalCopyRun, 8> laneRuns;
+  for (const FragmentAssemblyCopy& copy : sortedCopies) {
+    if (copy.lane < 0 || copy.lane >= static_cast<int64_t>(laneCount))
+      return failure();
+
+    if (!laneRuns.empty()) {
+      LaneLocalCopyRun& laneRun = laneRuns.back();
+      FragmentAssemblyCopyRun& run = laneRun.run;
+      if (run.source == copy.source && run.sourceType == copy.sourceType
+          && run.hostTargetIndex == copy.hostTargetIndex && laneRun.lane == copy.lane && run.byteSize == copy.byteSize
+          && run.sourceStartBytesByLane.size() == 1 && run.hostStartBytesByLane.size() == 1) {
+        int64_t previousSourceOffset = run.sourceStartBytesByLane.front() + (run.count - 1) * run.sourceStepBytes;
+        int64_t previousHostOffset = run.hostStartBytesByLane.front() + (run.count - 1) * run.hostStepBytes;
+        int64_t sourceDelta = copy.sourceByteOffset - previousSourceOffset;
+        int64_t hostDelta = copy.hostByteOffset - previousHostOffset;
+        if (run.count == 1) {
+          run.sourceStepBytes = sourceDelta;
+          run.hostStepBytes = hostDelta;
+          ++run.count;
+          continue;
+        }
+        if (run.sourceStepBytes == sourceDelta && run.hostStepBytes == hostDelta) {
+          ++run.count;
+          continue;
+        }
+      }
+    }
+
+    LaneLocalCopyRun laneRun;
+    laneRun.run.source = copy.source;
+    laneRun.run.sourceType = copy.sourceType;
+    laneRun.run.hostTargetIndex = copy.hostTargetIndex;
+    laneRun.run.count = 1;
+    laneRun.run.byteSize = copy.byteSize;
+    laneRun.run.sourceStartBytesByLane.push_back(copy.sourceByteOffset);
+    laneRun.run.hostStartBytesByLane.push_back(copy.hostByteOffset);
+    laneRun.lane = copy.lane;
+    laneRuns.push_back(std::move(laneRun));
+  }
+
+  SmallVector<FragmentAssemblyCopyRun, 8> mergedRuns;
+  for (const LaneLocalCopyRun& laneRun : laneRuns) {
+    size_t laneIndex = static_cast<size_t>(laneRun.lane);
+    auto mergedIt = llvm::find_if(mergedRuns, [&](const FragmentAssemblyCopyRun& run) {
+      return run.source == laneRun.run.source && run.sourceType == laneRun.run.sourceType
+             && run.hostTargetIndex == laneRun.run.hostTargetIndex && run.count == laneRun.run.count
+             && run.byteSize == laneRun.run.byteSize && run.sourceStepBytes == laneRun.run.sourceStepBytes
+             && run.hostStepBytes == laneRun.run.hostStepBytes && laneIndex < run.sourceStartBytesByLane.size()
+             && run.sourceStartBytesByLane[laneIndex] == std::numeric_limits<int64_t>::min();
+    });
+
+    if (mergedIt == mergedRuns.end()) {
+      FragmentAssemblyCopyRun merged = laneRun.run;
+      merged.sourceStartBytesByLane.assign(laneCount, std::numeric_limits<int64_t>::min());
+      merged.hostStartBytesByLane.assign(laneCount, std::numeric_limits<int64_t>::min());
+      merged.sourceStartBytesByLane[laneIndex] = laneRun.run.sourceStartBytesByLane.front();
+      merged.hostStartBytesByLane[laneIndex] = laneRun.run.hostStartBytesByLane.front();
+      mergedRuns.push_back(std::move(merged));
+      continue;
+    }
+
+    mergedIt->sourceStartBytesByLane[laneIndex] = laneRun.run.sourceStartBytesByLane.front();
+    mergedIt->hostStartBytesByLane[laneIndex] = laneRun.run.hostStartBytesByLane.front();
+  }
+
+  for (const FragmentAssemblyCopyRun& run : mergedRuns) {
+    if (llvm::any_of(run.sourceStartBytesByLane,
+                     [](int64_t value) { return value == std::numeric_limits<int64_t>::min(); }))
+      return failure();
+    if (llvm::any_of(run.hostStartBytesByLane,
+                     [](int64_t value) { return value == std::numeric_limits<int64_t>::min(); }))
+      return failure();
+  }
+
+  return mergedRuns;
+}
+
+static FailureOr<mlir::Value> emitFragmentAssemblyCopyRun(OpBuilder& builder,
+                                                          Location loc,
+                                                          const FragmentAssemblyCopyRun& run,
+                                                          mlir::Value hostTarget,
+                                                          Operation* anchor,
+                                                          std::optional<mlir::Value> laneArg,
+                                                          mlir::Value baseHostOffset,
+                                                          mlir::Value sourceRunStartDelta = {},
+                                                          mlir::Value hostRunStartDelta = {}) {
+  auto sizeAttr = pim::getCheckedI32Attr(builder, anchor, run.byteSize, "fragment assembly host copy byte size");
+  if (failed(sizeAttr))
+    return failure();
+
+  mlir::Value hostStart;
+  mlir::Value sourceStart;
+  if (laneArg) {
+    hostStart = createIndexedOffset(builder, loc, *laneArg, run.hostStartBytesByLane);
+    sourceStart = createIndexedOffset(builder, loc, *laneArg, run.sourceStartBytesByLane);
+  } else {
+    hostStart = arith::ConstantIndexOp::create(builder, loc, run.hostStartBytesByLane.front());
+    sourceStart = arith::ConstantIndexOp::create(builder, loc, run.sourceStartBytesByLane.front());
+  }
+
+  if (hostRunStartDelta)
+    hostStart = arith::AddIOp::create(builder, loc, hostStart, hostRunStartDelta).getResult();
+  if (sourceRunStartDelta)
+    sourceStart = arith::AddIOp::create(builder, loc, sourceStart, sourceRunStartDelta).getResult();
+  if (baseHostOffset)
+    hostStart = arith::AddIOp::create(builder, loc, baseHostOffset, hostStart).getResult();
+
+  if (run.count == 1) {
+    return pim::PimMemCopyDevToHostOp::create(builder,
+                                              loc,
+                                              hostTarget.getType(),
+                                              hostStart,
+                                              sourceStart,
+                                              hostTarget,
+                                              run.source,
+                                              *sizeAttr)
+      .getOutput();
+  }
+
+  mlir::Value lowerBound = arith::ConstantIndexOp::create(builder, loc, 0);
+  mlir::Value upperBound = arith::ConstantIndexOp::create(builder, loc, run.count);
+  mlir::Value step = arith::ConstantIndexOp::create(builder, loc, 1);
+  FailureOr<NormalizedLoopResult> loop = buildNormalizedScfFor(
+    builder,
+    loc,
+    lowerBound,
+    upperBound,
+    step,
+    ValueRange {hostTarget},
+    [&](OpBuilder& loopBuilder,
+        Location bodyLoc,
+        mlir::Value flatIndex,
+        ValueRange iterArgs,
+        SmallVectorImpl<mlir::Value>& yielded) {
+      mlir::Value hostOffset = createSteppedOffset(loopBuilder, bodyLoc, hostStart, flatIndex, run.hostStepBytes);
+      mlir::Value sourceOffset =
+        createSteppedOffset(loopBuilder, bodyLoc, sourceStart, flatIndex, run.sourceStepBytes);
+      mlir::Value copied =
+        pim::PimMemCopyDevToHostOp::create(loopBuilder,
+                                           bodyLoc,
+                                           iterArgs.front().getType(),
+                                           hostOffset,
+                                           sourceOffset,
+                                           iterArgs.front(),
+                                           run.source,
+                                           *sizeAttr)
+          .getOutput();
+      yielded.push_back(copied);
+      return success();
+    });
+  if (failed(loop))
+    return failure();
+  return loop->results.front();
+}
+
+static FailureOr<mlir::Value> emitFragmentAssemblyCopyRunFamily(OpBuilder& builder,
+                                                                Location loc,
+                                                                const FragmentAssemblyCopyRunFamily& family,
+                                                                mlir::Value hostTarget,
+                                                                Operation* anchor,
+                                                                std::optional<mlir::Value> laneArg,
+                                                                mlir::Value baseHostOffset) {
+  if (family.sourceRunStartDeltas.size() == 1)
+    return emitFragmentAssemblyCopyRun(
+      builder, loc, family.prototype, hostTarget, anchor, laneArg, baseHostOffset);
+
+  mlir::Value lowerBound = arith::ConstantIndexOp::create(builder, loc, 0);
+  mlir::Value upperBound = arith::ConstantIndexOp::create(builder, loc, family.sourceRunStartDeltas.size());
+  mlir::Value step = arith::ConstantIndexOp::create(builder, loc, 1);
+  FailureOr<NormalizedLoopResult> outerLoop = buildNormalizedScfFor(
+    builder,
+    loc,
+    lowerBound,
+    upperBound,
+    step,
+    ValueRange {hostTarget},
+    [&](OpBuilder& loopBuilder,
+        Location bodyLoc,
+        mlir::Value runIndex,
+        ValueRange iterArgs,
+        SmallVectorImpl<mlir::Value>& yielded) {
+      mlir::Value sourceRunStartDelta =
+        createIndexedOffset(loopBuilder, bodyLoc, runIndex, family.sourceRunStartDeltas);
+      mlir::Value hostRunStartDelta =
+        createIndexedOffset(loopBuilder, bodyLoc, runIndex, family.hostRunStartDeltas);
+      FailureOr<mlir::Value> copied = emitFragmentAssemblyCopyRun(loopBuilder,
+                                                                  bodyLoc,
+                                                                  family.prototype,
+                                                                  iterArgs.front(),
+                                                                  anchor,
+                                                                  laneArg,
+                                                                  baseHostOffset,
+                                                                  sourceRunStartDelta,
+                                                                  hostRunStartDelta);
+      if (failed(copied))
+        return failure();
+      yielded.push_back(*copied);
+      return success();
+    });
+  if (failed(outerLoop))
+    return failure();
+  return outerLoop->results.front();
+}
+
+FailureOr<mlir::Value> emitFragmentAssemblyCopyRuns(IRRewriter& rewriter,
+                                                    Location loc,
+                                                    ArrayRef<FragmentAssemblyCopyRun> runs,
+                                                    mlir::Value hostTarget,
+                                                    Operation* anchor,
+                                                    std::optional<mlir::Value> laneArg,
+                                                    mlir::Value baseHostOffset) {
+  for (const FragmentAssemblyCopyRunFamily& family : groupFragmentAssemblyCopyRunFamilies(runs)) {
+    FailureOr<mlir::Value> updatedHostTarget =
+      emitFragmentAssemblyCopyRunFamily(rewriter, loc, family, hostTarget, anchor, laneArg, baseHostOffset);
+    if (failed(updatedHostTarget))
+      return failure();
+    hostTarget = *updatedHostTarget;
+  }
+
+  return hostTarget;
+}
+
 } // namespace onnx_mlir