#include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Interfaces/DestinationStyleOpInterface.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "src/Accelerators/PIM/Common/PimCommon.hpp" #include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp" #include "src/Accelerators/PIM/Dialect/Pim/Analysis/LocalMemoryLifetimeAnalysis.hpp" #include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp" using namespace mlir; namespace onnx_mlir::pim { namespace { struct OperationOrdering { DenseMap position; DenseMap subtreeEnd; uint64_t next = 0; }; static void orderOperation(Operation* op, OperationOrdering& ordering) { uint64_t end = ordering.next; ordering.position[op] = ordering.next++; for (Region& region : op->getRegions()) for (Block& block : region) for (Operation& nested : block) { orderOperation(&nested, ordering); end = std::max(end, ordering.subtreeEnd.lookup(&nested)); } ordering.subtreeEnd[op] = end; } static OperationOrdering buildOrdering(Operation* coreLikeOp) { OperationOrdering ordering; if (!coreLikeOp || coreLikeOp->getNumRegions() != 1 || coreLikeOp->getRegion(0).empty()) return ordering; for (Operation& op : coreLikeOp->getRegion(0).front()) orderOperation(&op, ordering); return ordering; } static bool isRuntimeMemoryTouch(Operation* op) { return isa(op); } static bool isIgnoredUser(Operation* op) { return isLocalMemoryAliasOp(op) || isa(op) || isCoreStaticAddressOp(op); } static bool isWithin(Value value, Region* region) { if (auto argument = dyn_cast(value)) return argument.getOwner()->getParent() == region; if (Operation* definingOp = value.getDefiningOp()) return definingOp->getParentRegion() == region || region->isAncestor(definingOp->getParentRegion()); return false; } static std::pair getTouchRange(Value definingValue, Operation* user, const OperationOrdering& ordering) { uint64_t start = ordering.position.lookup(user); uint64_t end = start; for (Operation* current = user; current; current = current->getParentOp()) if (auto forOp = dyn_cast(current); forOp && !isWithin(definingValue, &forOp.getRegion())) { start = std::min(start, ordering.position.lookup(forOp)); end = std::max(end, ordering.subtreeEnd.lookup(forOp)); } return {start, end}; } static FailureOr getAllocationSize(memref::AllocOp allocation) { auto type = dyn_cast(allocation.getType()); if (!type) return failure(); auto bytes = getCheckedShapedTypeSizeInBytes(type, allocation, "memory allocation byte size"); if (failed(bytes)) return failure(); return checkedSize(*bytes, allocation, "memory allocation byte size"); } } // namespace bool isLocalMemoryAliasOp(Operation* op) { return isa(op); } LogicalResult walkLocalMemoryUses(Value root, LocalMemoryUseCallback callback) { llvm::SmallPtrSet visitedValues; llvm::SmallPtrSet visitedUses; SmallVector pending = {root}; while (!pending.empty()) { Value value = pending.pop_back_val(); if (!visitedValues.insert(value).second) continue; for (OpOperand& use : value.getUses()) { if (!visitedUses.insert(&use).second || failed(callback(use))) return failure(); Operation* user = use.getOwner(); if (isLocalMemoryAliasOp(user) && use.getOperandNumber() == 0) { if (user->getNumResults() != 1) return failure(); pending.push_back(user->getResult(0)); } if (auto dpsOp = dyn_cast(user); dpsOp && dpsOp.isDpsInit(&use)) pending.push_back(dpsOp.getTiedOpResult(&use)); if (auto forOp = dyn_cast(user)) for (auto [index, init] : llvm::enumerate(forOp.getInitArgsMutable())) if (&init == &use) { pending.push_back(forOp.getRegionIterArgs()[index]); pending.push_back(forOp.getResult(index)); } if (auto yieldOp = dyn_cast(user)) { unsigned index = use.getOperandNumber(); Operation* parent = yieldOp->getParentOp(); if (auto forOp = dyn_cast(parent)) pending.push_back(forOp.getResult(index)); else if (auto ifOp = dyn_cast(parent)) pending.push_back(ifOp.getResult(index)); else if (auto switchOp = dyn_cast(parent)) pending.push_back(switchOp.getResult(index)); else return failure(); } } } return success(); } FailureOr> analyzeLocalMemoryLifetimes(Operation* coreLikeOp) { SmallVector intervals; OperationOrdering ordering = buildOrdering(coreLikeOp); if (ordering.position.empty()) return intervals; uint64_t fallbackEnd = ordering.next - 1; bool failedSize = false; coreLikeOp->walk([&](memref::AllocOp allocation) { auto size = getAllocationSize(allocation); if (failed(size)) { failedSize = true; return; } LocalMemoryInterval interval {allocation, ordering.position.lookup(allocation), ordering.position.lookup(allocation), *size}; bool hasRuntimeTouch = false; bool needsFallback = false; if (failed(walkLocalMemoryUses(allocation.getResult(), [&](OpOperand& use) { Operation* user = use.getOwner(); if (isRuntimeMemoryTouch(user)) { auto [start, end] = getTouchRange(allocation.getResult(), user, ordering); if (!hasRuntimeTouch) { interval.start = start; interval.end = end; hasRuntimeTouch = true; } else { interval.start = std::min(interval.start, start); interval.end = std::max(interval.end, end); } } else if (!isIgnoredUser(user)) { needsFallback = true; } return success(); }))) needsFallback = true; if (!hasRuntimeTouch || needsFallback) { interval.start = std::min(interval.start, ordering.position.lookup(allocation)); interval.end = fallbackEnd; } intervals.push_back(interval); }); if (failedSize) return failure(); return intervals; } bool localMemoryLifetimesOverlap(const LocalMemoryInterval& lhs, const LocalMemoryInterval& rhs) { return !(lhs.end < rhs.start || rhs.end < lhs.start); } } // namespace onnx_mlir::pim