big refactor
Validate Operations / validate-operations (push) Has been cancelled

This commit is contained in:
NiccoloN
2026-08-04 11:28:05 +02:00
parent f4a3b012cc
commit 10b6ee6c32
150 changed files with 6737 additions and 4816 deletions
@@ -0,0 +1,136 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/DialectConversion.h"
#include "Conversion/ONNXToSpatial/Passes/Analyses/ONNXToSpatialVerifier.hpp"
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
#include "src/Accelerators/PIM/Common/Support/DebugDump.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Passes/Transforms/SpatialPlanLoweringPatterns.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/SpatialDataflowCsvExporter.hpp"
#include "src/Accelerators/PIM/Passes/PIMPasses.h"
#include "src/Dialect/ONNX/ONNXOps.hpp"
using namespace mlir;
namespace onnx_mlir {
namespace {
struct LowerSpatialPlansPass final
: PassWrapper<LowerSpatialPlansPass, OperationPass<ModuleOp>> {
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerSpatialPlansPass)
StringRef getArgument() const override { return "lower-spatial-plans"; }
StringRef getDescription() const override {
return "Lower selected Spatial planning ops to low-level Spatial IR.";
}
LowerSpatialPlansPass() = default;
LowerSpatialPlansPass(const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options,
spatial::SpatialDataflowExportStage exportStage)
: target(target), planningOptions(options), exportStage(exportStage), hasTarget(true) {}
void runOnOperation() override {
ModuleOp moduleOp = getOperation();
if (!hasTarget) {
moduleOp.emitError("Spatial plan lowering requires an injected SpatialTargetResources");
signalPassFailure();
return;
}
auto entryFunc = getPimEntryFunc(moduleOp);
if (failed(entryFunc)) {
moduleOp.emitError("failed to locate the PIM entry function during LowerSpatialPlans");
signalPassFailure();
return;
}
func::FuncOp funcOp = *entryFunc;
auto verifyLogicalPhase = [&](StringRef stage) -> bool {
if (succeeded(verifyLogicalSpatialGraphInvariants(funcOp)))
return true;
moduleOp.emitError() << "logical Spatial graph verification failed " << stage;
signalPassFailure();
return false;
};
if (!verifyLogicalPhase("at the start of LowerSpatialPlans"))
return;
if (failed(verifySelectedSpatialLayouts(funcOp, target))) {
moduleOp.emitError("selected Spatial layout verification failed");
signalPassFailure();
return;
}
MLIRContext* ctx = moduleOp.getContext();
RewritePatternSet patterns(ctx);
populateSpatialPlanLoweringPatterns(patterns, ctx, target, planningOptions);
ConversionTarget conversionTarget(*ctx);
conversionTarget.addLegalDialect<spatial::SpatialDialect,
tensor::TensorDialect,
linalg::LinalgDialect,
affine::AffineDialect,
arith::ArithDialect,
scf::SCFDialect,
func::FuncDialect>();
conversionTarget.addIllegalDialect<ONNXDialect>();
conversionTarget.addLegalOp<ONNXEntryPointOp>();
conversionTarget.addIllegalOp<spatial::SpatConv2DPlanOp,
spatial::SpatFlattenPlanOp,
spatial::SpatReluPlanOp,
spatial::SpatSiluPlanOp,
spatial::SpatResizeNearestPlanOp,
spatial::SpatMaxPool2DPlanOp,
spatial::SpatGlobalAveragePoolPlanOp,
spatial::SpatBiasAddPlanOp,
spatial::SpatAddPlanOp,
spatial::SpatConcatPlanOp,
spatial::SpatMaterializeLayoutOp>();
conversionTarget.addDynamicallyLegalOp<spatial::SpatBlueprintOp>(
[](spatial::SpatBlueprintOp blueprint) {
return spatial::isFragmentAssembly(blueprint.getMode());
});
if (failed(applyFullConversion(funcOp, conversionTarget,
std::move(patterns)))) {
moduleOp.emitError("failed to lower Spatial plans and layout materialization");
signalPassFailure();
return;
}
dumpModule(moduleOp, "spatial1_graph");
if (spatial::shouldExportSpatialDataflowStage(
exportStage, spatial::SpatialDataflowExportStage::Spatial1)
&& failed(spatial::exportSpatialDataflowCsvGraph(funcOp, "spatial1_graph"))) {
signalPassFailure();
return;
}
verifyLogicalPhase("at the end of LowerSpatialPlans");
}
spatial::SpatialTargetResources target;
ONNXToSpatialPlanningOptions planningOptions;
spatial::SpatialDataflowExportStage exportStage = spatial::SpatialDataflowExportStage::None;
bool hasTarget = false;
};
} // namespace
std::unique_ptr<Pass> createLowerSpatialPlansPass() {
return std::make_unique<LowerSpatialPlansPass>();
}
std::unique_ptr<Pass> createLowerSpatialPlansPass(
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options,
spatial::SpatialDataflowExportStage exportStage) {
return std::make_unique<LowerSpatialPlansPass>(target, options, exportStage);
}
} // namespace onnx_mlir
@@ -0,0 +1,281 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/IR/IRMapping.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Transforms/Passes.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "llvm/ADT/SmallVector.h"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp"
#include "Common/PimCommon.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/ComputeRegionBuilder.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/CompileTime.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Passes/Analyses/ONNXToSpatialVerifier.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
#include "src/Dialect/ONNX/ONNXOps.hpp"
using namespace mlir;
namespace onnx_mlir {
namespace {
struct ONNXToSpatialPass : PassWrapper<ONNXToSpatialPass, OperationPass<ModuleOp>> {
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ONNXToSpatialPass)
StringRef getArgument() const override { return "convert-onnx-to-spatial"; }
StringRef getDescription() const override { return "Lower ONNX ops to Spatial ops."; }
ONNXToSpatialPass() = default;
ONNXToSpatialPass(const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options)
: target(target), planningOptions(options), hasTarget(true) {}
ONNXToSpatialPass(const ONNXToSpatialPass& pass)
: target(pass.target), planningOptions(pass.planningOptions), hasTarget(pass.hasTarget) {}
void runOnOperation() override;
spatial::SpatialTargetResources target;
ONNXToSpatialPlanningOptions planningOptions;
bool hasTarget = false;
};
} // namespace
static void populateEmptyFunction(func::FuncOp funcOp) {
IRRewriter rewriter(funcOp.getContext());
IRMapping mapper;
SmallVector<spatial::SpatGraphCompute> computes(funcOp.getOps<spatial::SpatGraphCompute>());
SmallVector<spatial::SpatGraphComputeBatch> computeBatches(funcOp.getOps<spatial::SpatGraphComputeBatch>());
SmallVector<spatial::SpatConv2DPlanOp> convPlans(funcOp.getOps<spatial::SpatConv2DPlanOp>());
SmallVector<spatial::SpatBiasAddPlanOp> biasAddPlans(funcOp.getOps<spatial::SpatBiasAddPlanOp>());
SmallVector<spatial::SpatAddPlanOp> addPlans(funcOp.getOps<spatial::SpatAddPlanOp>());
SmallVector<spatial::SpatConcatPlanOp> concatPlans(funcOp.getOps<spatial::SpatConcatPlanOp>());
SmallVector<spatial::SpatFlattenPlanOp> flattenPlans(funcOp.getOps<spatial::SpatFlattenPlanOp>());
SmallVector<spatial::SpatReluPlanOp> reluPlans(funcOp.getOps<spatial::SpatReluPlanOp>());
SmallVector<spatial::SpatSiluPlanOp> siluPlans(funcOp.getOps<spatial::SpatSiluPlanOp>());
SmallVector<spatial::SpatResizeNearestPlanOp> resizePlans(
funcOp.getOps<spatial::SpatResizeNearestPlanOp>());
SmallVector<spatial::SpatMaxPool2DPlanOp> maxPoolPlans(funcOp.getOps<spatial::SpatMaxPool2DPlanOp>());
SmallVector<spatial::SpatGlobalAveragePoolPlanOp> globalAveragePoolPlans(
funcOp.getOps<spatial::SpatGlobalAveragePoolPlanOp>());
SmallVector<spatial::SpatBlueprintOp> blueprints(funcOp.getOps<spatial::SpatBlueprintOp>());
SmallVector<spatial::SpatMaterializeLayoutOp> materializers(funcOp.getOps<spatial::SpatMaterializeLayoutOp>());
if (!computes.empty() || !computeBatches.empty() || !convPlans.empty() || !biasAddPlans.empty() || !addPlans.empty()
|| !concatPlans.empty() || !flattenPlans.empty() || !reluPlans.empty() || !siluPlans.empty() || !resizePlans.empty()
|| !maxPoolPlans.empty() || !blueprints.empty()
|| !globalAveragePoolPlans.empty() || !materializers.empty()) {
return;
}
auto returnOp = cast<func::ReturnOp>(funcOp.getFunctionBody().front().getTerminator());
rewriter.setInsertionPoint(returnOp);
SmallVector<Type> sourceTypes;
SmallVector<Location> sourceLocs;
sourceTypes.reserve(funcOp.getNumArguments());
sourceLocs.reserve(funcOp.getNumArguments());
for (Value source : funcOp.getArguments()) {
sourceTypes.push_back(source.getType());
sourceLocs.push_back(source.getLoc());
}
auto newCompute = createEmptySpatGraphCompute(
rewriter, returnOp.getLoc(), returnOp.getOperandTypes(), {}, funcOp.getArguments(), sourceTypes, sourceLocs);
auto* newBlock = &newCompute.getBody().front();
for (auto [blockArg, computeArg] : llvm::zip(newBlock->getArguments(), newCompute.getOperands()))
mapper.map(computeArg, blockArg);
newCompute.getProperties().setOperandSegmentSizes({0, static_cast<int>(sourceTypes.size())});
rewriter.setInsertionPointToEnd(newBlock);
for (Operation& op : funcOp.getOps())
if (!isa<spatial::SpatGraphCompute, func::ReturnOp>(&op))
rewriter.clone(op, mapper);
auto yield = spatial::SpatYieldOp::create(rewriter, funcOp.getLoc(), returnOp.getOperands());
for (size_t i = 0; i < yield.getNumOperands(); ++i)
yield.setOperand(i, mapper.lookupOrDefault(yield.getOperand(i)));
for (Operation& op : llvm::make_early_inc_range(funcOp.getOps()))
if (!isa<spatial::SpatGraphCompute, func::ReturnOp>(&op)) {
op.dropAllUses();
rewriter.eraseOp(&op);
}
for (auto [index, computeResult] : llvm::enumerate(newCompute.getResults()))
returnOp.setOperand(index, computeResult);
}
void ONNXToSpatialPass::runOnOperation() {
ModuleOp moduleOp = getOperation();
if (!hasTarget) {
moduleOp.emitError("ONNX-to-Spatial lowering requires an injected SpatialTargetResources");
signalPassFailure();
return;
}
MLIRContext* ctx = &getContext();
ConversionTarget preTarget(*ctx);
preTarget.addLegalDialect<spatial::SpatialDialect,
ONNXDialect,
linalg::LinalgDialect,
tensor::TensorDialect,
affine::AffineDialect,
arith::ArithDialect,
scf::SCFDialect>();
preTarget.addIllegalOp<ONNXConstantOp>();
RewritePatternSet prePatterns(ctx);
populatePrePatterns(prePatterns, ctx);
if (failed(applyPartialConversion(moduleOp, preTarget, std::move(prePatterns)))) {
moduleOp.emitError("failed to apply ONNX-to-Spatial pre-rewrites");
signalPassFailure();
return;
}
RewritePatternSet matmulPatterns(ctx);
populateMatMulFusionPatterns(matmulPatterns, ctx, target);
if (failed(applyPatternsGreedily(moduleOp, std::move(matmulPatterns)))) {
moduleOp.emitError("failed to lower MatMul before producer conversion");
signalPassFailure();
return;
}
RewritePatternSet fusionPatterns(ctx);
populateElementwiseFusionPatterns(fusionPatterns, ctx);
if (failed(applyPatternsGreedily(moduleOp, std::move(fusionPatterns)))) {
moduleOp.emitError("failed to fuse layout-aware ONNX elementwise patterns");
signalPassFailure();
return;
}
auto entryFunc = getPimEntryFunc(moduleOp);
if (failed(entryFunc)) {
moduleOp.emitError("failed to locate the PIM entry function during ONNX-to-Spatial lowering");
signalPassFailure();
return;
}
ConversionTarget target(*ctx);
target.addLegalDialect<spatial::SpatialDialect,
ONNXDialect,
linalg::LinalgDialect,
tensor::TensorDialect,
affine::AffineDialect,
arith::ArithDialect,
scf::SCFDialect>();
target.addIllegalOp<ONNXMatMulOp>();
target.addIllegalOp<ONNXTransposeOp>();
target.addIllegalOp<ONNXAddOp>();
target.addIllegalOp<ONNXSubOp>();
target.addIllegalOp<ONNXDivOp>();
target.addIllegalOp<ONNXMulOp>();
target.addIllegalOp<ONNXGemmOp>();
target.addIllegalOp<ONNXConvOp>();
target.addIllegalOp<ONNXMaxPoolSingleOutOp>();
target.addIllegalOp<ONNXAveragePoolOp>();
target.addIllegalOp<ONNXReluOp>();
target.addIllegalOp<ONNXSigmoidOp>();
target.addIllegalOp<ONNXSoftmaxOp>();
target.addIllegalOp<ONNXConcatOp>();
target.addIllegalOp<ONNXFlattenOp>();
target.addIllegalOp<ONNXGatherOp>();
target.addIllegalOp<ONNXReshapeOp>();
target.addIllegalOp<ONNXResizeOp>();
target.addIllegalOp<ONNXSliceOp>();
target.addIllegalOp<ONNXLRNOp>();
target.addIllegalOp<ONNXReduceMeanOp>();
target.addIllegalOp<ONNXReduceMeanV13Op>();
target.addIllegalOp<ONNXSplitOp>();
RewritePatternSet conversionPatterns(ctx);
populateConversionPatterns(conversionPatterns, ctx, this->target, planningOptions);
if (failed(applyPartialConversion(moduleOp, target, std::move(conversionPatterns)))) {
moduleOp.emitError("failed to convert required ONNX ops to Spatial ops");
signalPassFailure();
return;
}
if (failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
moduleOp.emitError("logical Spatial graph verification failed after ONNX conversion");
signalPassFailure();
return;
}
ConversionTarget earlyPostTarget(*ctx);
earlyPostTarget.addLegalDialect<spatial::SpatialDialect,
ONNXDialect,
linalg::LinalgDialect,
tensor::TensorDialect,
affine::AffineDialect,
arith::ArithDialect,
scf::SCFDialect>();
annotateWeightsConstants(*entryFunc);
if (failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
moduleOp.emitError("logical Spatial graph verification failed after weight annotation");
signalPassFailure();
return;
}
ConversionTarget postTarget(*ctx);
postTarget.addLegalDialect<spatial::SpatialDialect,
ONNXDialect,
linalg::LinalgDialect,
tensor::TensorDialect,
affine::AffineDialect,
arith::ArithDialect,
scf::SCFDialect>();
postTarget.addDynamicallyLegalOp<spatial::SpatGraphCompute>(
[](spatial::SpatGraphCompute computeOp) { return !requiresPostRewrite(computeOp); });
postTarget.addDynamicallyLegalOp<spatial::SpatGraphComputeBatch>(
[](spatial::SpatGraphComputeBatch computeOp) { return !requiresPostRewrite(computeOp); });
if (failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
moduleOp.emitError("logical Spatial graph verification failed before post rewrites");
signalPassFailure();
return;
}
RewritePatternSet postPatterns(ctx);
populatePostPatterns(postPatterns, ctx);
if (failed(applyPartialConversion(*entryFunc, postTarget, std::move(postPatterns)))) {
moduleOp.emitError("failed to normalize weight-like Spatial compute operands before Spatial-to-PIM lowering");
signalPassFailure();
return;
}
populateEmptyFunction(*entryFunc);
PassManager canonicalizationPM(ctx);
canonicalizationPM.addPass(createCanonicalizerPass());
if (failed(canonicalizationPM.run(moduleOp)))
moduleOp.emitWarning("failed to run ONNXToSpatial canonicalization; continuing");
dumpModule(moduleOp, "spatial0");
if (failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
moduleOp.emitError("logical Spatial graph verification failed after ONNX-to-Spatial");
signalPassFailure();
return;
}
if (failed(verifyONNXToSpatial(*entryFunc))) {
moduleOp.emitError("ONNX-to-Spatial host legality verification failed");
signalPassFailure();
}
}
std::unique_ptr<Pass> createONNXToSpatialPass() { return std::make_unique<ONNXToSpatialPass>(); }
std::unique_ptr<Pass> createONNXToSpatialPass(
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options) {
return std::make_unique<ONNXToSpatialPass>(target, options);
}
} // namespace onnx_mlir
@@ -0,0 +1,92 @@
#pragma once
#include <optional>
#include "mlir/IR/PatternMatch.h"
#include "mlir/Support/LogicalResult.h"
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
namespace onnx_mlir {
struct RowStripPhysicalValue;
struct ONNXToSpatialPlanningOptions;
inline spatial::PhysicalLayout getSpatialPlanOperandLayout(mlir::Value value) {
if (auto materialize = value.getDefiningOp<spatial::SpatMaterializeLayoutOp>())
return materialize.getTargetPhysicalLayout();
if (auto blueprint = value.getDefiningOp<spatial::SpatBlueprintOp>())
return blueprint.getPhysicalLayout();
if (mlir::Operation* producer = value.getDefiningOp())
if (auto selected = spatial::getSelectedPhysicalLayout(producer))
return *selected;
return spatial::PhysicalLayout::DenseNCHW;
}
mlir::FailureOr<mlir::Value>
lowerDenseFlattenPlan(spatial::SpatFlattenPlanOp planOp,
mlir::Value input,
mlir::PatternRewriter& rewriter);
mlir::FailureOr<mlir::Value>
lowerSelectedConv2DPlan(spatial::SpatConv2DPlanOp planOp,
mlir::Value input,
mlir::Value weight,
mlir::Value bias,
std::optional<mlir::Value> rowStripInput,
bool emitRowStripLayout,
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options,
mlir::PatternRewriter& rewriter);
mlir::LogicalResult canLowerConvPlanToRowStrip(spatial::SpatConv2DPlanOp planOp,
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions* options = nullptr);
mlir::LogicalResult canConsumeAndProduceRowStrip(spatial::SpatConv2DPlanOp planOp,
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions* options = nullptr);
mlir::LogicalResult canLowerResizeNearestPlanToRowStrip(
spatial::SpatResizeNearestPlanOp planOp, const spatial::SpatialTargetResources& target);
mlir::FailureOr<mlir::Value> lowerSelectedResizeNearestPlan(
spatial::SpatResizeNearestPlanOp planOp,
mlir::Value input,
std::optional<mlir::Value> rowStripInput,
const spatial::SpatialTargetResources& target,
mlir::PatternRewriter& rewriter);
mlir::LogicalResult canLowerMaxPoolPlanToRowStrip(spatial::SpatMaxPool2DPlanOp planOp,
const spatial::SpatialTargetResources& target);
mlir::FailureOr<mlir::Value>
lowerDenseMaxPool2DPlan(spatial::SpatMaxPool2DPlanOp planOp,
mlir::Value input,
const spatial::SpatialTargetResources& target,
mlir::PatternRewriter& rewriter);
mlir::FailureOr<mlir::Value>
lowerSelectedMaxPool2DPlan(spatial::SpatMaxPool2DPlanOp planOp,
mlir::Value input,
std::optional<mlir::Value> rowStripInput,
const spatial::SpatialTargetResources& target,
mlir::PatternRewriter& rewriter);
mlir::LogicalResult
canLowerGlobalAveragePoolPlanToRowStrip(spatial::SpatGlobalAveragePoolPlanOp planOp,
const spatial::SpatialTargetResources& target);
mlir::FailureOr<mlir::Value>
lowerDenseGlobalAveragePoolPlan(spatial::SpatGlobalAveragePoolPlanOp planOp,
mlir::Value input,
const spatial::SpatialTargetResources& target,
mlir::PatternRewriter& rewriter);
mlir::FailureOr<mlir::Value>
lowerSelectedGlobalAveragePoolPlan(spatial::SpatGlobalAveragePoolPlanOp planOp,
mlir::Value input,
std::optional<mlir::Value> rowStripInput,
const spatial::SpatialTargetResources& target,
mlir::PatternRewriter& rewriter);
} // namespace onnx_mlir
@@ -0,0 +1,318 @@
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Pass/Pass.h"
#include "llvm/ADT/DenseMap.h"
#include "Conversion/ONNXToSpatial/Passes/Analyses/ONNXToSpatialVerifier.hpp"
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Passes/Transforms/PlanLowering.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
#include "src/Accelerators/PIM/Passes/PIMPasses.h"
#include <algorithm>
#include <limits>
using namespace mlir;
namespace onnx_mlir {
namespace {
struct SpatialLayoutSelection {
llvm::DenseMap<Operation*, unsigned> selectedAlternative;
llvm::DenseMap<Value, spatial::PhysicalLayout> resultLayouts;
};
static spatial::PhysicalLayout getKnownLayout(
const SpatialLayoutSelection& selection, Value value) {
if (auto it = selection.resultLayouts.find(value); it != selection.resultLayouts.end())
return it->second;
return getSpatialPlanOperandLayout(value);
}
static SmallVector<spatial::PhysicalLayout> getOperandLayouts(
Operation* op, const SpatialLayoutSelection& selection) {
SmallVector<spatial::PhysicalLayout> operandLayouts;
operandLayouts.reserve(op->getNumOperands());
for (Value operand : op->getOperands())
operandLayouts.push_back(getKnownLayout(selection, operand));
return operandLayouts;
}
class SpatialLayoutAnalysis {
public:
SpatialLayoutAnalysis(func::FuncOp funcOp,
const spatial::SpatialTargetResources& target)
: funcOp(funcOp), target(target) {}
FailureOr<SpatialLayoutSelection> run() {
SpatialLayoutSelection selection;
SmallVector<Operation*> planOps;
for (Operation& op : funcOp.getBody().front()) {
if (!isa<spatial::SpatialLayoutCapabilityInterface>(&op))
continue;
planOps.push_back(&op);
selection.resultLayouts[op.getResult(0)] = spatial::PhysicalLayout::DenseNCHW;
selection.selectedAlternative[&op] = 0;
}
const size_t maxRounds = 2 * planOps.size() + 1;
for (size_t round = 0; round < maxRounds; ++round) {
bool changed = false;
SmallVector<Operation*> order(planOps);
if (round % 2)
std::reverse(order.begin(), order.end());
for (Operation* op : order) {
FailureOr<SmallVector<spatial::LayoutAlternative>> alternatives =
getAlternatives(op, selection);
if (failed(alternatives))
return failure();
unsigned currentIndex = selection.selectedAlternative.lookup(op);
if (currentIndex >= alternatives->size())
currentIndex = 0;
if (selection.selectedAlternative.lookup(op) != currentIndex) {
selection.selectedAlternative[op] = currentIndex;
changed = true;
}
Value result = op->getResult(0);
if (selection.resultLayouts.lookup(result) !=
(*alternatives)[currentIndex].resultLayout) {
selection.resultLayouts[result] = (*alternatives)[currentIndex].resultLayout;
changed = true;
}
int64_t bestCost = alternativeCost(op, (*alternatives)[currentIndex], selection);
unsigned bestIndex = currentIndex;
for (auto [index, alternative] : llvm::enumerate(*alternatives)) {
int64_t cost = alternativeCost(op, alternative, selection);
if (cost < bestCost) {
bestCost = cost;
bestIndex = index;
}
}
if (bestIndex == currentIndex)
continue;
selection.selectedAlternative[op] = bestIndex;
selection.resultLayouts[result] = (*alternatives)[bestIndex].resultLayout;
changed = true;
}
if (!changed)
return selection;
}
funcOp.emitError("Spatial layout selection did not converge within its bounded iteration budget");
return failure();
}
FailureOr<SmallVector<spatial::LayoutAlternative>> getAlternatives(
Operation* op, const SpatialLayoutSelection& selection) {
auto capability = dyn_cast<spatial::SpatialLayoutCapabilityInterface>(op);
if (!capability)
return failure();
SmallVector<spatial::LayoutAlternative> alternatives =
capability.getLayoutAlternatives(target, getOperandLayouts(op, selection));
if (alternatives.empty())
return op->emitOpError("does not advertise a legal Spatial layout alternative"), failure();
for (const spatial::LayoutAlternative& alternative : alternatives) {
if (alternative.operandLayouts.size() != op->getNumOperands())
return op->emitOpError("advertises a layout alternative with the wrong operand count"), failure();
}
if (llvm::any_of(op->getResult(0).getUses(), [](OpOperand& use) {
return isa<func::ReturnOp>(use.getOwner());
})
&& llvm::none_of(alternatives, [](const spatial::LayoutAlternative& alternative) {
return alternative.resultLayout == spatial::PhysicalLayout::DenseNCHW;
}))
return op->emitOpError("does not provide the required DenseNCHW function-result layout"), failure();
return alternatives;
}
private:
int64_t alternativeCost(Operation* op,
const spatial::LayoutAlternative& alternative,
const SpatialLayoutSelection& selection) {
if (llvm::any_of(op->getResult(0).getUses(), [](OpOperand& use) {
return isa<func::ReturnOp>(use.getOwner());
})
&& alternative.resultLayout != spatial::PhysicalLayout::DenseNCHW)
return std::numeric_limits<int64_t>::max() / 4;
int64_t cost = alternative.intrinsicCost;
SmallVector<spatial::PhysicalLayout> operandLayouts = getOperandLayouts(op, selection);
for (auto [actual, required] : llvm::zip(operandLayouts, alternative.operandLayouts))
cost += actual != required;
Value result = op->getResult(0);
for (OpOperand& use : result.getUses()) {
auto user = dyn_cast<spatial::SpatialLayoutCapabilityInterface>(use.getOwner());
if (!user)
continue;
SmallVector<spatial::PhysicalLayout> userOperandLayouts =
getOperandLayouts(use.getOwner(), selection);
for (auto [index, operand] : llvm::enumerate(use.getOwner()->getOperands()))
if (operand == result)
userOperandLayouts[index] = alternative.resultLayout;
SmallVector<spatial::LayoutAlternative> userAlternatives =
user.getLayoutAlternatives(target, userOperandLayouts);
if (llvm::none_of(userAlternatives,
[&](const spatial::LayoutAlternative& userAlternative) {
return userAlternative.operandLayouts.size()
== use.getOwner()->getNumOperands()
&& userAlternative.operandLayouts[use.getOperandNumber()]
== alternative.resultLayout;
}))
++cost;
}
return cost;
}
func::FuncOp funcOp;
const spatial::SpatialTargetResources& target;
};
static LogicalResult materializeMismatchedUses(
IRRewriter& rewriter, const SpatialLayoutSelection& selection,
Operation* op, SpatialLayoutAnalysis& analysis) {
Value value = op->getResult(0);
spatial::PhysicalLayout sourceLayout = getKnownLayout(selection, value);
SmallVector<std::pair<OpOperand*, spatial::PhysicalLayout>> mismatches;
for (OpOperand& use : value.getUses()) {
Operation* userOp = use.getOwner();
auto capability = dyn_cast<spatial::SpatialLayoutCapabilityInterface>(userOp);
if (!capability) {
if (isa<func::ReturnOp>(userOp) || sourceLayout == spatial::PhysicalLayout::DenseNCHW)
continue;
mismatches.push_back({&use, spatial::PhysicalLayout::DenseNCHW});
continue;
}
FailureOr<SmallVector<spatial::LayoutAlternative>> alternatives =
analysis.getAlternatives(userOp, selection);
if (failed(alternatives))
return failure();
unsigned selectedIndex = selection.selectedAlternative.lookup(userOp);
if (selectedIndex >= alternatives->size())
return userOp->emitOpError()
<< "has no selected Spatial layout alternative (index " << selectedIndex
<< ", alternatives " << alternatives->size() << ")",
failure();
spatial::PhysicalLayout required =
(*alternatives)[selectedIndex].operandLayouts[use.getOperandNumber()];
if (required != sourceLayout)
mismatches.push_back({&use, required});
}
for (auto [use, required] : mismatches) {
Operation* userOp = use->getOwner();
rewriter.setInsertionPoint(userOp);
auto materialized = spatial::SpatMaterializeLayoutOp::create(
rewriter, userOp->getLoc(), use->get().getType(), use->get(),
spatial::LogicalLayoutAttr::get(
rewriter.getContext(), spatial::LogicalLayout::NCHW),
spatial::PhysicalLayoutAttr::get(rewriter.getContext(), sourceLayout),
spatial::PhysicalLayoutAttr::get(rewriter.getContext(), required));
use->set(materialized.getResult());
}
return success();
}
static LogicalResult verifySelectedLayouts(
const SpatialLayoutSelection& selection,
ArrayRef<Operation*> planOps,
SpatialLayoutAnalysis& analysis) {
for (Operation* op : planOps) {
auto selected = spatial::getSelectedPhysicalLayout(op);
if (!selected)
return op->emitOpError("requires a selected physical layout"), failure();
FailureOr<SmallVector<spatial::LayoutAlternative>> alternatives =
analysis.getAlternatives(op, selection);
if (failed(alternatives))
return failure();
unsigned selectedIndex = selection.selectedAlternative.lookup(op);
if (selectedIndex >= alternatives->size())
return op->emitOpError()
<< "has no selected Spatial layout alternative (index " << selectedIndex
<< ", alternatives " << alternatives->size() << ")",
failure();
const spatial::LayoutAlternative& alternative = (*alternatives)[selectedIndex];
if (alternative.resultLayout != *selected
|| getOperandLayouts(op, selection) != alternative.operandLayouts)
return op->emitOpError("selected physical layout does not satisfy its exact layout contract"), failure();
}
return success();
}
struct SpatialLayoutPlanningPass final
: PassWrapper<SpatialLayoutPlanningPass, OperationPass<ModuleOp>> {
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SpatialLayoutPlanningPass)
StringRef getArgument() const override { return "spatial-layout-planning"; }
StringRef getDescription() const override {
return "Select Spatial layout alternatives and insert explicit reconciliation barriers.";
}
SpatialLayoutPlanningPass() = default;
explicit SpatialLayoutPlanningPass(const spatial::SpatialTargetResources& target)
: target(target), hasTarget(true) {}
void runOnOperation() override {
ModuleOp moduleOp = getOperation();
if (!hasTarget) {
moduleOp.emitError("Spatial layout planning requires an injected SpatialTargetResources");
signalPassFailure();
return;
}
auto entryFunc = getPimEntryFunc(moduleOp);
if (failed(entryFunc)) {
moduleOp.emitError("failed to locate the PIM entry function during Spatial layout planning");
signalPassFailure();
return;
}
func::FuncOp funcOp = *entryFunc;
SpatialLayoutAnalysis analysis(funcOp, target);
FailureOr<SpatialLayoutSelection> selection = analysis.run();
if (failed(selection)) {
signalPassFailure();
return;
}
SmallVector<Operation*> planOps;
for (Operation& op : funcOp.getBody().front())
if (isa<spatial::SpatialLayoutCapabilityInterface>(&op))
planOps.push_back(&op);
IRRewriter rewriter(&getContext());
for (Operation* op : planOps) {
op->setAttr(spatial::kSelectedLayoutAttrName,
spatial::PhysicalLayoutAttr::get(
rewriter.getContext(), selection->resultLayouts.lookup(op->getResult(0))));
if (failed(materializeMismatchedUses(
rewriter, *selection, op, analysis))) {
signalPassFailure();
return;
}
}
if (failed(verifySelectedLayouts(*selection, planOps, analysis))
|| failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
moduleOp.emitError("Spatial layout planning verification failed");
signalPassFailure();
return;
}
}
spatial::SpatialTargetResources target;
bool hasTarget = false;
};
} // namespace
std::unique_ptr<Pass> createSpatialLayoutPlanningPass() {
return std::make_unique<SpatialLayoutPlanningPass>();
}
std::unique_ptr<Pass> createSpatialLayoutPlanningPass(
const spatial::SpatialTargetResources& target) {
return std::make_unique<SpatialLayoutPlanningPass>(target);
}
} // namespace onnx_mlir
@@ -0,0 +1,845 @@
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Transforms/DialectConversion.h"
#include "Conversion/ONNXToSpatial/Passes/Analyses/ONNXToSpatialVerifier.hpp"
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
#include "src/Accelerators/PIM/Common/Support/DebugDump.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/BiasAddUtils.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/MatrixProductLowering.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/RowStripLayoutUtils.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns.hpp"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Passes/Transforms/PlanLowering.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/SpatialDataflowCsvExporter.hpp"
#include "src/Accelerators/PIM/Passes/PIMPasses.h"
using namespace mlir;
namespace onnx_mlir {
namespace {
static FailureOr<RowStripPhysicalValue> getRowStripValue(Value value) {
return getRowStripPhysicalValue(value);
}
static FailureOr<Value> publishRowStripValue(Operation* planOp,
Value storage,
PatternRewriter& rewriter) {
auto logicalType = dyn_cast<RankedTensorType>(planOp->getResult(0).getType());
if (!logicalType)
return planOp->emitOpError("requires ranked logical output type"), failure();
FailureOr<RowStripPhysicalValue> value = describeRowStripPhysicalValue(storage, logicalType);
if (failed(value))
return planOp->emitOpError("lowering produced invalid row-strip physical storage"), failure();
FailureOr<Value> blueprint = createRowStripStorageBlueprint(
storage, logicalType, rewriter, planOp->getLoc());
if (failed(blueprint))
return planOp->emitOpError("failed to create row-strip storage Blueprint"), failure();
rewriter.replaceOp(planOp, *blueprint);
return *blueprint;
}
static bool isRowStripSelected(Operation* op) {
auto selected = spatial::getSelectedPhysicalLayout(op);
return selected && *selected == spatial::PhysicalLayout::NHWCRowStrip;
}
static bool isDenseSelected(Operation* op) {
auto selected = spatial::getSelectedPhysicalLayout(op);
return selected && *selected == spatial::PhysicalLayout::DenseNCHW;
}
static spatial::PhysicalLayout getKnownPhysicalLayout(Value value) {
return getSpatialPlanOperandLayout(value);
}
static LogicalResult verifySelectedLayouts(
func::FuncOp funcOp, const spatial::SpatialTargetResources& target) {
LogicalResult result = success();
funcOp.walk([&](Operation* op) {
auto capability = dyn_cast<spatial::SpatialLayoutCapabilityInterface>(op);
if (!capability)
return;
auto selected = spatial::getSelectedPhysicalLayout(op);
if (!selected) {
op->emitOpError("requires a selected physical layout from SpatialLayoutPlanning");
result = failure();
return;
}
if (*selected != spatial::PhysicalLayout::DenseNCHW
&& *selected != spatial::PhysicalLayout::NHWCRowStrip) {
op->emitOpError("has an unsupported selected physical layout");
result = failure();
return;
}
SmallVector<spatial::PhysicalLayout> operandLayouts;
operandLayouts.reserve(op->getNumOperands());
for (Value operand : op->getOperands())
operandLayouts.push_back(getKnownPhysicalLayout(operand));
auto alternatives = capability.getLayoutAlternatives(target, operandLayouts);
if (llvm::none_of(alternatives, [&](const spatial::LayoutAlternative& alternative) {
return alternative.resultLayout == *selected
&& alternative.operandLayouts == operandLayouts;
})) {
op->emitOpError("selected physical layout is not lowerable for its explicit operand layouts");
result = failure();
}
});
return result;
}
static FailureOr<Value>
lowerRowStripRelu(const RowStripPhysicalValue& input, spatial::SpatReluPlanOp planOp, PatternRewriter& rewriter) {
return applyRowStripRelu(input, rewriter, planOp.getLoc());
}
static FailureOr<Value>
lowerRowStripSilu(const RowStripPhysicalValue& input, spatial::SpatSiluPlanOp planOp, PatternRewriter& rewriter) {
return applyRowStripSilu(input, rewriter, planOp.getLoc());
}
static FailureOr<Value> lowerRowStripAdd(const RowStripPhysicalValue& lhs,
const RowStripPhysicalValue& rhs,
spatial::SpatAddPlanOp planOp,
PatternRewriter& rewriter) {
return applyRowStripAdd(lhs, rhs, rewriter, planOp.getLoc());
}
static FailureOr<Value> lowerRowStripConcat(ArrayRef<RowStripPhysicalValue> inputs,
spatial::SpatConcatPlanOp planOp,
PatternRewriter& rewriter) {
auto outputType = dyn_cast<RankedTensorType>(planOp.getOutput().getType());
if (!outputType)
return failure();
return applyRowStripConcat(inputs, outputType, rewriter, planOp.getLoc());
}
static FailureOr<Value>
materializeRowStripToDense(const RowStripPhysicalValue& rowStripValue, Location loc, PatternRewriter& rewriter) {
if (rowStripValue.logicalType.getRank() != 4 || !rowStripValue.logicalType.hasStaticShape())
return failure();
return createRowStripAssemblyBlueprint(rowStripValue, rewriter, loc);
}
static FailureOr<Value> materializeDenseToRowStrip(
Value input, RankedTensorType logicalType, Location loc, PatternRewriter& rewriter) {
if (!logicalType || !logicalType.hasStaticShape() || logicalType.getRank() != 4
|| logicalType.getDimSize(0) != 1)
return failure();
auto nhwcType = RankedTensorType::get(
{1, logicalType.getDimSize(2), logicalType.getDimSize(3), logicalType.getDimSize(1)},
logicalType.getElementType(), logicalType.getEncoding());
auto rowsType = RankedTensorType::get(
{logicalType.getDimSize(2) * logicalType.getDimSize(3), logicalType.getDimSize(1)},
logicalType.getElementType(), logicalType.getEncoding());
auto rowsCompute = createSpatCompute<1>(
rewriter, loc, rowsType, {}, input, [&](Value denseInput) {
Value nhwc = createLinalgTranspose(
denseInput, nhwcType, {0, 2, 3, 1}, rewriter, loc);
Value rows = tensor::CollapseShapeOp::create(
rewriter, loc, rowsType, nhwc,
SmallVector<ReassociationIndices> {{0, 1, 2}, {3}});
spatial::SpatYieldOp::create(rewriter, loc, rows);
});
Value rows = rowsCompute->getResult(0);
FailureOr<Value> storage = createRowStripStorageFromRows(rows, logicalType, rewriter, loc);
if (failed(storage))
return failure();
return createRowStripStorageBlueprint(*storage, logicalType, rewriter, loc);
}
static FailureOr<Value> lowerDenseBatchBiasAdd(Value input, Value bias, RankedTensorType resultType,
PatternRewriter& rewriter, Location loc) {
auto producer = input.getDefiningOp<spatial::SpatGraphComputeBatch>();
auto inputType = dyn_cast<RankedTensorType>(input.getType());
auto biasType = dyn_cast<RankedTensorType>(bias.getType());
if (!producer || !inputType || !biasType || !inputType.hasStaticShape() || !biasType.hasStaticShape()
|| !resultType.hasStaticShape() || inputType.getDimSize(0) != producer.getLaneCount()
|| biasType.getDimSize(0) != producer.getLaneCount() || resultType.getDimSize(0) != producer.getLaneCount())
return failure();
auto inputFragmentType = spatial::getGraphBatchFragmentType(inputType, producer.getLaneCount());
auto outputFragmentType = spatial::getGraphBatchFragmentType(resultType, producer.getLaneCount());
if (failed(inputFragmentType) || failed(outputFragmentType) || inputFragmentType->getRank() != biasType.getRank()
|| inputFragmentType->getDimSize(0) != 1 || inputFragmentType->getShape().drop_front() != biasType.getShape().drop_front()
|| inputFragmentType->getRank() != outputFragmentType->getRank() + 1)
return failure();
for (auto [inputDim, outputDim] : llvm::zip(inputFragmentType->getShape().drop_front(), outputFragmentType->getShape()))
if (outputDim > inputDim)
return failure();
auto batch = createSpatComputeBatch(rewriter, loc, TypeRange {resultType}, producer.getLaneCount(), {}, ValueRange {input, bias},
[&](detail::SpatComputeBatchBodyArgs args) -> LogicalResult {
FailureOr<Value> fragment = extractGraphBatchPhysicalFragment(rewriter, loc, args.inputs[0], args.lane, *inputFragmentType);
if (failed(fragment))
return failure();
MixedSliceGeometry biasSlice;
for (int64_t dim : inputFragmentType->getShape()) {
biasSlice.offsets.push_back(biasSlice.offsets.empty() ? OpFoldResult(args.lane) : rewriter.getIndexAttr(0));
biasSlice.sizes.push_back(rewriter.getIndexAttr(dim));
biasSlice.strides.push_back(rewriter.getIndexAttr(1));
}
Value biasFragment = extractMixedSliceOrIdentity(rewriter, loc, args.inputs[1], *inputFragmentType, biasSlice);
if (!biasFragment)
return failure();
Value added = spatial::SpatVAddOp::create(rewriter, loc, *inputFragmentType, *fragment, biasFragment);
MixedSliceGeometry outputSlice;
outputSlice.offsets.assign(inputFragmentType->getRank(), rewriter.getIndexAttr(0));
outputSlice.sizes.push_back(rewriter.getIndexAttr(1));
outputSlice.strides.assign(inputFragmentType->getRank(), rewriter.getIndexAttr(1));
for (int64_t dim : outputFragmentType->getShape())
outputSlice.sizes.push_back(rewriter.getIndexAttr(dim));
Value output = extractMixedSliceOrIdentity(rewriter, loc, added, *outputFragmentType, outputSlice);
if (!output)
return failure();
publishGraphBatchPhysicalFragment(rewriter, loc, output, args.outputs.front(), args.lane);
return success();
});
if (failed(batch))
return failure();
return batch->getResult(0);
}
struct LowerDenseReluPlan final : OpConversionPattern<spatial::SpatReluPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatReluPlanOp planOp,
spatial::SpatReluPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
auto selected = spatial::getSelectedPhysicalLayout(planOp.getOperation());
if (!selected || *selected != spatial::PhysicalLayout::DenseNCHW)
return failure();
auto computeOp = createSpatCompute<1>(
rewriter, planOp.getLoc(), planOp.getOutput().getType(), {}, adaptor.getInput(), [&](Value x) {
auto relu = spatial::SpatReluOp::create(rewriter, planOp.getLoc(), planOp.getOutput().getType(), x);
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), relu.getResult());
});
rewriter.replaceOp(planOp, computeOp.getResults());
return success();
}
};
struct LowerDenseSiluPlan final : OpConversionPattern<spatial::SpatSiluPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatSiluPlanOp planOp,
spatial::SpatSiluPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
auto selected = spatial::getSelectedPhysicalLayout(planOp.getOperation());
if (!selected || *selected != spatial::PhysicalLayout::DenseNCHW)
return failure();
auto computeOp = createSpatCompute<1>(
rewriter, planOp.getLoc(), planOp.getOutput().getType(), {}, adaptor.getInput(), [&](Value x) {
Value sigmoid = spatial::SpatSigmoidOp::create(
rewriter, planOp.getLoc(), planOp.getOutput().getType(), x).getResult();
Value silu = spatial::SpatVMulOp::create(
rewriter, planOp.getLoc(), planOp.getOutput().getType(), x, sigmoid).getResult();
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), silu);
});
rewriter.replaceOp(planOp, computeOp.getResults());
return success();
}
};
struct LowerDenseResizePlan final : OpConversionPattern<spatial::SpatResizeNearestPlanOp> {
explicit LowerDenseResizePlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
: OpConversionPattern<spatial::SpatResizeNearestPlanOp>(ctx), target(target) {}
LogicalResult matchAndRewrite(spatial::SpatResizeNearestPlanOp planOp,
spatial::SpatResizeNearestPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isDenseSelected(planOp.getOperation()))
return failure();
FailureOr<Value> lowered = lowerSelectedResizeNearestPlan(
planOp, adaptor.getInput(), std::nullopt, target, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected dense nearest Resize plan");
rewriter.replaceOp(planOp, *lowered);
return success();
}
const spatial::SpatialTargetResources& target;
};
struct LowerDenseBiasAddPlan final : OpConversionPattern<spatial::SpatBiasAddPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatBiasAddPlanOp planOp,
spatial::SpatBiasAddPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isDenseSelected(planOp.getOperation()))
return failure();
auto resultType = dyn_cast<RankedTensorType>(planOp.getOutput().getType());
if (!resultType)
return planOp.emitOpError("requires ranked output type");
FailureOr<Value> denseBias = materializeDenseBiasAddTensor(
adaptor.getBias(), resultType, rewriter, planOp.getLoc());
if (failed(denseBias))
return planOp.emitOpError("failed to materialize dense Conv-style bias");
if (adaptor.getInput().getDefiningOp<spatial::SpatGraphComputeBatch>()) {
FailureOr<Value> lowered = lowerDenseBatchBiasAdd(
adaptor.getInput(), *denseBias, resultType, rewriter, planOp.getLoc());
if (succeeded(lowered)) {
rewriter.replaceOp(planOp, *lowered);
return success();
}
}
auto computeOp = createSpatCompute<2>(
rewriter,
planOp.getLoc(),
planOp.getOutput().getType(),
{},
ValueRange {adaptor.getInput(), *denseBias},
[&](Value x, Value y) {
auto added = spatial::SpatVAddOp::create(
rewriter, planOp.getLoc(), planOp.getOutput().getType(), x, y);
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), added.getResult());
});
rewriter.replaceOp(planOp, computeOp.getResults());
return success();
}
};
struct LowerDenseAddPlan final : OpConversionPattern<spatial::SpatAddPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatAddPlanOp planOp,
spatial::SpatAddPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isDenseSelected(planOp.getOperation()))
return failure();
auto compute = createSpatCompute<2>(
rewriter,
planOp.getLoc(),
planOp.getOutput().getType(),
{},
ValueRange {adaptor.getLhs(), adaptor.getRhs()},
[&](Value lhsValue, Value rhsValue) {
Value added = spatial::SpatVAddOp::create(
rewriter, planOp.getLoc(), planOp.getOutput().getType(), lhsValue, rhsValue);
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), added);
});
rewriter.replaceOp(planOp, compute.getResults());
return success();
}
};
struct LowerDenseConcatPlan final : OpConversionPattern<spatial::SpatConcatPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatConcatPlanOp planOp,
spatial::SpatConcatPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isDenseSelected(planOp.getOperation()))
return failure();
auto compute = createSpatCompute(
rewriter,
planOp.getLoc(),
TypeRange {planOp.getOutput().getType()},
{},
adaptor.getInputs(),
[&](ValueRange values) {
Value concatenated = spatial::SpatConcatOp::create(
rewriter,
planOp.getLoc(),
planOp.getOutput().getType(),
rewriter.getI64IntegerAttr(planOp.getAxis()),
values);
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), concatenated);
});
rewriter.replaceOp(planOp, compute.getResults());
return success();
}
};
static LogicalResult lowerAddPlan(spatial::SpatAddPlanOp planOp,
Value lhsValue, Value rhsValue,
PatternRewriter& rewriter) {
FailureOr<RowStripPhysicalValue> lhs = getRowStripValue(lhsValue);
FailureOr<RowStripPhysicalValue> rhs = getRowStripValue(rhsValue);
if (isRowStripSelected(planOp.getOperation()) && failed(lhs)) {
if (getKnownPhysicalLayout(lhsValue) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
return planOp.emitOpError("selected row-strip Add plan requires row-strip inputs");
}
if (isRowStripSelected(planOp.getOperation()) && failed(rhs)) {
if (getKnownPhysicalLayout(rhsValue) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
return planOp.emitOpError("selected row-strip Add plan requires row-strip inputs");
}
if (isRowStripSelected(planOp.getOperation())) {
rewriter.setInsertionPoint(planOp);
FailureOr<Value> lowered = lowerRowStripAdd(*lhs, *rhs, planOp, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Spatial add plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
return planOp.emitOpError("dense Add plan was not lowered by the selected-plan patterns");
}
static LogicalResult lowerConcatPlan(spatial::SpatConcatPlanOp planOp,
ValueRange inputValues,
PatternRewriter& rewriter) {
SmallVector<RowStripPhysicalValue> inputs;
for (Value input : inputValues) {
FailureOr<RowStripPhysicalValue> physical = getRowStripValue(input);
if (failed(physical)) {
inputs.clear();
break;
}
inputs.push_back(*physical);
}
if (isRowStripSelected(planOp.getOperation()) && inputs.size() != inputValues.size()) {
if (llvm::any_of(inputValues, [](Value input) {
return getKnownPhysicalLayout(input) == spatial::PhysicalLayout::NHWCRowStrip;
}))
return failure();
return planOp.emitOpError("selected row-strip Concat plan requires row-strip inputs");
}
if (isRowStripSelected(planOp.getOperation())) {
rewriter.setInsertionPoint(planOp);
FailureOr<Value> lowered = lowerRowStripConcat(inputs, planOp, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Spatial concat plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
return planOp.emitOpError("dense Concat plan was not lowered by the selected-plan patterns");
}
struct LowerSelectedConvPlan final : OpConversionPattern<spatial::SpatConv2DPlanOp> {
explicit LowerSelectedConvPlan(MLIRContext* ctx,
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options)
: OpConversionPattern<spatial::SpatConv2DPlanOp>(ctx), target(target), options(options) {}
LogicalResult matchAndRewrite(spatial::SpatConv2DPlanOp planOp,
spatial::SpatConv2DPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (isDenseSelected(planOp.getOperation())) {
FailureOr<Value> lowered = lowerSelectedConv2DPlan(
planOp, adaptor.getInput(), adaptor.getWeight(), adaptor.getBias(),
std::nullopt, /*emitRowStripLayout=*/false, target, options, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected dense Spatial Conv plan");
rewriter.replaceOp(planOp, *lowered);
return success();
}
if (!isRowStripSelected(planOp.getOperation()))
return failure();
FailureOr<RowStripPhysicalValue> rowStripInput = getRowStripValue(adaptor.getInput());
if (failed(rowStripInput)
&& getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
std::optional<Value> physicalInput;
if (succeeded(rowStripInput))
physicalInput = rowStripInput->storage;
FailureOr<Value> lowered = lowerSelectedConv2DPlan(
planOp, adaptor.getInput(), adaptor.getWeight(), adaptor.getBias(),
physicalInput, /*emitRowStripLayout=*/true, target, options, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Spatial Conv plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
const spatial::SpatialTargetResources& target;
const ONNXToSpatialPlanningOptions& options;
};
struct LowerRowStripReluPlan final : OpConversionPattern<spatial::SpatReluPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatReluPlanOp planOp,
spatial::SpatReluPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isRowStripSelected(planOp.getOperation()))
return failure();
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
if (failed(input)) {
if (getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
return planOp.emitOpError("selected row-strip ReLU plan requires a row-strip input");
}
FailureOr<Value> lowered = lowerRowStripRelu(*input, planOp, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Spatial ReLU plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
};
struct LowerRowStripSiluPlan final : OpConversionPattern<spatial::SpatSiluPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatSiluPlanOp planOp,
spatial::SpatSiluPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isRowStripSelected(planOp.getOperation()))
return failure();
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
if (failed(input)) {
if (getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
return planOp.emitOpError("selected row-strip SiLU plan requires a row-strip input");
}
FailureOr<Value> lowered = lowerRowStripSilu(*input, planOp, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Spatial SiLU plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
};
struct LowerRowStripResizePlan final : OpConversionPattern<spatial::SpatResizeNearestPlanOp> {
explicit LowerRowStripResizePlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
: OpConversionPattern<spatial::SpatResizeNearestPlanOp>(ctx), target(target) {}
LogicalResult matchAndRewrite(spatial::SpatResizeNearestPlanOp planOp,
spatial::SpatResizeNearestPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isRowStripSelected(planOp.getOperation()))
return failure();
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
if (failed(input)) {
if (getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
return planOp.emitOpError("selected row-strip Resize plan requires a row-strip input");
}
FailureOr<Value> lowered = lowerSelectedResizeNearestPlan(
planOp, adaptor.getInput(), input->storage, target, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Resize plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
const spatial::SpatialTargetResources& target;
};
struct LowerDenseMaxPoolPlan final : OpConversionPattern<spatial::SpatMaxPool2DPlanOp> {
explicit LowerDenseMaxPoolPlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
: OpConversionPattern<spatial::SpatMaxPool2DPlanOp>(ctx), target(target) {}
LogicalResult matchAndRewrite(spatial::SpatMaxPool2DPlanOp planOp,
spatial::SpatMaxPool2DPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isDenseSelected(planOp.getOperation()))
return failure();
FailureOr<Value> lowered = lowerDenseMaxPool2DPlan(
planOp, adaptor.getInput(), target, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected dense Spatial MaxPool plan");
rewriter.replaceOp(planOp, *lowered);
return success();
}
const spatial::SpatialTargetResources& target;
};
struct LowerRowStripMaxPoolPlan final : OpConversionPattern<spatial::SpatMaxPool2DPlanOp> {
explicit LowerRowStripMaxPoolPlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
: OpConversionPattern<spatial::SpatMaxPool2DPlanOp>(ctx), target(target) {}
LogicalResult matchAndRewrite(spatial::SpatMaxPool2DPlanOp planOp,
spatial::SpatMaxPool2DPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isRowStripSelected(planOp.getOperation()))
return failure();
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
if (failed(input)
&& getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
std::optional<Value> physicalInput;
if (succeeded(input))
physicalInput = input->storage;
FailureOr<Value> lowered = lowerSelectedMaxPool2DPlan(
planOp, adaptor.getInput(), physicalInput, target, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Spatial MaxPool plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
const spatial::SpatialTargetResources& target;
};
struct LowerRowStripGlobalAveragePoolPlan
final : OpConversionPattern<spatial::SpatGlobalAveragePoolPlanOp> {
explicit LowerRowStripGlobalAveragePoolPlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
: OpConversionPattern<spatial::SpatGlobalAveragePoolPlanOp>(ctx), target(target) {}
LogicalResult matchAndRewrite(spatial::SpatGlobalAveragePoolPlanOp planOp,
spatial::SpatGlobalAveragePoolPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isRowStripSelected(planOp.getOperation()))
return failure();
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
if (failed(input)
&& getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
std::optional<Value> physicalInput;
if (succeeded(input))
physicalInput = input->storage;
FailureOr<Value> lowered = lowerSelectedGlobalAveragePoolPlan(
planOp, adaptor.getInput(), physicalInput, target, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Spatial global AveragePool plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
const spatial::SpatialTargetResources& target;
};
struct LowerDenseGlobalAveragePoolPlan
final : OpConversionPattern<spatial::SpatGlobalAveragePoolPlanOp> {
explicit LowerDenseGlobalAveragePoolPlan(MLIRContext* ctx,
const spatial::SpatialTargetResources& target)
: OpConversionPattern<spatial::SpatGlobalAveragePoolPlanOp>(ctx), target(target) {}
LogicalResult matchAndRewrite(spatial::SpatGlobalAveragePoolPlanOp planOp,
spatial::SpatGlobalAveragePoolPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isDenseSelected(planOp.getOperation()))
return failure();
FailureOr<Value> lowered = lowerDenseGlobalAveragePoolPlan(
planOp, adaptor.getInput(), target, rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected dense Spatial global AveragePool plan");
rewriter.replaceOp(planOp, *lowered);
return success();
}
const spatial::SpatialTargetResources& target;
};
struct LowerRowStripBiasAddPlan final : OpConversionPattern<spatial::SpatBiasAddPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatBiasAddPlanOp planOp,
spatial::SpatBiasAddPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isRowStripSelected(planOp.getOperation()))
return failure();
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
if (failed(input)) {
if (getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
return failure();
return planOp.emitOpError("selected row-strip bias_add plan requires a row-strip input");
}
FailureOr<Value> lowered = applyRowStripBiasAdd(
*input, adaptor.getBias(), rewriter, planOp.getLoc());
if (failed(lowered))
return planOp.emitOpError("failed to lower selected row-strip Spatial bias_add plan");
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
return failure();
return success();
}
};
struct LowerRowStripAddPlan final : OpConversionPattern<spatial::SpatAddPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatAddPlanOp planOp,
spatial::SpatAddPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isRowStripSelected(planOp.getOperation()))
return failure();
return lowerAddPlan(planOp, adaptor.getLhs(), adaptor.getRhs(), rewriter);
}
};
struct LowerRowStripConcatPlan final : OpConversionPattern<spatial::SpatConcatPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatConcatPlanOp planOp,
spatial::SpatConcatPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isRowStripSelected(planOp.getOperation()))
return failure();
return lowerConcatPlan(planOp, adaptor.getInputs(), rewriter);
}
};
struct LowerMaterializeLayout final
: OpConversionPattern<spatial::SpatMaterializeLayoutOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatMaterializeLayoutOp materializeOp,
spatial::SpatMaterializeLayoutOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
auto source = materializeOp.getSourcePhysicalLayout();
auto target = materializeOp.getTargetPhysicalLayout();
if (source == spatial::PhysicalLayout::DenseNCHW
&& target == spatial::PhysicalLayout::DenseNCHW) {
rewriter.replaceOp(materializeOp, adaptor.getInput());
return success();
}
if (source == spatial::PhysicalLayout::DenseNCHW
&& target == spatial::PhysicalLayout::NHWCRowStrip) {
auto logicalType = dyn_cast<RankedTensorType>(adaptor.getInput().getType());
if (!logicalType)
return materializeOp.emitOpError("requires a ranked dense input"), failure();
FailureOr<Value> rowStrip = materializeDenseToRowStrip(
adaptor.getInput(), logicalType, materializeOp.getLoc(), rewriter);
if (failed(rowStrip))
return materializeOp.emitOpError(
"failed to materialize dense NCHW storage to row-strip layout"), failure();
rewriter.replaceOp(materializeOp, *rowStrip);
return success();
}
if (source != spatial::PhysicalLayout::NHWCRowStrip
|| target != spatial::PhysicalLayout::DenseNCHW)
return materializeOp.emitOpError(
"unsupported Spatial layout materialization direction"), failure();
auto inputType = dyn_cast<RankedTensorType>(adaptor.getInput().getType());
if (!inputType)
return materializeOp.emitOpError("requires a ranked row-strip input"), failure();
FailureOr<RowStripPhysicalValue> rowStripValue =
getRowStripValue(adaptor.getInput());
if (failed(rowStripValue))
return failure();
FailureOr<Value> dense = materializeRowStripToDense(
*rowStripValue, materializeOp.getLoc(), rewriter);
if (failed(dense))
return materializeOp.emitOpError(
"failed to materialize row-strip storage to dense NCHW"), failure();
rewriter.replaceOp(materializeOp, *dense);
return success();
}
};
struct LowerSelectedFlattenPlan final
: OpConversionPattern<spatial::SpatFlattenPlanOp> {
using OpConversionPattern::OpConversionPattern;
LogicalResult matchAndRewrite(spatial::SpatFlattenPlanOp planOp,
spatial::SpatFlattenPlanOpAdaptor adaptor,
ConversionPatternRewriter& rewriter) const override {
if (!isDenseSelected(planOp.getOperation()))
return failure();
FailureOr<RowStripPhysicalValue> rowStripInput = getRowStripValue(adaptor.getInput());
if (succeeded(rowStripInput)) {
if (failed(canLowerFlattenFromRowStrip(planOp, target))
|| failed(lowerFlattenFromRowStrip(*rowStripInput, planOp, target, rewriter)))
return planOp.emitOpError("failed to lower selected Spatial Flatten plan"), failure();
return success();
}
FailureOr<Value> lowered = lowerDenseFlattenPlan(planOp, adaptor.getInput(), rewriter);
if (failed(lowered))
return planOp.emitOpError("failed to lower selected dense Spatial Flatten plan"), failure();
rewriter.replaceOp(planOp, *lowered);
return success();
}
explicit LowerSelectedFlattenPlan(MLIRContext* context,
const spatial::SpatialTargetResources& target)
: OpConversionPattern<spatial::SpatFlattenPlanOp>(context), target(target) {}
const spatial::SpatialTargetResources& target;
};
struct EraseDeadPhysicalViewBlueprint final
: OpRewritePattern<spatial::SpatBlueprintOp> {
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(spatial::SpatBlueprintOp blueprint,
PatternRewriter& rewriter) const override {
if (!spatial::isPhysicalView(blueprint.getMode()) || !blueprint.use_empty())
return failure();
rewriter.eraseOp(blueprint);
return success();
}
};
static void populateConvPlanLoweringPatterns(
RewritePatternSet& patterns, MLIRContext* ctx,
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options) {
patterns.add<LowerSelectedConvPlan>(ctx, target, options);
}
static void populateElementwisePlanLoweringPatterns(
RewritePatternSet& patterns, MLIRContext* ctx) {
patterns.add<LowerDenseReluPlan,
LowerRowStripReluPlan,
LowerDenseSiluPlan,
LowerRowStripSiluPlan,
LowerDenseBiasAddPlan,
LowerRowStripBiasAddPlan,
LowerDenseAddPlan,
LowerRowStripAddPlan>(ctx);
}
static void populatePoolPlanLoweringPatterns(
RewritePatternSet& patterns, MLIRContext* ctx,
const spatial::SpatialTargetResources& target) {
patterns.add<LowerDenseMaxPoolPlan,
LowerRowStripMaxPoolPlan,
LowerDenseGlobalAveragePoolPlan,
LowerRowStripGlobalAveragePoolPlan>(ctx, target);
}
static void populateResizePlanLoweringPatterns(
RewritePatternSet& patterns, MLIRContext* ctx,
const spatial::SpatialTargetResources& target) {
patterns.add<LowerDenseResizePlan, LowerRowStripResizePlan>(ctx, target);
}
static void populateConcatPlanLoweringPatterns(
RewritePatternSet& patterns, MLIRContext* ctx) {
patterns.add<LowerDenseConcatPlan, LowerRowStripConcatPlan>(ctx);
}
static void populateFlattenPlanLoweringPatterns(
RewritePatternSet& patterns, MLIRContext* ctx,
const spatial::SpatialTargetResources& target) {
patterns.add<LowerSelectedFlattenPlan>(ctx, target);
}
static void populateLayoutMaterializationPatterns(
RewritePatternSet& patterns, MLIRContext* ctx) {
patterns.add<LowerMaterializeLayout, EraseDeadPhysicalViewBlueprint>(ctx);
}
} // namespace
void populateSpatialPlanLoweringPatterns(
RewritePatternSet& patterns, MLIRContext* ctx,
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options) {
populateConvPlanLoweringPatterns(patterns, ctx, target, options);
populateElementwisePlanLoweringPatterns(patterns, ctx);
populatePoolPlanLoweringPatterns(patterns, ctx, target);
populateResizePlanLoweringPatterns(patterns, ctx, target);
populateConcatPlanLoweringPatterns(patterns, ctx);
populateFlattenPlanLoweringPatterns(patterns, ctx, target);
populateLayoutMaterializationPatterns(patterns, ctx);
}
LogicalResult verifySelectedSpatialLayouts(
func::FuncOp funcOp, const spatial::SpatialTargetResources& target) {
return verifySelectedLayouts(funcOp, target);
}
} // namespace onnx_mlir
@@ -0,0 +1,20 @@
#pragma once
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Support/LogicalResult.h"
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialTargetResources.hpp"
namespace onnx_mlir {
void populateSpatialPlanLoweringPatterns(
mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx,
const spatial::SpatialTargetResources& target,
const ONNXToSpatialPlanningOptions& options);
mlir::LogicalResult verifySelectedSpatialLayouts(
mlir::func::FuncOp funcOp, const spatial::SpatialTargetResources& target);
} // namespace onnx_mlir