This commit is contained in:
@@ -0,0 +1,136 @@
|
||||
#include "mlir/Dialect/Affine/IR/AffineOps.h"
|
||||
#include "mlir/Dialect/Arith/IR/Arith.h"
|
||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||
#include "mlir/Dialect/Linalg/IR/Linalg.h"
|
||||
#include "mlir/Dialect/SCF/IR/SCF.h"
|
||||
#include "mlir/Dialect/Tensor/IR/Tensor.h"
|
||||
#include "mlir/Pass/Pass.h"
|
||||
#include "mlir/Transforms/DialectConversion.h"
|
||||
|
||||
#include "Conversion/ONNXToSpatial/Passes/Analyses/ONNXToSpatialVerifier.hpp"
|
||||
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/DebugDump.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Passes/Transforms/SpatialPlanLoweringPatterns.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/SpatialDataflowCsvExporter.hpp"
|
||||
#include "src/Accelerators/PIM/Passes/PIMPasses.h"
|
||||
#include "src/Dialect/ONNX/ONNXOps.hpp"
|
||||
|
||||
using namespace mlir;
|
||||
|
||||
namespace onnx_mlir {
|
||||
namespace {
|
||||
|
||||
struct LowerSpatialPlansPass final
|
||||
: PassWrapper<LowerSpatialPlansPass, OperationPass<ModuleOp>> {
|
||||
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LowerSpatialPlansPass)
|
||||
|
||||
StringRef getArgument() const override { return "lower-spatial-plans"; }
|
||||
StringRef getDescription() const override {
|
||||
return "Lower selected Spatial planning ops to low-level Spatial IR.";
|
||||
}
|
||||
|
||||
LowerSpatialPlansPass() = default;
|
||||
LowerSpatialPlansPass(const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options,
|
||||
spatial::SpatialDataflowExportStage exportStage)
|
||||
: target(target), planningOptions(options), exportStage(exportStage), hasTarget(true) {}
|
||||
|
||||
void runOnOperation() override {
|
||||
ModuleOp moduleOp = getOperation();
|
||||
if (!hasTarget) {
|
||||
moduleOp.emitError("Spatial plan lowering requires an injected SpatialTargetResources");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
auto entryFunc = getPimEntryFunc(moduleOp);
|
||||
if (failed(entryFunc)) {
|
||||
moduleOp.emitError("failed to locate the PIM entry function during LowerSpatialPlans");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
func::FuncOp funcOp = *entryFunc;
|
||||
auto verifyLogicalPhase = [&](StringRef stage) -> bool {
|
||||
if (succeeded(verifyLogicalSpatialGraphInvariants(funcOp)))
|
||||
return true;
|
||||
moduleOp.emitError() << "logical Spatial graph verification failed " << stage;
|
||||
signalPassFailure();
|
||||
return false;
|
||||
};
|
||||
|
||||
if (!verifyLogicalPhase("at the start of LowerSpatialPlans"))
|
||||
return;
|
||||
if (failed(verifySelectedSpatialLayouts(funcOp, target))) {
|
||||
moduleOp.emitError("selected Spatial layout verification failed");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
MLIRContext* ctx = moduleOp.getContext();
|
||||
RewritePatternSet patterns(ctx);
|
||||
populateSpatialPlanLoweringPatterns(patterns, ctx, target, planningOptions);
|
||||
|
||||
ConversionTarget conversionTarget(*ctx);
|
||||
conversionTarget.addLegalDialect<spatial::SpatialDialect,
|
||||
tensor::TensorDialect,
|
||||
linalg::LinalgDialect,
|
||||
affine::AffineDialect,
|
||||
arith::ArithDialect,
|
||||
scf::SCFDialect,
|
||||
func::FuncDialect>();
|
||||
conversionTarget.addIllegalDialect<ONNXDialect>();
|
||||
conversionTarget.addLegalOp<ONNXEntryPointOp>();
|
||||
conversionTarget.addIllegalOp<spatial::SpatConv2DPlanOp,
|
||||
spatial::SpatFlattenPlanOp,
|
||||
spatial::SpatReluPlanOp,
|
||||
spatial::SpatSiluPlanOp,
|
||||
spatial::SpatResizeNearestPlanOp,
|
||||
spatial::SpatMaxPool2DPlanOp,
|
||||
spatial::SpatGlobalAveragePoolPlanOp,
|
||||
spatial::SpatBiasAddPlanOp,
|
||||
spatial::SpatAddPlanOp,
|
||||
spatial::SpatConcatPlanOp,
|
||||
spatial::SpatMaterializeLayoutOp>();
|
||||
conversionTarget.addDynamicallyLegalOp<spatial::SpatBlueprintOp>(
|
||||
[](spatial::SpatBlueprintOp blueprint) {
|
||||
return spatial::isFragmentAssembly(blueprint.getMode());
|
||||
});
|
||||
|
||||
if (failed(applyFullConversion(funcOp, conversionTarget,
|
||||
std::move(patterns)))) {
|
||||
moduleOp.emitError("failed to lower Spatial plans and layout materialization");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
dumpModule(moduleOp, "spatial1_graph");
|
||||
if (spatial::shouldExportSpatialDataflowStage(
|
||||
exportStage, spatial::SpatialDataflowExportStage::Spatial1)
|
||||
&& failed(spatial::exportSpatialDataflowCsvGraph(funcOp, "spatial1_graph"))) {
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
verifyLogicalPhase("at the end of LowerSpatialPlans");
|
||||
}
|
||||
|
||||
spatial::SpatialTargetResources target;
|
||||
ONNXToSpatialPlanningOptions planningOptions;
|
||||
spatial::SpatialDataflowExportStage exportStage = spatial::SpatialDataflowExportStage::None;
|
||||
bool hasTarget = false;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
std::unique_ptr<Pass> createLowerSpatialPlansPass() {
|
||||
return std::make_unique<LowerSpatialPlansPass>();
|
||||
}
|
||||
|
||||
std::unique_ptr<Pass> createLowerSpatialPlansPass(
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options,
|
||||
spatial::SpatialDataflowExportStage exportStage) {
|
||||
return std::make_unique<LowerSpatialPlansPass>(target, options, exportStage);
|
||||
}
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -0,0 +1,281 @@
|
||||
#include "mlir/Dialect/Affine/IR/AffineOps.h"
|
||||
#include "mlir/Dialect/Arith/IR/Arith.h"
|
||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||
#include "mlir/Dialect/Linalg/IR/Linalg.h"
|
||||
#include "mlir/Dialect/SCF/IR/SCF.h"
|
||||
#include "mlir/Dialect/Tensor/IR/Tensor.h"
|
||||
#include "mlir/IR/IRMapping.h"
|
||||
#include "mlir/Pass/Pass.h"
|
||||
#include "mlir/Pass/PassManager.h"
|
||||
#include "mlir/Transforms/Passes.h"
|
||||
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
|
||||
|
||||
#include "llvm/ADT/SmallVector.h"
|
||||
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp"
|
||||
#include "Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/ComputeRegionBuilder.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/CompileTime.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Passes/Analyses/ONNXToSpatialVerifier.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
#include "src/Dialect/ONNX/ONNXOps.hpp"
|
||||
|
||||
using namespace mlir;
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
namespace {
|
||||
|
||||
struct ONNXToSpatialPass : PassWrapper<ONNXToSpatialPass, OperationPass<ModuleOp>> {
|
||||
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ONNXToSpatialPass)
|
||||
StringRef getArgument() const override { return "convert-onnx-to-spatial"; }
|
||||
StringRef getDescription() const override { return "Lower ONNX ops to Spatial ops."; }
|
||||
|
||||
ONNXToSpatialPass() = default;
|
||||
ONNXToSpatialPass(const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options)
|
||||
: target(target), planningOptions(options), hasTarget(true) {}
|
||||
ONNXToSpatialPass(const ONNXToSpatialPass& pass)
|
||||
: target(pass.target), planningOptions(pass.planningOptions), hasTarget(pass.hasTarget) {}
|
||||
|
||||
void runOnOperation() override;
|
||||
|
||||
spatial::SpatialTargetResources target;
|
||||
ONNXToSpatialPlanningOptions planningOptions;
|
||||
bool hasTarget = false;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
static void populateEmptyFunction(func::FuncOp funcOp) {
|
||||
IRRewriter rewriter(funcOp.getContext());
|
||||
IRMapping mapper;
|
||||
SmallVector<spatial::SpatGraphCompute> computes(funcOp.getOps<spatial::SpatGraphCompute>());
|
||||
SmallVector<spatial::SpatGraphComputeBatch> computeBatches(funcOp.getOps<spatial::SpatGraphComputeBatch>());
|
||||
SmallVector<spatial::SpatConv2DPlanOp> convPlans(funcOp.getOps<spatial::SpatConv2DPlanOp>());
|
||||
SmallVector<spatial::SpatBiasAddPlanOp> biasAddPlans(funcOp.getOps<spatial::SpatBiasAddPlanOp>());
|
||||
SmallVector<spatial::SpatAddPlanOp> addPlans(funcOp.getOps<spatial::SpatAddPlanOp>());
|
||||
SmallVector<spatial::SpatConcatPlanOp> concatPlans(funcOp.getOps<spatial::SpatConcatPlanOp>());
|
||||
SmallVector<spatial::SpatFlattenPlanOp> flattenPlans(funcOp.getOps<spatial::SpatFlattenPlanOp>());
|
||||
SmallVector<spatial::SpatReluPlanOp> reluPlans(funcOp.getOps<spatial::SpatReluPlanOp>());
|
||||
SmallVector<spatial::SpatSiluPlanOp> siluPlans(funcOp.getOps<spatial::SpatSiluPlanOp>());
|
||||
SmallVector<spatial::SpatResizeNearestPlanOp> resizePlans(
|
||||
funcOp.getOps<spatial::SpatResizeNearestPlanOp>());
|
||||
SmallVector<spatial::SpatMaxPool2DPlanOp> maxPoolPlans(funcOp.getOps<spatial::SpatMaxPool2DPlanOp>());
|
||||
SmallVector<spatial::SpatGlobalAveragePoolPlanOp> globalAveragePoolPlans(
|
||||
funcOp.getOps<spatial::SpatGlobalAveragePoolPlanOp>());
|
||||
SmallVector<spatial::SpatBlueprintOp> blueprints(funcOp.getOps<spatial::SpatBlueprintOp>());
|
||||
SmallVector<spatial::SpatMaterializeLayoutOp> materializers(funcOp.getOps<spatial::SpatMaterializeLayoutOp>());
|
||||
if (!computes.empty() || !computeBatches.empty() || !convPlans.empty() || !biasAddPlans.empty() || !addPlans.empty()
|
||||
|| !concatPlans.empty() || !flattenPlans.empty() || !reluPlans.empty() || !siluPlans.empty() || !resizePlans.empty()
|
||||
|| !maxPoolPlans.empty() || !blueprints.empty()
|
||||
|| !globalAveragePoolPlans.empty() || !materializers.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto returnOp = cast<func::ReturnOp>(funcOp.getFunctionBody().front().getTerminator());
|
||||
rewriter.setInsertionPoint(returnOp);
|
||||
|
||||
SmallVector<Type> sourceTypes;
|
||||
SmallVector<Location> sourceLocs;
|
||||
sourceTypes.reserve(funcOp.getNumArguments());
|
||||
sourceLocs.reserve(funcOp.getNumArguments());
|
||||
for (Value source : funcOp.getArguments()) {
|
||||
sourceTypes.push_back(source.getType());
|
||||
sourceLocs.push_back(source.getLoc());
|
||||
}
|
||||
|
||||
auto newCompute = createEmptySpatGraphCompute(
|
||||
rewriter, returnOp.getLoc(), returnOp.getOperandTypes(), {}, funcOp.getArguments(), sourceTypes, sourceLocs);
|
||||
auto* newBlock = &newCompute.getBody().front();
|
||||
for (auto [blockArg, computeArg] : llvm::zip(newBlock->getArguments(), newCompute.getOperands()))
|
||||
mapper.map(computeArg, blockArg);
|
||||
newCompute.getProperties().setOperandSegmentSizes({0, static_cast<int>(sourceTypes.size())});
|
||||
|
||||
rewriter.setInsertionPointToEnd(newBlock);
|
||||
for (Operation& op : funcOp.getOps())
|
||||
if (!isa<spatial::SpatGraphCompute, func::ReturnOp>(&op))
|
||||
rewriter.clone(op, mapper);
|
||||
|
||||
auto yield = spatial::SpatYieldOp::create(rewriter, funcOp.getLoc(), returnOp.getOperands());
|
||||
for (size_t i = 0; i < yield.getNumOperands(); ++i)
|
||||
yield.setOperand(i, mapper.lookupOrDefault(yield.getOperand(i)));
|
||||
|
||||
for (Operation& op : llvm::make_early_inc_range(funcOp.getOps()))
|
||||
if (!isa<spatial::SpatGraphCompute, func::ReturnOp>(&op)) {
|
||||
op.dropAllUses();
|
||||
rewriter.eraseOp(&op);
|
||||
}
|
||||
|
||||
for (auto [index, computeResult] : llvm::enumerate(newCompute.getResults()))
|
||||
returnOp.setOperand(index, computeResult);
|
||||
}
|
||||
|
||||
void ONNXToSpatialPass::runOnOperation() {
|
||||
ModuleOp moduleOp = getOperation();
|
||||
if (!hasTarget) {
|
||||
moduleOp.emitError("ONNX-to-Spatial lowering requires an injected SpatialTargetResources");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
MLIRContext* ctx = &getContext();
|
||||
|
||||
ConversionTarget preTarget(*ctx);
|
||||
preTarget.addLegalDialect<spatial::SpatialDialect,
|
||||
ONNXDialect,
|
||||
linalg::LinalgDialect,
|
||||
tensor::TensorDialect,
|
||||
affine::AffineDialect,
|
||||
arith::ArithDialect,
|
||||
scf::SCFDialect>();
|
||||
preTarget.addIllegalOp<ONNXConstantOp>();
|
||||
|
||||
RewritePatternSet prePatterns(ctx);
|
||||
populatePrePatterns(prePatterns, ctx);
|
||||
if (failed(applyPartialConversion(moduleOp, preTarget, std::move(prePatterns)))) {
|
||||
moduleOp.emitError("failed to apply ONNX-to-Spatial pre-rewrites");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
RewritePatternSet matmulPatterns(ctx);
|
||||
populateMatMulFusionPatterns(matmulPatterns, ctx, target);
|
||||
if (failed(applyPatternsGreedily(moduleOp, std::move(matmulPatterns)))) {
|
||||
moduleOp.emitError("failed to lower MatMul before producer conversion");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
RewritePatternSet fusionPatterns(ctx);
|
||||
populateElementwiseFusionPatterns(fusionPatterns, ctx);
|
||||
if (failed(applyPatternsGreedily(moduleOp, std::move(fusionPatterns)))) {
|
||||
moduleOp.emitError("failed to fuse layout-aware ONNX elementwise patterns");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
auto entryFunc = getPimEntryFunc(moduleOp);
|
||||
if (failed(entryFunc)) {
|
||||
moduleOp.emitError("failed to locate the PIM entry function during ONNX-to-Spatial lowering");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
ConversionTarget target(*ctx);
|
||||
target.addLegalDialect<spatial::SpatialDialect,
|
||||
ONNXDialect,
|
||||
linalg::LinalgDialect,
|
||||
tensor::TensorDialect,
|
||||
affine::AffineDialect,
|
||||
arith::ArithDialect,
|
||||
scf::SCFDialect>();
|
||||
target.addIllegalOp<ONNXMatMulOp>();
|
||||
target.addIllegalOp<ONNXTransposeOp>();
|
||||
target.addIllegalOp<ONNXAddOp>();
|
||||
target.addIllegalOp<ONNXSubOp>();
|
||||
target.addIllegalOp<ONNXDivOp>();
|
||||
target.addIllegalOp<ONNXMulOp>();
|
||||
target.addIllegalOp<ONNXGemmOp>();
|
||||
target.addIllegalOp<ONNXConvOp>();
|
||||
target.addIllegalOp<ONNXMaxPoolSingleOutOp>();
|
||||
target.addIllegalOp<ONNXAveragePoolOp>();
|
||||
target.addIllegalOp<ONNXReluOp>();
|
||||
target.addIllegalOp<ONNXSigmoidOp>();
|
||||
target.addIllegalOp<ONNXSoftmaxOp>();
|
||||
target.addIllegalOp<ONNXConcatOp>();
|
||||
target.addIllegalOp<ONNXFlattenOp>();
|
||||
target.addIllegalOp<ONNXGatherOp>();
|
||||
target.addIllegalOp<ONNXReshapeOp>();
|
||||
target.addIllegalOp<ONNXResizeOp>();
|
||||
target.addIllegalOp<ONNXSliceOp>();
|
||||
target.addIllegalOp<ONNXLRNOp>();
|
||||
target.addIllegalOp<ONNXReduceMeanOp>();
|
||||
target.addIllegalOp<ONNXReduceMeanV13Op>();
|
||||
target.addIllegalOp<ONNXSplitOp>();
|
||||
|
||||
RewritePatternSet conversionPatterns(ctx);
|
||||
populateConversionPatterns(conversionPatterns, ctx, this->target, planningOptions);
|
||||
if (failed(applyPartialConversion(moduleOp, target, std::move(conversionPatterns)))) {
|
||||
moduleOp.emitError("failed to convert required ONNX ops to Spatial ops");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
if (failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
|
||||
moduleOp.emitError("logical Spatial graph verification failed after ONNX conversion");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
ConversionTarget earlyPostTarget(*ctx);
|
||||
earlyPostTarget.addLegalDialect<spatial::SpatialDialect,
|
||||
ONNXDialect,
|
||||
linalg::LinalgDialect,
|
||||
tensor::TensorDialect,
|
||||
affine::AffineDialect,
|
||||
arith::ArithDialect,
|
||||
scf::SCFDialect>();
|
||||
|
||||
annotateWeightsConstants(*entryFunc);
|
||||
|
||||
if (failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
|
||||
moduleOp.emitError("logical Spatial graph verification failed after weight annotation");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
ConversionTarget postTarget(*ctx);
|
||||
postTarget.addLegalDialect<spatial::SpatialDialect,
|
||||
ONNXDialect,
|
||||
linalg::LinalgDialect,
|
||||
tensor::TensorDialect,
|
||||
affine::AffineDialect,
|
||||
arith::ArithDialect,
|
||||
scf::SCFDialect>();
|
||||
postTarget.addDynamicallyLegalOp<spatial::SpatGraphCompute>(
|
||||
[](spatial::SpatGraphCompute computeOp) { return !requiresPostRewrite(computeOp); });
|
||||
postTarget.addDynamicallyLegalOp<spatial::SpatGraphComputeBatch>(
|
||||
[](spatial::SpatGraphComputeBatch computeOp) { return !requiresPostRewrite(computeOp); });
|
||||
|
||||
if (failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
|
||||
moduleOp.emitError("logical Spatial graph verification failed before post rewrites");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
RewritePatternSet postPatterns(ctx);
|
||||
populatePostPatterns(postPatterns, ctx);
|
||||
if (failed(applyPartialConversion(*entryFunc, postTarget, std::move(postPatterns)))) {
|
||||
moduleOp.emitError("failed to normalize weight-like Spatial compute operands before Spatial-to-PIM lowering");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
populateEmptyFunction(*entryFunc);
|
||||
|
||||
PassManager canonicalizationPM(ctx);
|
||||
canonicalizationPM.addPass(createCanonicalizerPass());
|
||||
if (failed(canonicalizationPM.run(moduleOp)))
|
||||
moduleOp.emitWarning("failed to run ONNXToSpatial canonicalization; continuing");
|
||||
|
||||
dumpModule(moduleOp, "spatial0");
|
||||
|
||||
if (failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
|
||||
moduleOp.emitError("logical Spatial graph verification failed after ONNX-to-Spatial");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
if (failed(verifyONNXToSpatial(*entryFunc))) {
|
||||
moduleOp.emitError("ONNX-to-Spatial host legality verification failed");
|
||||
signalPassFailure();
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Pass> createONNXToSpatialPass() { return std::make_unique<ONNXToSpatialPass>(); }
|
||||
|
||||
std::unique_ptr<Pass> createONNXToSpatialPass(
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options) {
|
||||
return std::make_unique<ONNXToSpatialPass>(target, options);
|
||||
}
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -0,0 +1,92 @@
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
|
||||
#include "mlir/IR/PatternMatch.h"
|
||||
#include "mlir/Support/LogicalResult.h"
|
||||
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
struct RowStripPhysicalValue;
|
||||
struct ONNXToSpatialPlanningOptions;
|
||||
|
||||
inline spatial::PhysicalLayout getSpatialPlanOperandLayout(mlir::Value value) {
|
||||
if (auto materialize = value.getDefiningOp<spatial::SpatMaterializeLayoutOp>())
|
||||
return materialize.getTargetPhysicalLayout();
|
||||
if (auto blueprint = value.getDefiningOp<spatial::SpatBlueprintOp>())
|
||||
return blueprint.getPhysicalLayout();
|
||||
if (mlir::Operation* producer = value.getDefiningOp())
|
||||
if (auto selected = spatial::getSelectedPhysicalLayout(producer))
|
||||
return *selected;
|
||||
return spatial::PhysicalLayout::DenseNCHW;
|
||||
}
|
||||
|
||||
mlir::FailureOr<mlir::Value>
|
||||
lowerDenseFlattenPlan(spatial::SpatFlattenPlanOp planOp,
|
||||
mlir::Value input,
|
||||
mlir::PatternRewriter& rewriter);
|
||||
|
||||
mlir::FailureOr<mlir::Value>
|
||||
lowerSelectedConv2DPlan(spatial::SpatConv2DPlanOp planOp,
|
||||
mlir::Value input,
|
||||
mlir::Value weight,
|
||||
mlir::Value bias,
|
||||
std::optional<mlir::Value> rowStripInput,
|
||||
bool emitRowStripLayout,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options,
|
||||
mlir::PatternRewriter& rewriter);
|
||||
|
||||
mlir::LogicalResult canLowerConvPlanToRowStrip(spatial::SpatConv2DPlanOp planOp,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions* options = nullptr);
|
||||
mlir::LogicalResult canConsumeAndProduceRowStrip(spatial::SpatConv2DPlanOp planOp,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions* options = nullptr);
|
||||
|
||||
mlir::LogicalResult canLowerResizeNearestPlanToRowStrip(
|
||||
spatial::SpatResizeNearestPlanOp planOp, const spatial::SpatialTargetResources& target);
|
||||
|
||||
mlir::FailureOr<mlir::Value> lowerSelectedResizeNearestPlan(
|
||||
spatial::SpatResizeNearestPlanOp planOp,
|
||||
mlir::Value input,
|
||||
std::optional<mlir::Value> rowStripInput,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
mlir::PatternRewriter& rewriter);
|
||||
|
||||
mlir::LogicalResult canLowerMaxPoolPlanToRowStrip(spatial::SpatMaxPool2DPlanOp planOp,
|
||||
const spatial::SpatialTargetResources& target);
|
||||
|
||||
mlir::FailureOr<mlir::Value>
|
||||
lowerDenseMaxPool2DPlan(spatial::SpatMaxPool2DPlanOp planOp,
|
||||
mlir::Value input,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
mlir::PatternRewriter& rewriter);
|
||||
|
||||
mlir::FailureOr<mlir::Value>
|
||||
lowerSelectedMaxPool2DPlan(spatial::SpatMaxPool2DPlanOp planOp,
|
||||
mlir::Value input,
|
||||
std::optional<mlir::Value> rowStripInput,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
mlir::PatternRewriter& rewriter);
|
||||
|
||||
mlir::LogicalResult
|
||||
canLowerGlobalAveragePoolPlanToRowStrip(spatial::SpatGlobalAveragePoolPlanOp planOp,
|
||||
const spatial::SpatialTargetResources& target);
|
||||
|
||||
mlir::FailureOr<mlir::Value>
|
||||
lowerDenseGlobalAveragePoolPlan(spatial::SpatGlobalAveragePoolPlanOp planOp,
|
||||
mlir::Value input,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
mlir::PatternRewriter& rewriter);
|
||||
|
||||
mlir::FailureOr<mlir::Value>
|
||||
lowerSelectedGlobalAveragePoolPlan(spatial::SpatGlobalAveragePoolPlanOp planOp,
|
||||
mlir::Value input,
|
||||
std::optional<mlir::Value> rowStripInput,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
mlir::PatternRewriter& rewriter);
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -0,0 +1,318 @@
|
||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||
#include "mlir/IR/PatternMatch.h"
|
||||
#include "mlir/Pass/Pass.h"
|
||||
|
||||
#include "llvm/ADT/DenseMap.h"
|
||||
|
||||
#include "Conversion/ONNXToSpatial/Passes/Analyses/ONNXToSpatialVerifier.hpp"
|
||||
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Passes/Transforms/PlanLowering.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
#include "src/Accelerators/PIM/Passes/PIMPasses.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
|
||||
using namespace mlir;
|
||||
|
||||
namespace onnx_mlir {
|
||||
namespace {
|
||||
|
||||
struct SpatialLayoutSelection {
|
||||
llvm::DenseMap<Operation*, unsigned> selectedAlternative;
|
||||
llvm::DenseMap<Value, spatial::PhysicalLayout> resultLayouts;
|
||||
};
|
||||
|
||||
static spatial::PhysicalLayout getKnownLayout(
|
||||
const SpatialLayoutSelection& selection, Value value) {
|
||||
if (auto it = selection.resultLayouts.find(value); it != selection.resultLayouts.end())
|
||||
return it->second;
|
||||
return getSpatialPlanOperandLayout(value);
|
||||
}
|
||||
|
||||
static SmallVector<spatial::PhysicalLayout> getOperandLayouts(
|
||||
Operation* op, const SpatialLayoutSelection& selection) {
|
||||
SmallVector<spatial::PhysicalLayout> operandLayouts;
|
||||
operandLayouts.reserve(op->getNumOperands());
|
||||
for (Value operand : op->getOperands())
|
||||
operandLayouts.push_back(getKnownLayout(selection, operand));
|
||||
return operandLayouts;
|
||||
}
|
||||
|
||||
class SpatialLayoutAnalysis {
|
||||
public:
|
||||
SpatialLayoutAnalysis(func::FuncOp funcOp,
|
||||
const spatial::SpatialTargetResources& target)
|
||||
: funcOp(funcOp), target(target) {}
|
||||
|
||||
FailureOr<SpatialLayoutSelection> run() {
|
||||
SpatialLayoutSelection selection;
|
||||
SmallVector<Operation*> planOps;
|
||||
for (Operation& op : funcOp.getBody().front()) {
|
||||
if (!isa<spatial::SpatialLayoutCapabilityInterface>(&op))
|
||||
continue;
|
||||
planOps.push_back(&op);
|
||||
selection.resultLayouts[op.getResult(0)] = spatial::PhysicalLayout::DenseNCHW;
|
||||
selection.selectedAlternative[&op] = 0;
|
||||
}
|
||||
|
||||
const size_t maxRounds = 2 * planOps.size() + 1;
|
||||
for (size_t round = 0; round < maxRounds; ++round) {
|
||||
bool changed = false;
|
||||
SmallVector<Operation*> order(planOps);
|
||||
if (round % 2)
|
||||
std::reverse(order.begin(), order.end());
|
||||
|
||||
for (Operation* op : order) {
|
||||
FailureOr<SmallVector<spatial::LayoutAlternative>> alternatives =
|
||||
getAlternatives(op, selection);
|
||||
if (failed(alternatives))
|
||||
return failure();
|
||||
|
||||
unsigned currentIndex = selection.selectedAlternative.lookup(op);
|
||||
if (currentIndex >= alternatives->size())
|
||||
currentIndex = 0;
|
||||
if (selection.selectedAlternative.lookup(op) != currentIndex) {
|
||||
selection.selectedAlternative[op] = currentIndex;
|
||||
changed = true;
|
||||
}
|
||||
Value result = op->getResult(0);
|
||||
if (selection.resultLayouts.lookup(result) !=
|
||||
(*alternatives)[currentIndex].resultLayout) {
|
||||
selection.resultLayouts[result] = (*alternatives)[currentIndex].resultLayout;
|
||||
changed = true;
|
||||
}
|
||||
int64_t bestCost = alternativeCost(op, (*alternatives)[currentIndex], selection);
|
||||
unsigned bestIndex = currentIndex;
|
||||
for (auto [index, alternative] : llvm::enumerate(*alternatives)) {
|
||||
int64_t cost = alternativeCost(op, alternative, selection);
|
||||
if (cost < bestCost) {
|
||||
bestCost = cost;
|
||||
bestIndex = index;
|
||||
}
|
||||
}
|
||||
if (bestIndex == currentIndex)
|
||||
continue;
|
||||
selection.selectedAlternative[op] = bestIndex;
|
||||
selection.resultLayouts[result] = (*alternatives)[bestIndex].resultLayout;
|
||||
changed = true;
|
||||
}
|
||||
if (!changed)
|
||||
return selection;
|
||||
}
|
||||
funcOp.emitError("Spatial layout selection did not converge within its bounded iteration budget");
|
||||
return failure();
|
||||
}
|
||||
|
||||
FailureOr<SmallVector<spatial::LayoutAlternative>> getAlternatives(
|
||||
Operation* op, const SpatialLayoutSelection& selection) {
|
||||
auto capability = dyn_cast<spatial::SpatialLayoutCapabilityInterface>(op);
|
||||
if (!capability)
|
||||
return failure();
|
||||
SmallVector<spatial::LayoutAlternative> alternatives =
|
||||
capability.getLayoutAlternatives(target, getOperandLayouts(op, selection));
|
||||
if (alternatives.empty())
|
||||
return op->emitOpError("does not advertise a legal Spatial layout alternative"), failure();
|
||||
for (const spatial::LayoutAlternative& alternative : alternatives) {
|
||||
if (alternative.operandLayouts.size() != op->getNumOperands())
|
||||
return op->emitOpError("advertises a layout alternative with the wrong operand count"), failure();
|
||||
}
|
||||
if (llvm::any_of(op->getResult(0).getUses(), [](OpOperand& use) {
|
||||
return isa<func::ReturnOp>(use.getOwner());
|
||||
})
|
||||
&& llvm::none_of(alternatives, [](const spatial::LayoutAlternative& alternative) {
|
||||
return alternative.resultLayout == spatial::PhysicalLayout::DenseNCHW;
|
||||
}))
|
||||
return op->emitOpError("does not provide the required DenseNCHW function-result layout"), failure();
|
||||
return alternatives;
|
||||
}
|
||||
|
||||
private:
|
||||
int64_t alternativeCost(Operation* op,
|
||||
const spatial::LayoutAlternative& alternative,
|
||||
const SpatialLayoutSelection& selection) {
|
||||
if (llvm::any_of(op->getResult(0).getUses(), [](OpOperand& use) {
|
||||
return isa<func::ReturnOp>(use.getOwner());
|
||||
})
|
||||
&& alternative.resultLayout != spatial::PhysicalLayout::DenseNCHW)
|
||||
return std::numeric_limits<int64_t>::max() / 4;
|
||||
|
||||
int64_t cost = alternative.intrinsicCost;
|
||||
SmallVector<spatial::PhysicalLayout> operandLayouts = getOperandLayouts(op, selection);
|
||||
for (auto [actual, required] : llvm::zip(operandLayouts, alternative.operandLayouts))
|
||||
cost += actual != required;
|
||||
|
||||
Value result = op->getResult(0);
|
||||
for (OpOperand& use : result.getUses()) {
|
||||
auto user = dyn_cast<spatial::SpatialLayoutCapabilityInterface>(use.getOwner());
|
||||
if (!user)
|
||||
continue;
|
||||
SmallVector<spatial::PhysicalLayout> userOperandLayouts =
|
||||
getOperandLayouts(use.getOwner(), selection);
|
||||
for (auto [index, operand] : llvm::enumerate(use.getOwner()->getOperands()))
|
||||
if (operand == result)
|
||||
userOperandLayouts[index] = alternative.resultLayout;
|
||||
SmallVector<spatial::LayoutAlternative> userAlternatives =
|
||||
user.getLayoutAlternatives(target, userOperandLayouts);
|
||||
if (llvm::none_of(userAlternatives,
|
||||
[&](const spatial::LayoutAlternative& userAlternative) {
|
||||
return userAlternative.operandLayouts.size()
|
||||
== use.getOwner()->getNumOperands()
|
||||
&& userAlternative.operandLayouts[use.getOperandNumber()]
|
||||
== alternative.resultLayout;
|
||||
}))
|
||||
++cost;
|
||||
}
|
||||
return cost;
|
||||
}
|
||||
|
||||
func::FuncOp funcOp;
|
||||
const spatial::SpatialTargetResources& target;
|
||||
};
|
||||
|
||||
static LogicalResult materializeMismatchedUses(
|
||||
IRRewriter& rewriter, const SpatialLayoutSelection& selection,
|
||||
Operation* op, SpatialLayoutAnalysis& analysis) {
|
||||
Value value = op->getResult(0);
|
||||
spatial::PhysicalLayout sourceLayout = getKnownLayout(selection, value);
|
||||
SmallVector<std::pair<OpOperand*, spatial::PhysicalLayout>> mismatches;
|
||||
for (OpOperand& use : value.getUses()) {
|
||||
Operation* userOp = use.getOwner();
|
||||
auto capability = dyn_cast<spatial::SpatialLayoutCapabilityInterface>(userOp);
|
||||
if (!capability) {
|
||||
if (isa<func::ReturnOp>(userOp) || sourceLayout == spatial::PhysicalLayout::DenseNCHW)
|
||||
continue;
|
||||
mismatches.push_back({&use, spatial::PhysicalLayout::DenseNCHW});
|
||||
continue;
|
||||
}
|
||||
FailureOr<SmallVector<spatial::LayoutAlternative>> alternatives =
|
||||
analysis.getAlternatives(userOp, selection);
|
||||
if (failed(alternatives))
|
||||
return failure();
|
||||
unsigned selectedIndex = selection.selectedAlternative.lookup(userOp);
|
||||
if (selectedIndex >= alternatives->size())
|
||||
return userOp->emitOpError()
|
||||
<< "has no selected Spatial layout alternative (index " << selectedIndex
|
||||
<< ", alternatives " << alternatives->size() << ")",
|
||||
failure();
|
||||
spatial::PhysicalLayout required =
|
||||
(*alternatives)[selectedIndex].operandLayouts[use.getOperandNumber()];
|
||||
if (required != sourceLayout)
|
||||
mismatches.push_back({&use, required});
|
||||
}
|
||||
|
||||
for (auto [use, required] : mismatches) {
|
||||
Operation* userOp = use->getOwner();
|
||||
rewriter.setInsertionPoint(userOp);
|
||||
auto materialized = spatial::SpatMaterializeLayoutOp::create(
|
||||
rewriter, userOp->getLoc(), use->get().getType(), use->get(),
|
||||
spatial::LogicalLayoutAttr::get(
|
||||
rewriter.getContext(), spatial::LogicalLayout::NCHW),
|
||||
spatial::PhysicalLayoutAttr::get(rewriter.getContext(), sourceLayout),
|
||||
spatial::PhysicalLayoutAttr::get(rewriter.getContext(), required));
|
||||
use->set(materialized.getResult());
|
||||
}
|
||||
return success();
|
||||
}
|
||||
|
||||
static LogicalResult verifySelectedLayouts(
|
||||
const SpatialLayoutSelection& selection,
|
||||
ArrayRef<Operation*> planOps,
|
||||
SpatialLayoutAnalysis& analysis) {
|
||||
for (Operation* op : planOps) {
|
||||
auto selected = spatial::getSelectedPhysicalLayout(op);
|
||||
if (!selected)
|
||||
return op->emitOpError("requires a selected physical layout"), failure();
|
||||
FailureOr<SmallVector<spatial::LayoutAlternative>> alternatives =
|
||||
analysis.getAlternatives(op, selection);
|
||||
if (failed(alternatives))
|
||||
return failure();
|
||||
unsigned selectedIndex = selection.selectedAlternative.lookup(op);
|
||||
if (selectedIndex >= alternatives->size())
|
||||
return op->emitOpError()
|
||||
<< "has no selected Spatial layout alternative (index " << selectedIndex
|
||||
<< ", alternatives " << alternatives->size() << ")",
|
||||
failure();
|
||||
const spatial::LayoutAlternative& alternative = (*alternatives)[selectedIndex];
|
||||
if (alternative.resultLayout != *selected
|
||||
|| getOperandLayouts(op, selection) != alternative.operandLayouts)
|
||||
return op->emitOpError("selected physical layout does not satisfy its exact layout contract"), failure();
|
||||
}
|
||||
return success();
|
||||
}
|
||||
|
||||
struct SpatialLayoutPlanningPass final
|
||||
: PassWrapper<SpatialLayoutPlanningPass, OperationPass<ModuleOp>> {
|
||||
MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(SpatialLayoutPlanningPass)
|
||||
|
||||
StringRef getArgument() const override { return "spatial-layout-planning"; }
|
||||
StringRef getDescription() const override {
|
||||
return "Select Spatial layout alternatives and insert explicit reconciliation barriers.";
|
||||
}
|
||||
|
||||
SpatialLayoutPlanningPass() = default;
|
||||
explicit SpatialLayoutPlanningPass(const spatial::SpatialTargetResources& target)
|
||||
: target(target), hasTarget(true) {}
|
||||
|
||||
void runOnOperation() override {
|
||||
ModuleOp moduleOp = getOperation();
|
||||
if (!hasTarget) {
|
||||
moduleOp.emitError("Spatial layout planning requires an injected SpatialTargetResources");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
auto entryFunc = getPimEntryFunc(moduleOp);
|
||||
if (failed(entryFunc)) {
|
||||
moduleOp.emitError("failed to locate the PIM entry function during Spatial layout planning");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
func::FuncOp funcOp = *entryFunc;
|
||||
SpatialLayoutAnalysis analysis(funcOp, target);
|
||||
FailureOr<SpatialLayoutSelection> selection = analysis.run();
|
||||
if (failed(selection)) {
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
|
||||
SmallVector<Operation*> planOps;
|
||||
for (Operation& op : funcOp.getBody().front())
|
||||
if (isa<spatial::SpatialLayoutCapabilityInterface>(&op))
|
||||
planOps.push_back(&op);
|
||||
|
||||
IRRewriter rewriter(&getContext());
|
||||
for (Operation* op : planOps) {
|
||||
op->setAttr(spatial::kSelectedLayoutAttrName,
|
||||
spatial::PhysicalLayoutAttr::get(
|
||||
rewriter.getContext(), selection->resultLayouts.lookup(op->getResult(0))));
|
||||
if (failed(materializeMismatchedUses(
|
||||
rewriter, *selection, op, analysis))) {
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (failed(verifySelectedLayouts(*selection, planOps, analysis))
|
||||
|| failed(verifyLogicalSpatialGraphInvariants(*entryFunc))) {
|
||||
moduleOp.emitError("Spatial layout planning verification failed");
|
||||
signalPassFailure();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
spatial::SpatialTargetResources target;
|
||||
bool hasTarget = false;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
std::unique_ptr<Pass> createSpatialLayoutPlanningPass() {
|
||||
return std::make_unique<SpatialLayoutPlanningPass>();
|
||||
}
|
||||
|
||||
std::unique_ptr<Pass> createSpatialLayoutPlanningPass(
|
||||
const spatial::SpatialTargetResources& target) {
|
||||
return std::make_unique<SpatialLayoutPlanningPass>(target);
|
||||
}
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -0,0 +1,845 @@
|
||||
#include "mlir/Dialect/Affine/IR/AffineOps.h"
|
||||
#include "mlir/Dialect/Arith/IR/Arith.h"
|
||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||
#include "mlir/Dialect/Linalg/IR/Linalg.h"
|
||||
#include "mlir/Dialect/SCF/IR/SCF.h"
|
||||
#include "mlir/Dialect/Tensor/IR/Tensor.h"
|
||||
#include "mlir/Transforms/DialectConversion.h"
|
||||
|
||||
#include "Conversion/ONNXToSpatial/Passes/Analyses/ONNXToSpatialVerifier.hpp"
|
||||
#include "src/Accelerators/PIM/Common/PimCommon.hpp"
|
||||
#include "src/Accelerators/PIM/Common/Support/DebugDump.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/BiasAddUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/Common.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/MatrixProductLowering.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/RowStripLayoutUtils.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Patterns.hpp"
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Passes/Transforms/PlanLowering.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/SpatialDataflowCsvExporter.hpp"
|
||||
#include "src/Accelerators/PIM/Passes/PIMPasses.h"
|
||||
|
||||
using namespace mlir;
|
||||
|
||||
namespace onnx_mlir {
|
||||
namespace {
|
||||
|
||||
static FailureOr<RowStripPhysicalValue> getRowStripValue(Value value) {
|
||||
return getRowStripPhysicalValue(value);
|
||||
}
|
||||
|
||||
static FailureOr<Value> publishRowStripValue(Operation* planOp,
|
||||
Value storage,
|
||||
PatternRewriter& rewriter) {
|
||||
auto logicalType = dyn_cast<RankedTensorType>(planOp->getResult(0).getType());
|
||||
if (!logicalType)
|
||||
return planOp->emitOpError("requires ranked logical output type"), failure();
|
||||
FailureOr<RowStripPhysicalValue> value = describeRowStripPhysicalValue(storage, logicalType);
|
||||
if (failed(value))
|
||||
return planOp->emitOpError("lowering produced invalid row-strip physical storage"), failure();
|
||||
FailureOr<Value> blueprint = createRowStripStorageBlueprint(
|
||||
storage, logicalType, rewriter, planOp->getLoc());
|
||||
if (failed(blueprint))
|
||||
return planOp->emitOpError("failed to create row-strip storage Blueprint"), failure();
|
||||
rewriter.replaceOp(planOp, *blueprint);
|
||||
return *blueprint;
|
||||
}
|
||||
|
||||
static bool isRowStripSelected(Operation* op) {
|
||||
auto selected = spatial::getSelectedPhysicalLayout(op);
|
||||
return selected && *selected == spatial::PhysicalLayout::NHWCRowStrip;
|
||||
}
|
||||
|
||||
static bool isDenseSelected(Operation* op) {
|
||||
auto selected = spatial::getSelectedPhysicalLayout(op);
|
||||
return selected && *selected == spatial::PhysicalLayout::DenseNCHW;
|
||||
}
|
||||
|
||||
static spatial::PhysicalLayout getKnownPhysicalLayout(Value value) {
|
||||
return getSpatialPlanOperandLayout(value);
|
||||
}
|
||||
|
||||
static LogicalResult verifySelectedLayouts(
|
||||
func::FuncOp funcOp, const spatial::SpatialTargetResources& target) {
|
||||
LogicalResult result = success();
|
||||
funcOp.walk([&](Operation* op) {
|
||||
auto capability = dyn_cast<spatial::SpatialLayoutCapabilityInterface>(op);
|
||||
if (!capability)
|
||||
return;
|
||||
auto selected = spatial::getSelectedPhysicalLayout(op);
|
||||
if (!selected) {
|
||||
op->emitOpError("requires a selected physical layout from SpatialLayoutPlanning");
|
||||
result = failure();
|
||||
return;
|
||||
}
|
||||
if (*selected != spatial::PhysicalLayout::DenseNCHW
|
||||
&& *selected != spatial::PhysicalLayout::NHWCRowStrip) {
|
||||
op->emitOpError("has an unsupported selected physical layout");
|
||||
result = failure();
|
||||
return;
|
||||
}
|
||||
SmallVector<spatial::PhysicalLayout> operandLayouts;
|
||||
operandLayouts.reserve(op->getNumOperands());
|
||||
for (Value operand : op->getOperands())
|
||||
operandLayouts.push_back(getKnownPhysicalLayout(operand));
|
||||
auto alternatives = capability.getLayoutAlternatives(target, operandLayouts);
|
||||
if (llvm::none_of(alternatives, [&](const spatial::LayoutAlternative& alternative) {
|
||||
return alternative.resultLayout == *selected
|
||||
&& alternative.operandLayouts == operandLayouts;
|
||||
})) {
|
||||
op->emitOpError("selected physical layout is not lowerable for its explicit operand layouts");
|
||||
result = failure();
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
static FailureOr<Value>
|
||||
lowerRowStripRelu(const RowStripPhysicalValue& input, spatial::SpatReluPlanOp planOp, PatternRewriter& rewriter) {
|
||||
return applyRowStripRelu(input, rewriter, planOp.getLoc());
|
||||
}
|
||||
|
||||
static FailureOr<Value>
|
||||
lowerRowStripSilu(const RowStripPhysicalValue& input, spatial::SpatSiluPlanOp planOp, PatternRewriter& rewriter) {
|
||||
return applyRowStripSilu(input, rewriter, planOp.getLoc());
|
||||
}
|
||||
|
||||
static FailureOr<Value> lowerRowStripAdd(const RowStripPhysicalValue& lhs,
|
||||
const RowStripPhysicalValue& rhs,
|
||||
spatial::SpatAddPlanOp planOp,
|
||||
PatternRewriter& rewriter) {
|
||||
return applyRowStripAdd(lhs, rhs, rewriter, planOp.getLoc());
|
||||
}
|
||||
|
||||
static FailureOr<Value> lowerRowStripConcat(ArrayRef<RowStripPhysicalValue> inputs,
|
||||
spatial::SpatConcatPlanOp planOp,
|
||||
PatternRewriter& rewriter) {
|
||||
auto outputType = dyn_cast<RankedTensorType>(planOp.getOutput().getType());
|
||||
if (!outputType)
|
||||
return failure();
|
||||
return applyRowStripConcat(inputs, outputType, rewriter, planOp.getLoc());
|
||||
}
|
||||
|
||||
static FailureOr<Value>
|
||||
materializeRowStripToDense(const RowStripPhysicalValue& rowStripValue, Location loc, PatternRewriter& rewriter) {
|
||||
if (rowStripValue.logicalType.getRank() != 4 || !rowStripValue.logicalType.hasStaticShape())
|
||||
return failure();
|
||||
return createRowStripAssemblyBlueprint(rowStripValue, rewriter, loc);
|
||||
}
|
||||
|
||||
static FailureOr<Value> materializeDenseToRowStrip(
|
||||
Value input, RankedTensorType logicalType, Location loc, PatternRewriter& rewriter) {
|
||||
if (!logicalType || !logicalType.hasStaticShape() || logicalType.getRank() != 4
|
||||
|| logicalType.getDimSize(0) != 1)
|
||||
return failure();
|
||||
auto nhwcType = RankedTensorType::get(
|
||||
{1, logicalType.getDimSize(2), logicalType.getDimSize(3), logicalType.getDimSize(1)},
|
||||
logicalType.getElementType(), logicalType.getEncoding());
|
||||
auto rowsType = RankedTensorType::get(
|
||||
{logicalType.getDimSize(2) * logicalType.getDimSize(3), logicalType.getDimSize(1)},
|
||||
logicalType.getElementType(), logicalType.getEncoding());
|
||||
auto rowsCompute = createSpatCompute<1>(
|
||||
rewriter, loc, rowsType, {}, input, [&](Value denseInput) {
|
||||
Value nhwc = createLinalgTranspose(
|
||||
denseInput, nhwcType, {0, 2, 3, 1}, rewriter, loc);
|
||||
Value rows = tensor::CollapseShapeOp::create(
|
||||
rewriter, loc, rowsType, nhwc,
|
||||
SmallVector<ReassociationIndices> {{0, 1, 2}, {3}});
|
||||
spatial::SpatYieldOp::create(rewriter, loc, rows);
|
||||
});
|
||||
Value rows = rowsCompute->getResult(0);
|
||||
FailureOr<Value> storage = createRowStripStorageFromRows(rows, logicalType, rewriter, loc);
|
||||
if (failed(storage))
|
||||
return failure();
|
||||
return createRowStripStorageBlueprint(*storage, logicalType, rewriter, loc);
|
||||
}
|
||||
|
||||
static FailureOr<Value> lowerDenseBatchBiasAdd(Value input, Value bias, RankedTensorType resultType,
|
||||
PatternRewriter& rewriter, Location loc) {
|
||||
auto producer = input.getDefiningOp<spatial::SpatGraphComputeBatch>();
|
||||
auto inputType = dyn_cast<RankedTensorType>(input.getType());
|
||||
auto biasType = dyn_cast<RankedTensorType>(bias.getType());
|
||||
if (!producer || !inputType || !biasType || !inputType.hasStaticShape() || !biasType.hasStaticShape()
|
||||
|| !resultType.hasStaticShape() || inputType.getDimSize(0) != producer.getLaneCount()
|
||||
|| biasType.getDimSize(0) != producer.getLaneCount() || resultType.getDimSize(0) != producer.getLaneCount())
|
||||
return failure();
|
||||
auto inputFragmentType = spatial::getGraphBatchFragmentType(inputType, producer.getLaneCount());
|
||||
auto outputFragmentType = spatial::getGraphBatchFragmentType(resultType, producer.getLaneCount());
|
||||
if (failed(inputFragmentType) || failed(outputFragmentType) || inputFragmentType->getRank() != biasType.getRank()
|
||||
|| inputFragmentType->getDimSize(0) != 1 || inputFragmentType->getShape().drop_front() != biasType.getShape().drop_front()
|
||||
|| inputFragmentType->getRank() != outputFragmentType->getRank() + 1)
|
||||
return failure();
|
||||
for (auto [inputDim, outputDim] : llvm::zip(inputFragmentType->getShape().drop_front(), outputFragmentType->getShape()))
|
||||
if (outputDim > inputDim)
|
||||
return failure();
|
||||
|
||||
auto batch = createSpatComputeBatch(rewriter, loc, TypeRange {resultType}, producer.getLaneCount(), {}, ValueRange {input, bias},
|
||||
[&](detail::SpatComputeBatchBodyArgs args) -> LogicalResult {
|
||||
FailureOr<Value> fragment = extractGraphBatchPhysicalFragment(rewriter, loc, args.inputs[0], args.lane, *inputFragmentType);
|
||||
if (failed(fragment))
|
||||
return failure();
|
||||
MixedSliceGeometry biasSlice;
|
||||
for (int64_t dim : inputFragmentType->getShape()) {
|
||||
biasSlice.offsets.push_back(biasSlice.offsets.empty() ? OpFoldResult(args.lane) : rewriter.getIndexAttr(0));
|
||||
biasSlice.sizes.push_back(rewriter.getIndexAttr(dim));
|
||||
biasSlice.strides.push_back(rewriter.getIndexAttr(1));
|
||||
}
|
||||
Value biasFragment = extractMixedSliceOrIdentity(rewriter, loc, args.inputs[1], *inputFragmentType, biasSlice);
|
||||
if (!biasFragment)
|
||||
return failure();
|
||||
Value added = spatial::SpatVAddOp::create(rewriter, loc, *inputFragmentType, *fragment, biasFragment);
|
||||
MixedSliceGeometry outputSlice;
|
||||
outputSlice.offsets.assign(inputFragmentType->getRank(), rewriter.getIndexAttr(0));
|
||||
outputSlice.sizes.push_back(rewriter.getIndexAttr(1));
|
||||
outputSlice.strides.assign(inputFragmentType->getRank(), rewriter.getIndexAttr(1));
|
||||
for (int64_t dim : outputFragmentType->getShape())
|
||||
outputSlice.sizes.push_back(rewriter.getIndexAttr(dim));
|
||||
Value output = extractMixedSliceOrIdentity(rewriter, loc, added, *outputFragmentType, outputSlice);
|
||||
if (!output)
|
||||
return failure();
|
||||
publishGraphBatchPhysicalFragment(rewriter, loc, output, args.outputs.front(), args.lane);
|
||||
return success();
|
||||
});
|
||||
if (failed(batch))
|
||||
return failure();
|
||||
return batch->getResult(0);
|
||||
}
|
||||
|
||||
struct LowerDenseReluPlan final : OpConversionPattern<spatial::SpatReluPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatReluPlanOp planOp,
|
||||
spatial::SpatReluPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
auto selected = spatial::getSelectedPhysicalLayout(planOp.getOperation());
|
||||
if (!selected || *selected != spatial::PhysicalLayout::DenseNCHW)
|
||||
return failure();
|
||||
|
||||
auto computeOp = createSpatCompute<1>(
|
||||
rewriter, planOp.getLoc(), planOp.getOutput().getType(), {}, adaptor.getInput(), [&](Value x) {
|
||||
auto relu = spatial::SpatReluOp::create(rewriter, planOp.getLoc(), planOp.getOutput().getType(), x);
|
||||
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), relu.getResult());
|
||||
});
|
||||
rewriter.replaceOp(planOp, computeOp.getResults());
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerDenseSiluPlan final : OpConversionPattern<spatial::SpatSiluPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatSiluPlanOp planOp,
|
||||
spatial::SpatSiluPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
auto selected = spatial::getSelectedPhysicalLayout(planOp.getOperation());
|
||||
if (!selected || *selected != spatial::PhysicalLayout::DenseNCHW)
|
||||
return failure();
|
||||
|
||||
auto computeOp = createSpatCompute<1>(
|
||||
rewriter, planOp.getLoc(), planOp.getOutput().getType(), {}, adaptor.getInput(), [&](Value x) {
|
||||
Value sigmoid = spatial::SpatSigmoidOp::create(
|
||||
rewriter, planOp.getLoc(), planOp.getOutput().getType(), x).getResult();
|
||||
Value silu = spatial::SpatVMulOp::create(
|
||||
rewriter, planOp.getLoc(), planOp.getOutput().getType(), x, sigmoid).getResult();
|
||||
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), silu);
|
||||
});
|
||||
rewriter.replaceOp(planOp, computeOp.getResults());
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerDenseResizePlan final : OpConversionPattern<spatial::SpatResizeNearestPlanOp> {
|
||||
explicit LowerDenseResizePlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
|
||||
: OpConversionPattern<spatial::SpatResizeNearestPlanOp>(ctx), target(target) {}
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatResizeNearestPlanOp planOp,
|
||||
spatial::SpatResizeNearestPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isDenseSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<Value> lowered = lowerSelectedResizeNearestPlan(
|
||||
planOp, adaptor.getInput(), std::nullopt, target, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected dense nearest Resize plan");
|
||||
rewriter.replaceOp(planOp, *lowered);
|
||||
return success();
|
||||
}
|
||||
|
||||
const spatial::SpatialTargetResources& target;
|
||||
};
|
||||
|
||||
struct LowerDenseBiasAddPlan final : OpConversionPattern<spatial::SpatBiasAddPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatBiasAddPlanOp planOp,
|
||||
spatial::SpatBiasAddPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isDenseSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
auto resultType = dyn_cast<RankedTensorType>(planOp.getOutput().getType());
|
||||
if (!resultType)
|
||||
return planOp.emitOpError("requires ranked output type");
|
||||
|
||||
FailureOr<Value> denseBias = materializeDenseBiasAddTensor(
|
||||
adaptor.getBias(), resultType, rewriter, planOp.getLoc());
|
||||
if (failed(denseBias))
|
||||
return planOp.emitOpError("failed to materialize dense Conv-style bias");
|
||||
if (adaptor.getInput().getDefiningOp<spatial::SpatGraphComputeBatch>()) {
|
||||
FailureOr<Value> lowered = lowerDenseBatchBiasAdd(
|
||||
adaptor.getInput(), *denseBias, resultType, rewriter, planOp.getLoc());
|
||||
if (succeeded(lowered)) {
|
||||
rewriter.replaceOp(planOp, *lowered);
|
||||
return success();
|
||||
}
|
||||
}
|
||||
auto computeOp = createSpatCompute<2>(
|
||||
rewriter,
|
||||
planOp.getLoc(),
|
||||
planOp.getOutput().getType(),
|
||||
{},
|
||||
ValueRange {adaptor.getInput(), *denseBias},
|
||||
[&](Value x, Value y) {
|
||||
auto added = spatial::SpatVAddOp::create(
|
||||
rewriter, planOp.getLoc(), planOp.getOutput().getType(), x, y);
|
||||
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), added.getResult());
|
||||
});
|
||||
rewriter.replaceOp(planOp, computeOp.getResults());
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerDenseAddPlan final : OpConversionPattern<spatial::SpatAddPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatAddPlanOp planOp,
|
||||
spatial::SpatAddPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isDenseSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
auto compute = createSpatCompute<2>(
|
||||
rewriter,
|
||||
planOp.getLoc(),
|
||||
planOp.getOutput().getType(),
|
||||
{},
|
||||
ValueRange {adaptor.getLhs(), adaptor.getRhs()},
|
||||
[&](Value lhsValue, Value rhsValue) {
|
||||
Value added = spatial::SpatVAddOp::create(
|
||||
rewriter, planOp.getLoc(), planOp.getOutput().getType(), lhsValue, rhsValue);
|
||||
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), added);
|
||||
});
|
||||
rewriter.replaceOp(planOp, compute.getResults());
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerDenseConcatPlan final : OpConversionPattern<spatial::SpatConcatPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatConcatPlanOp planOp,
|
||||
spatial::SpatConcatPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isDenseSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
auto compute = createSpatCompute(
|
||||
rewriter,
|
||||
planOp.getLoc(),
|
||||
TypeRange {planOp.getOutput().getType()},
|
||||
{},
|
||||
adaptor.getInputs(),
|
||||
[&](ValueRange values) {
|
||||
Value concatenated = spatial::SpatConcatOp::create(
|
||||
rewriter,
|
||||
planOp.getLoc(),
|
||||
planOp.getOutput().getType(),
|
||||
rewriter.getI64IntegerAttr(planOp.getAxis()),
|
||||
values);
|
||||
spatial::SpatYieldOp::create(rewriter, planOp.getLoc(), concatenated);
|
||||
});
|
||||
rewriter.replaceOp(planOp, compute.getResults());
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
static LogicalResult lowerAddPlan(spatial::SpatAddPlanOp planOp,
|
||||
Value lhsValue, Value rhsValue,
|
||||
PatternRewriter& rewriter) {
|
||||
FailureOr<RowStripPhysicalValue> lhs = getRowStripValue(lhsValue);
|
||||
FailureOr<RowStripPhysicalValue> rhs = getRowStripValue(rhsValue);
|
||||
if (isRowStripSelected(planOp.getOperation()) && failed(lhs)) {
|
||||
if (getKnownPhysicalLayout(lhsValue) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
return planOp.emitOpError("selected row-strip Add plan requires row-strip inputs");
|
||||
}
|
||||
if (isRowStripSelected(planOp.getOperation()) && failed(rhs)) {
|
||||
if (getKnownPhysicalLayout(rhsValue) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
return planOp.emitOpError("selected row-strip Add plan requires row-strip inputs");
|
||||
}
|
||||
if (isRowStripSelected(planOp.getOperation())) {
|
||||
rewriter.setInsertionPoint(planOp);
|
||||
FailureOr<Value> lowered = lowerRowStripAdd(*lhs, *rhs, planOp, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Spatial add plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
return planOp.emitOpError("dense Add plan was not lowered by the selected-plan patterns");
|
||||
}
|
||||
|
||||
static LogicalResult lowerConcatPlan(spatial::SpatConcatPlanOp planOp,
|
||||
ValueRange inputValues,
|
||||
PatternRewriter& rewriter) {
|
||||
SmallVector<RowStripPhysicalValue> inputs;
|
||||
for (Value input : inputValues) {
|
||||
FailureOr<RowStripPhysicalValue> physical = getRowStripValue(input);
|
||||
if (failed(physical)) {
|
||||
inputs.clear();
|
||||
break;
|
||||
}
|
||||
inputs.push_back(*physical);
|
||||
}
|
||||
if (isRowStripSelected(planOp.getOperation()) && inputs.size() != inputValues.size()) {
|
||||
if (llvm::any_of(inputValues, [](Value input) {
|
||||
return getKnownPhysicalLayout(input) == spatial::PhysicalLayout::NHWCRowStrip;
|
||||
}))
|
||||
return failure();
|
||||
return planOp.emitOpError("selected row-strip Concat plan requires row-strip inputs");
|
||||
}
|
||||
if (isRowStripSelected(planOp.getOperation())) {
|
||||
rewriter.setInsertionPoint(planOp);
|
||||
FailureOr<Value> lowered = lowerRowStripConcat(inputs, planOp, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Spatial concat plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
return planOp.emitOpError("dense Concat plan was not lowered by the selected-plan patterns");
|
||||
}
|
||||
|
||||
struct LowerSelectedConvPlan final : OpConversionPattern<spatial::SpatConv2DPlanOp> {
|
||||
explicit LowerSelectedConvPlan(MLIRContext* ctx,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options)
|
||||
: OpConversionPattern<spatial::SpatConv2DPlanOp>(ctx), target(target), options(options) {}
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatConv2DPlanOp planOp,
|
||||
spatial::SpatConv2DPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (isDenseSelected(planOp.getOperation())) {
|
||||
FailureOr<Value> lowered = lowerSelectedConv2DPlan(
|
||||
planOp, adaptor.getInput(), adaptor.getWeight(), adaptor.getBias(),
|
||||
std::nullopt, /*emitRowStripLayout=*/false, target, options, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected dense Spatial Conv plan");
|
||||
rewriter.replaceOp(planOp, *lowered);
|
||||
return success();
|
||||
}
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
|
||||
FailureOr<RowStripPhysicalValue> rowStripInput = getRowStripValue(adaptor.getInput());
|
||||
if (failed(rowStripInput)
|
||||
&& getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
std::optional<Value> physicalInput;
|
||||
if (succeeded(rowStripInput))
|
||||
physicalInput = rowStripInput->storage;
|
||||
FailureOr<Value> lowered = lowerSelectedConv2DPlan(
|
||||
planOp, adaptor.getInput(), adaptor.getWeight(), adaptor.getBias(),
|
||||
physicalInput, /*emitRowStripLayout=*/true, target, options, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Spatial Conv plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
|
||||
const spatial::SpatialTargetResources& target;
|
||||
const ONNXToSpatialPlanningOptions& options;
|
||||
};
|
||||
|
||||
struct LowerRowStripReluPlan final : OpConversionPattern<spatial::SpatReluPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatReluPlanOp planOp,
|
||||
spatial::SpatReluPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
|
||||
if (failed(input)) {
|
||||
if (getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
return planOp.emitOpError("selected row-strip ReLU plan requires a row-strip input");
|
||||
}
|
||||
FailureOr<Value> lowered = lowerRowStripRelu(*input, planOp, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Spatial ReLU plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerRowStripSiluPlan final : OpConversionPattern<spatial::SpatSiluPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatSiluPlanOp planOp,
|
||||
spatial::SpatSiluPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
|
||||
if (failed(input)) {
|
||||
if (getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
return planOp.emitOpError("selected row-strip SiLU plan requires a row-strip input");
|
||||
}
|
||||
FailureOr<Value> lowered = lowerRowStripSilu(*input, planOp, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Spatial SiLU plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerRowStripResizePlan final : OpConversionPattern<spatial::SpatResizeNearestPlanOp> {
|
||||
explicit LowerRowStripResizePlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
|
||||
: OpConversionPattern<spatial::SpatResizeNearestPlanOp>(ctx), target(target) {}
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatResizeNearestPlanOp planOp,
|
||||
spatial::SpatResizeNearestPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
|
||||
if (failed(input)) {
|
||||
if (getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
return planOp.emitOpError("selected row-strip Resize plan requires a row-strip input");
|
||||
}
|
||||
FailureOr<Value> lowered = lowerSelectedResizeNearestPlan(
|
||||
planOp, adaptor.getInput(), input->storage, target, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Resize plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
|
||||
const spatial::SpatialTargetResources& target;
|
||||
};
|
||||
|
||||
struct LowerDenseMaxPoolPlan final : OpConversionPattern<spatial::SpatMaxPool2DPlanOp> {
|
||||
explicit LowerDenseMaxPoolPlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
|
||||
: OpConversionPattern<spatial::SpatMaxPool2DPlanOp>(ctx), target(target) {}
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatMaxPool2DPlanOp planOp,
|
||||
spatial::SpatMaxPool2DPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isDenseSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<Value> lowered = lowerDenseMaxPool2DPlan(
|
||||
planOp, adaptor.getInput(), target, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected dense Spatial MaxPool plan");
|
||||
rewriter.replaceOp(planOp, *lowered);
|
||||
return success();
|
||||
}
|
||||
|
||||
const spatial::SpatialTargetResources& target;
|
||||
};
|
||||
|
||||
struct LowerRowStripMaxPoolPlan final : OpConversionPattern<spatial::SpatMaxPool2DPlanOp> {
|
||||
explicit LowerRowStripMaxPoolPlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
|
||||
: OpConversionPattern<spatial::SpatMaxPool2DPlanOp>(ctx), target(target) {}
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatMaxPool2DPlanOp planOp,
|
||||
spatial::SpatMaxPool2DPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
|
||||
if (failed(input)
|
||||
&& getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
std::optional<Value> physicalInput;
|
||||
if (succeeded(input))
|
||||
physicalInput = input->storage;
|
||||
FailureOr<Value> lowered = lowerSelectedMaxPool2DPlan(
|
||||
planOp, adaptor.getInput(), physicalInput, target, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Spatial MaxPool plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
|
||||
const spatial::SpatialTargetResources& target;
|
||||
};
|
||||
|
||||
struct LowerRowStripGlobalAveragePoolPlan
|
||||
final : OpConversionPattern<spatial::SpatGlobalAveragePoolPlanOp> {
|
||||
explicit LowerRowStripGlobalAveragePoolPlan(MLIRContext* ctx, const spatial::SpatialTargetResources& target)
|
||||
: OpConversionPattern<spatial::SpatGlobalAveragePoolPlanOp>(ctx), target(target) {}
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatGlobalAveragePoolPlanOp planOp,
|
||||
spatial::SpatGlobalAveragePoolPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
|
||||
if (failed(input)
|
||||
&& getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
std::optional<Value> physicalInput;
|
||||
if (succeeded(input))
|
||||
physicalInput = input->storage;
|
||||
FailureOr<Value> lowered = lowerSelectedGlobalAveragePoolPlan(
|
||||
planOp, adaptor.getInput(), physicalInput, target, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Spatial global AveragePool plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
|
||||
const spatial::SpatialTargetResources& target;
|
||||
};
|
||||
|
||||
struct LowerDenseGlobalAveragePoolPlan
|
||||
final : OpConversionPattern<spatial::SpatGlobalAveragePoolPlanOp> {
|
||||
explicit LowerDenseGlobalAveragePoolPlan(MLIRContext* ctx,
|
||||
const spatial::SpatialTargetResources& target)
|
||||
: OpConversionPattern<spatial::SpatGlobalAveragePoolPlanOp>(ctx), target(target) {}
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatGlobalAveragePoolPlanOp planOp,
|
||||
spatial::SpatGlobalAveragePoolPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isDenseSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<Value> lowered = lowerDenseGlobalAveragePoolPlan(
|
||||
planOp, adaptor.getInput(), target, rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected dense Spatial global AveragePool plan");
|
||||
rewriter.replaceOp(planOp, *lowered);
|
||||
return success();
|
||||
}
|
||||
|
||||
const spatial::SpatialTargetResources& target;
|
||||
};
|
||||
|
||||
struct LowerRowStripBiasAddPlan final : OpConversionPattern<spatial::SpatBiasAddPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatBiasAddPlanOp planOp,
|
||||
spatial::SpatBiasAddPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<RowStripPhysicalValue> input = getRowStripValue(adaptor.getInput());
|
||||
if (failed(input)) {
|
||||
if (getKnownPhysicalLayout(adaptor.getInput()) == spatial::PhysicalLayout::NHWCRowStrip)
|
||||
return failure();
|
||||
return planOp.emitOpError("selected row-strip bias_add plan requires a row-strip input");
|
||||
}
|
||||
FailureOr<Value> lowered = applyRowStripBiasAdd(
|
||||
*input, adaptor.getBias(), rewriter, planOp.getLoc());
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected row-strip Spatial bias_add plan");
|
||||
if (failed(publishRowStripValue(planOp, *lowered, rewriter)))
|
||||
return failure();
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerRowStripAddPlan final : OpConversionPattern<spatial::SpatAddPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatAddPlanOp planOp,
|
||||
spatial::SpatAddPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
return lowerAddPlan(planOp, adaptor.getLhs(), adaptor.getRhs(), rewriter);
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerRowStripConcatPlan final : OpConversionPattern<spatial::SpatConcatPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatConcatPlanOp planOp,
|
||||
spatial::SpatConcatPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isRowStripSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
return lowerConcatPlan(planOp, adaptor.getInputs(), rewriter);
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerMaterializeLayout final
|
||||
: OpConversionPattern<spatial::SpatMaterializeLayoutOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatMaterializeLayoutOp materializeOp,
|
||||
spatial::SpatMaterializeLayoutOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
auto source = materializeOp.getSourcePhysicalLayout();
|
||||
auto target = materializeOp.getTargetPhysicalLayout();
|
||||
if (source == spatial::PhysicalLayout::DenseNCHW
|
||||
&& target == spatial::PhysicalLayout::DenseNCHW) {
|
||||
rewriter.replaceOp(materializeOp, adaptor.getInput());
|
||||
return success();
|
||||
}
|
||||
if (source == spatial::PhysicalLayout::DenseNCHW
|
||||
&& target == spatial::PhysicalLayout::NHWCRowStrip) {
|
||||
auto logicalType = dyn_cast<RankedTensorType>(adaptor.getInput().getType());
|
||||
if (!logicalType)
|
||||
return materializeOp.emitOpError("requires a ranked dense input"), failure();
|
||||
FailureOr<Value> rowStrip = materializeDenseToRowStrip(
|
||||
adaptor.getInput(), logicalType, materializeOp.getLoc(), rewriter);
|
||||
if (failed(rowStrip))
|
||||
return materializeOp.emitOpError(
|
||||
"failed to materialize dense NCHW storage to row-strip layout"), failure();
|
||||
rewriter.replaceOp(materializeOp, *rowStrip);
|
||||
return success();
|
||||
}
|
||||
if (source != spatial::PhysicalLayout::NHWCRowStrip
|
||||
|| target != spatial::PhysicalLayout::DenseNCHW)
|
||||
return materializeOp.emitOpError(
|
||||
"unsupported Spatial layout materialization direction"), failure();
|
||||
auto inputType = dyn_cast<RankedTensorType>(adaptor.getInput().getType());
|
||||
if (!inputType)
|
||||
return materializeOp.emitOpError("requires a ranked row-strip input"), failure();
|
||||
FailureOr<RowStripPhysicalValue> rowStripValue =
|
||||
getRowStripValue(adaptor.getInput());
|
||||
if (failed(rowStripValue))
|
||||
return failure();
|
||||
FailureOr<Value> dense = materializeRowStripToDense(
|
||||
*rowStripValue, materializeOp.getLoc(), rewriter);
|
||||
if (failed(dense))
|
||||
return materializeOp.emitOpError(
|
||||
"failed to materialize row-strip storage to dense NCHW"), failure();
|
||||
rewriter.replaceOp(materializeOp, *dense);
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
struct LowerSelectedFlattenPlan final
|
||||
: OpConversionPattern<spatial::SpatFlattenPlanOp> {
|
||||
using OpConversionPattern::OpConversionPattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatFlattenPlanOp planOp,
|
||||
spatial::SpatFlattenPlanOpAdaptor adaptor,
|
||||
ConversionPatternRewriter& rewriter) const override {
|
||||
if (!isDenseSelected(planOp.getOperation()))
|
||||
return failure();
|
||||
FailureOr<RowStripPhysicalValue> rowStripInput = getRowStripValue(adaptor.getInput());
|
||||
if (succeeded(rowStripInput)) {
|
||||
if (failed(canLowerFlattenFromRowStrip(planOp, target))
|
||||
|| failed(lowerFlattenFromRowStrip(*rowStripInput, planOp, target, rewriter)))
|
||||
return planOp.emitOpError("failed to lower selected Spatial Flatten plan"), failure();
|
||||
return success();
|
||||
}
|
||||
FailureOr<Value> lowered = lowerDenseFlattenPlan(planOp, adaptor.getInput(), rewriter);
|
||||
if (failed(lowered))
|
||||
return planOp.emitOpError("failed to lower selected dense Spatial Flatten plan"), failure();
|
||||
rewriter.replaceOp(planOp, *lowered);
|
||||
return success();
|
||||
}
|
||||
|
||||
explicit LowerSelectedFlattenPlan(MLIRContext* context,
|
||||
const spatial::SpatialTargetResources& target)
|
||||
: OpConversionPattern<spatial::SpatFlattenPlanOp>(context), target(target) {}
|
||||
|
||||
const spatial::SpatialTargetResources& target;
|
||||
};
|
||||
|
||||
struct EraseDeadPhysicalViewBlueprint final
|
||||
: OpRewritePattern<spatial::SpatBlueprintOp> {
|
||||
using OpRewritePattern::OpRewritePattern;
|
||||
|
||||
LogicalResult matchAndRewrite(spatial::SpatBlueprintOp blueprint,
|
||||
PatternRewriter& rewriter) const override {
|
||||
if (!spatial::isPhysicalView(blueprint.getMode()) || !blueprint.use_empty())
|
||||
return failure();
|
||||
rewriter.eraseOp(blueprint);
|
||||
return success();
|
||||
}
|
||||
};
|
||||
|
||||
static void populateConvPlanLoweringPatterns(
|
||||
RewritePatternSet& patterns, MLIRContext* ctx,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options) {
|
||||
patterns.add<LowerSelectedConvPlan>(ctx, target, options);
|
||||
}
|
||||
|
||||
static void populateElementwisePlanLoweringPatterns(
|
||||
RewritePatternSet& patterns, MLIRContext* ctx) {
|
||||
patterns.add<LowerDenseReluPlan,
|
||||
LowerRowStripReluPlan,
|
||||
LowerDenseSiluPlan,
|
||||
LowerRowStripSiluPlan,
|
||||
LowerDenseBiasAddPlan,
|
||||
LowerRowStripBiasAddPlan,
|
||||
LowerDenseAddPlan,
|
||||
LowerRowStripAddPlan>(ctx);
|
||||
}
|
||||
|
||||
static void populatePoolPlanLoweringPatterns(
|
||||
RewritePatternSet& patterns, MLIRContext* ctx,
|
||||
const spatial::SpatialTargetResources& target) {
|
||||
patterns.add<LowerDenseMaxPoolPlan,
|
||||
LowerRowStripMaxPoolPlan,
|
||||
LowerDenseGlobalAveragePoolPlan,
|
||||
LowerRowStripGlobalAveragePoolPlan>(ctx, target);
|
||||
}
|
||||
|
||||
static void populateResizePlanLoweringPatterns(
|
||||
RewritePatternSet& patterns, MLIRContext* ctx,
|
||||
const spatial::SpatialTargetResources& target) {
|
||||
patterns.add<LowerDenseResizePlan, LowerRowStripResizePlan>(ctx, target);
|
||||
}
|
||||
|
||||
static void populateConcatPlanLoweringPatterns(
|
||||
RewritePatternSet& patterns, MLIRContext* ctx) {
|
||||
patterns.add<LowerDenseConcatPlan, LowerRowStripConcatPlan>(ctx);
|
||||
}
|
||||
|
||||
static void populateFlattenPlanLoweringPatterns(
|
||||
RewritePatternSet& patterns, MLIRContext* ctx,
|
||||
const spatial::SpatialTargetResources& target) {
|
||||
patterns.add<LowerSelectedFlattenPlan>(ctx, target);
|
||||
}
|
||||
|
||||
static void populateLayoutMaterializationPatterns(
|
||||
RewritePatternSet& patterns, MLIRContext* ctx) {
|
||||
patterns.add<LowerMaterializeLayout, EraseDeadPhysicalViewBlueprint>(ctx);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void populateSpatialPlanLoweringPatterns(
|
||||
RewritePatternSet& patterns, MLIRContext* ctx,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options) {
|
||||
populateConvPlanLoweringPatterns(patterns, ctx, target, options);
|
||||
populateElementwisePlanLoweringPatterns(patterns, ctx);
|
||||
populatePoolPlanLoweringPatterns(patterns, ctx, target);
|
||||
populateResizePlanLoweringPatterns(patterns, ctx, target);
|
||||
populateConcatPlanLoweringPatterns(patterns, ctx);
|
||||
populateFlattenPlanLoweringPatterns(patterns, ctx, target);
|
||||
populateLayoutMaterializationPatterns(patterns, ctx);
|
||||
}
|
||||
|
||||
LogicalResult verifySelectedSpatialLayouts(
|
||||
func::FuncOp funcOp, const spatial::SpatialTargetResources& target) {
|
||||
return verifySelectedLayouts(funcOp, target);
|
||||
}
|
||||
|
||||
} // namespace onnx_mlir
|
||||
@@ -0,0 +1,20 @@
|
||||
#pragma once
|
||||
|
||||
#include "mlir/Dialect/Func/IR/FuncOps.h"
|
||||
#include "mlir/IR/PatternMatch.h"
|
||||
#include "mlir/Support/LogicalResult.h"
|
||||
|
||||
#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp"
|
||||
#include "src/Accelerators/PIM/Dialect/Spatial/SpatialTargetResources.hpp"
|
||||
|
||||
namespace onnx_mlir {
|
||||
|
||||
void populateSpatialPlanLoweringPatterns(
|
||||
mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx,
|
||||
const spatial::SpatialTargetResources& target,
|
||||
const ONNXToSpatialPlanningOptions& options);
|
||||
|
||||
mlir::LogicalResult verifySelectedSpatialLayouts(
|
||||
mlir::func::FuncOp funcOp, const spatial::SpatialTargetResources& target);
|
||||
|
||||
} // namespace onnx_mlir
|
||||
Reference in New Issue
Block a user