From 742df111e301bdebc3e6967c1d67f418177e32f7 Mon Sep 17 00:00:00 2001 From: NiccoloN Date: Wed, 25 Mar 2026 11:03:03 +0100 Subject: [PATCH] add relu lowering add relu validation add spatial compute helper minor refactors --- src/PIM/Compiler/PimCompilerUtils.cpp | 8 +-- .../Conversion/ONNXToSpatial/CMakeLists.txt | 3 +- src/PIM/Conversion/ONNXToSpatial/Common.hpp | 46 +++++++++---- .../Conversion/ONNXToSpatial/ONNXToSpatial.td | 19 +++--- .../ONNXToSpatial/ONNXToSpatialPass.cpp | 33 ++++----- src/PIM/Conversion/ONNXToSpatial/Patterns.hpp | 12 ++-- .../ONNXToSpatial/Patterns/Math/Conv.cpp | 2 +- .../ONNXToSpatial/Patterns/Math/Gemm.cpp | 5 +- .../ONNXToSpatial/Patterns/NN/Pool.cpp | 2 +- .../ONNXToSpatial/Patterns/NN/Relu.cpp | 33 +++++++++ .../ONNXToSpatial/Patterns/Tensor/Concat.cpp | 5 +- .../ONNXToSpatial/Patterns/Tensor/Reshape.cpp | 2 +- .../Conversion/SpatialToPim/SpatialToPim.td | 17 +++-- src/PIM/Dialect/Pim/Pim.td | 23 ------- .../OpBufferizationInterfaces.cpp | 63 ++++++++++++++---- .../Bufferization/PimBufferization.td | 1 - .../Bufferization/PimBufferizationPass.cpp | 2 +- .../SpatialBufferizableOpInterface.cpp | 3 - src/PIM/Pass/PIMPasses.h | 8 +-- .../ConstantFolding/ConstantFoldingPass.cpp | 2 +- src/PIM/Pass/Pim/MaterializeConstantsPass.cpp | 2 +- src/PIM/Pass/Pim/VerificationPass.cpp | 2 +- src/PIM/PimAccelerator.cpp | 10 +-- validation/operations/README.md | 9 +++ validation/operations/gen_tests.py | 62 ++++++++++++++++- validation/operations/relu/4d/relu_4d.onnx | Bin 0 -> 91 bytes .../relu/after_conv/relu_after_conv.onnx | Bin 0 -> 434 bytes .../relu/after_gemm/relu_after_gemm.onnx | Bin 0 -> 8452 bytes .../operations/relu/basic/relu_basic.onnx | Bin 0 -> 78 bytes 29 files changed, 258 insertions(+), 116 deletions(-) create mode 100644 src/PIM/Conversion/ONNXToSpatial/Patterns/NN/Relu.cpp create mode 100644 validation/operations/relu/4d/relu_4d.onnx create mode 100644 validation/operations/relu/after_conv/relu_after_conv.onnx create mode 100644 validation/operations/relu/after_gemm/relu_after_gemm.onnx create mode 100644 validation/operations/relu/basic/relu_basic.onnx diff --git a/src/PIM/Compiler/PimCompilerUtils.cpp b/src/PIM/Compiler/PimCompilerUtils.cpp index 3ecc059..dd6b418 100644 --- a/src/PIM/Compiler/PimCompilerUtils.cpp +++ b/src/PIM/Compiler/PimCompilerUtils.cpp @@ -39,16 +39,16 @@ void addPassesPim(OwningOpRef& module, } if (pimEmissionTarget >= EmitPimBufferized) { - pm.addPass(createBufferizePimPass()); + pm.addPass(createPimBufferizationPass()); // pm.addPass(createCountInstructionPass()); pm.addPass(createMessagePass("Pim bufferized")); } if (pimEmissionTarget >= EmitPimCodegen) { - pm.addPass(createConstantFoldingPass()); + pm.addPass(createPimConstantFoldingPass()); pm.addPass(createMessagePass("Pim constants folded")); - pm.addPass(createMaterializeConstantsPass()); - pm.addPass(createVerificationPass()); + pm.addPass(createPimMaterializeConstantsPass()); + pm.addPass(createPimVerificationPass()); pm.addPass(createMessagePass("Pim verified")); pm.addPass(createEmitPimJsonPass()); // pm.addPass(createCountInstructionPass()); diff --git a/src/PIM/Conversion/ONNXToSpatial/CMakeLists.txt b/src/PIM/Conversion/ONNXToSpatial/CMakeLists.txt index 1e383c8..245ff6f 100644 --- a/src/PIM/Conversion/ONNXToSpatial/CMakeLists.txt +++ b/src/PIM/Conversion/ONNXToSpatial/CMakeLists.txt @@ -3,10 +3,11 @@ mlir_tablegen(ONNXToSpatial.hpp.inc -gen-rewriters "-I${ONNX_MLIR_SRC_ROOT}") add_public_tablegen_target(ONNXToSpatialIncGen) add_pim_library(OMONNXToSpatial - Patterns/Math/Gemm.cpp Patterns/Math/Conv.cpp + Patterns/Math/Gemm.cpp Patterns/Math/MatMul.cpp Patterns/NN/Pool.cpp + Patterns/NN/Relu.cpp Patterns/Tensor/Concat.cpp Patterns/Tensor/Reshape.cpp ONNXToSpatialPass.cpp diff --git a/src/PIM/Conversion/ONNXToSpatial/Common.hpp b/src/PIM/Conversion/ONNXToSpatial/Common.hpp index 2e2b801..864148d 100644 --- a/src/PIM/Conversion/ONNXToSpatial/Common.hpp +++ b/src/PIM/Conversion/ONNXToSpatial/Common.hpp @@ -1,15 +1,13 @@ #pragma once #include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/IR/Block.h" #include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/PatternMatch.h" +#include "mlir/IR/ValueRange.h" #include "mlir/Support/LLVM.h" #include "mlir/Transforms/DialectConversion.h" -#include "llvm/Support/LogicalResult.h" - #include -#include #include #include @@ -58,14 +56,6 @@ inline auto getFilterCount(const ShapedType& shapedType) { using HSliceId = size_t; using CoreId = size_t; -enum class MapOperations { - None, - ONNXSoftmaxOp, - ONNXReluOp, - ONNXLeakyReluOp, - ONNXExpOp -}; - template > constexpr C ceilIntegerDivide(A a, B b) { static_assert(std::is_integral_v, "A must be an integer type"); @@ -114,6 +104,38 @@ inline auto getTensorShape(mlir::Value tensor) { return mlir::cast(tensor.getType()).getShape(); } +namespace detail { + +template +void invokeWithBlockArgs(Fn&& fn, mlir::Block* block, std::index_sequence) { + std::forward(fn)(block->getArgument(Is)...); +} + +} // namespace detail + +template +spatial::SpatWeightedCompute createSpatCompute(mlir::ConversionPatternRewriter& rewriter, + mlir::Location loc, + mlir::TypeRange resultTypes, + mlir::ValueRange weights, + mlir::ValueRange inputs, + BodyFn&& body) { + assert(inputs.size() == NumInputs && "NumInputs must match the number of input values"); + auto computeOp = spatial::SpatWeightedCompute::create(rewriter, loc, resultTypes, weights, inputs); + + auto* block = new mlir::Block(); + for (mlir::Value input : inputs) + block->addArgument(input.getType(), loc); + + computeOp.getBody().push_back(block); + rewriter.setInsertionPointToStart(block); + + detail::invokeWithBlockArgs(std::forward(body), block, std::make_index_sequence {}); + + rewriter.setInsertionPointAfter(computeOp); + return computeOp; +} + llvm::SmallVector sliceTensor(const mlir::Value& tensorToSlice, size_t axis, int64_t sliceSize, diff --git a/src/PIM/Conversion/ONNXToSpatial/ONNXToSpatial.td b/src/PIM/Conversion/ONNXToSpatial/ONNXToSpatial.td index cb3401b..1789e45 100644 --- a/src/PIM/Conversion/ONNXToSpatial/ONNXToSpatial.td +++ b/src/PIM/Conversion/ONNXToSpatial/ONNXToSpatial.td @@ -8,7 +8,7 @@ include "src/Dialect/ONNX/ONNX.td" include "src/Accelerators/PIM/Dialect/Spatial/Spatial.td" #endif // OP_BASE -def onnxToArithConstantOp : Pat< +def onnxToArithConstant : Pat< (ONNXConstantOp $sparse_value, $value, $value_float, $value_floats, $value_int, $value_ints, $value_string, $value_strings), (Arith_ConstantOp $value) >; @@ -19,7 +19,7 @@ def IsRank2Result: Constraint< CPred<"cast($0.getType()).getRank() == 2">, "Result is rank 2">; -def matMulAddToGemmPattern : Pat< +def matMulAddToGemm : Pat< (ONNXAddOp (ONNXMatMulOp:$matmulres $A, $B), $C), (ONNXGemmOp $A, $B, $C, /* alpha = */ (NativeCodeCall<"$_builder.getF32FloatAttr(1)">), @@ -30,7 +30,7 @@ def matMulAddToGemmPattern : Pat< [(IsRank2Result $matmulres)] >; -def matMulToGemmPattern : Pat< +def matMulToGemm : Pat< (ONNXMatMulOp:$matmulres $A, $B), ( ONNXGemmOp $A, $B, @@ -45,14 +45,13 @@ def matMulToGemmPattern : Pat< // ONNXConvOp + ONNXAddOp to ONNXConvOp pattern -// This pattern is used to fuse an ONNXConvOp and an ONNXAddOp into a single -// ONNXConvOp with a bias. -def convAddToConvWithBiasPatternLeft : Pat< +// This pattern is used to fuse an ONNXConvOp and an ONNXAddOp into a single ONNXConvOp with a bias. +def convAddToConvWithBiasLeft : Pat< (ONNXAddOp $add_operand, (ONNXConvOp:$convres $x, $w, $bias, $auto_pad, $dilations, $group, $kernel_shape, $pad, $strides)), (ONNXConvOp $x, $w, $add_operand, $auto_pad, $dilations, $group, $kernel_shape, $pad, $strides) >; -def convAddToConvWithBiasPatternRight : Pat< +def convAddToConvWithBiasRight : Pat< (ONNXAddOp (ONNXConvOp:$convres $x, $w, $bias, $auto_pad, $dilations, $group, $kernel_shape, $pad, $strides), $add_operand), (ONNXConvOp $x, $w, $add_operand, $auto_pad, $dilations, $group, $kernel_shape, $pad, $strides) >; @@ -61,7 +60,7 @@ def convAddToConvWithBiasPatternRight : Pat< def replaceWithOperationOfValue : NativeCodeCall<"$0">; -def removeLRNPattern : Pat< +def removeLRN : Pat< (ONNXLRNOp $A, $_, $_, $_, $_), (replaceWithOperationOfValue $A) >; @@ -70,10 +69,10 @@ def HaveSameStaticShape: Constraint< CPred<"onnx_mlir::haveSameStaticShape($0, $1)">, "Two tensors have the same static shape">; -def removeFlattenSameShapePattern : Pat< +def removeFlattenSameShape : Pat< (ONNXFlattenOp:$flattenOp $A, $axis), (replaceWithOperationOfValue $A), [(HaveSameStaticShape $flattenOp, $A)] ->; // Add closing parenthesis here +>; #endif // ONNX_TO_SPATIAL diff --git a/src/PIM/Conversion/ONNXToSpatial/ONNXToSpatialPass.cpp b/src/PIM/Conversion/ONNXToSpatial/ONNXToSpatialPass.cpp index 960939f..a6dc98f 100644 --- a/src/PIM/Conversion/ONNXToSpatial/ONNXToSpatialPass.cpp +++ b/src/PIM/Conversion/ONNXToSpatial/ONNXToSpatialPass.cpp @@ -50,12 +50,12 @@ void ONNXToSpatialPass::runOnOperation() { MLIRContext* ctx = &getContext(); RewritePatternSet mergeActivationPatterns(ctx); - mergeActivationPatterns.add(ctx); - mergeActivationPatterns.add(ctx); - mergeActivationPatterns.add(ctx); - mergeActivationPatterns.add(ctx); - mergeActivationPatterns.add(ctx); - mergeActivationPatterns.add(ctx); + mergeActivationPatterns.add(ctx); + mergeActivationPatterns.add(ctx); + mergeActivationPatterns.add(ctx); + mergeActivationPatterns.add(ctx); + mergeActivationPatterns.add(ctx); + mergeActivationPatterns.add(ctx); populateMatMulRewritePatterns(mergeActivationPatterns, ctx); if (failed(applyPatternsGreedily(moduleOp, std::move(mergeActivationPatterns)))) @@ -74,23 +74,24 @@ void ONNXToSpatialPass::runOnOperation() { [](ONNXMatMulOp op) { return cast(op.getY().getType()).getRank() != 2; }); target.addIllegalOp(); target.addIllegalOp(); - target.addIllegalOp(); target.addIllegalOp(); target.addIllegalOp(); - target.addIllegalOp(); + target.addIllegalOp(); target.addIllegalOp(); - target.addIllegalOp(); + target.addIllegalOp(); target.addIllegalOp(); + target.addIllegalOp(); + target.addIllegalOp(); RewritePatternSet patterns(ctx); - patterns.add(ctx); + patterns.add(ctx); - populateConvOpPatterns(patterns, ctx); - populatePoolTilingPattern(patterns, ctx); - populateOnnxGemmOpPatterns(patterns, ctx); - populateReshapeConversionPattern(patterns, ctx); - - populateONNXConcatToTensorConcatPattern(patterns, ctx); + populateGemmPatterns(patterns, ctx); + populateConvPatterns(patterns, ctx); + populatePoolPatterns(patterns, ctx); + populateReluPatterns(patterns, ctx); + populateConcatPatterns(patterns, ctx); + populateReshapePatterns(patterns, ctx); if (failed(applyPartialConversion(moduleOp, target, std::move(patterns)))) { signalPassFailure(); diff --git a/src/PIM/Conversion/ONNXToSpatial/Patterns.hpp b/src/PIM/Conversion/ONNXToSpatial/Patterns.hpp index de4ae31..58f9a10 100644 --- a/src/PIM/Conversion/ONNXToSpatial/Patterns.hpp +++ b/src/PIM/Conversion/ONNXToSpatial/Patterns.hpp @@ -5,16 +5,18 @@ namespace onnx_mlir { -void populateConvOpPatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); +void populateConvPatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); + +void populateGemmPatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); void populateMatMulRewritePatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); -void populateOnnxGemmOpPatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); +void populatePoolPatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); -void populatePoolTilingPattern(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); +void populateReluPatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); -void populateONNXConcatToTensorConcatPattern(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); +void populateConcatPatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); -void populateReshapeConversionPattern(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); +void populateReshapePatterns(mlir::RewritePatternSet& patterns, mlir::MLIRContext* ctx); } // namespace onnx_mlir diff --git a/src/PIM/Conversion/ONNXToSpatial/Patterns/Math/Conv.cpp b/src/PIM/Conversion/ONNXToSpatial/Patterns/Math/Conv.cpp index 981510b..09eb23e 100644 --- a/src/PIM/Conversion/ONNXToSpatial/Patterns/Math/Conv.cpp +++ b/src/PIM/Conversion/ONNXToSpatial/Patterns/Math/Conv.cpp @@ -260,6 +260,6 @@ LogicalResult ConvToGemm::matchAndRewrite(ONNXConvOp convOp, return success(); } -void populateConvOpPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.insert(ctx); } +void populateConvPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.insert(ctx); } } // namespace onnx_mlir diff --git a/src/PIM/Conversion/ONNXToSpatial/Patterns/Math/Gemm.cpp b/src/PIM/Conversion/ONNXToSpatial/Patterns/Math/Gemm.cpp index d8c7cb0..7ac5bfb 100644 --- a/src/PIM/Conversion/ONNXToSpatial/Patterns/Math/Gemm.cpp +++ b/src/PIM/Conversion/ONNXToSpatial/Patterns/Math/Gemm.cpp @@ -58,8 +58,7 @@ struct GemmToManyGemv : OpConversionPattern { }; struct GemvToSpatialCompute : OpConversionPattern { - GemvToSpatialCompute(MLIRContext* ctx) - : OpConversionPattern(ctx, 1) {} + using OpConversionPattern::OpConversionPattern; LogicalResult matchAndRewrite(ONNXGemmOp gemmOp, ONNXGemmOpAdaptor gemmOpAdaptor, @@ -352,7 +351,7 @@ LogicalResult GemvToSpatialCompute::matchAndRewrite(ONNXGemmOp gemmOp, return success(); } -void populateOnnxGemmOpPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { +void populateGemmPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.insert(ctx); patterns.insert(ctx); } diff --git a/src/PIM/Conversion/ONNXToSpatial/Patterns/NN/Pool.cpp b/src/PIM/Conversion/ONNXToSpatial/Patterns/NN/Pool.cpp index f558c47..281eab0 100644 --- a/src/PIM/Conversion/ONNXToSpatial/Patterns/NN/Pool.cpp +++ b/src/PIM/Conversion/ONNXToSpatial/Patterns/NN/Pool.cpp @@ -257,7 +257,7 @@ struct PoolToSpatialCompute } // namespace -void populatePoolTilingPattern(RewritePatternSet& patterns, MLIRContext* ctx) { +void populatePoolPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.insert>(ctx); patterns.insert>(ctx); } diff --git a/src/PIM/Conversion/ONNXToSpatial/Patterns/NN/Relu.cpp b/src/PIM/Conversion/ONNXToSpatial/Patterns/NN/Relu.cpp new file mode 100644 index 0000000..b922581 --- /dev/null +++ b/src/PIM/Conversion/ONNXToSpatial/Patterns/NN/Relu.cpp @@ -0,0 +1,33 @@ +#include "mlir/Transforms/DialectConversion.h" + +#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common.hpp" +#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" +#include "src/Dialect/ONNX/ONNXOps.hpp" + +using namespace mlir; + +namespace onnx_mlir { +namespace { + +struct ReluToSpatialCompute : OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + + LogicalResult + matchAndRewrite(ONNXReluOp reluOp, ONNXReluOpAdaptor adaptor, ConversionPatternRewriter& rewriter) const override { + Location loc = reluOp.getLoc(); + Type resultType = reluOp.getResult().getType(); + constexpr size_t numInputs = 1; + auto computeOp = createSpatCompute(rewriter, loc, resultType, {}, adaptor.getX(), [&](Value x) { + auto spatReluOp = spatial::SpatReluOp::create(rewriter, loc, resultType, x); + spatial::SpatYieldOp::create(rewriter, loc, spatReluOp.getResult()); + }); + rewriter.replaceOp(reluOp, computeOp); + return success(); + } +}; + +} // namespace + +void populateReluPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.add(ctx); } + +} // namespace onnx_mlir diff --git a/src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Concat.cpp b/src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Concat.cpp index 36ffc95..84271a8 100644 --- a/src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Concat.cpp +++ b/src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Concat.cpp @@ -9,8 +9,7 @@ using namespace mlir; namespace onnx_mlir { struct Concat : public OpConversionPattern { - Concat(MLIRContext* ctx) - : OpConversionPattern(ctx) {} + using OpConversionPattern::OpConversionPattern; LogicalResult matchAndRewrite(ONNXConcatOp maxpoolOp, ONNXConcatOpAdaptor adaptor, @@ -24,7 +23,7 @@ struct Concat : public OpConversionPattern { } }; -void populateONNXConcatToTensorConcatPattern(RewritePatternSet& patterns, MLIRContext* ctx) { +void populateConcatPatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.insert(ctx); } diff --git a/src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Reshape.cpp b/src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Reshape.cpp index befa1c5..4499c7c 100644 --- a/src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Reshape.cpp +++ b/src/PIM/Conversion/ONNXToSpatial/Patterns/Tensor/Reshape.cpp @@ -114,6 +114,6 @@ struct Reshape : OpConversionPattern { } // namespace -void populateReshapeConversionPattern(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.insert(ctx); } +void populateReshapePatterns(RewritePatternSet& patterns, MLIRContext* ctx) { patterns.insert(ctx); } } // namespace onnx_mlir diff --git a/src/PIM/Conversion/SpatialToPim/SpatialToPim.td b/src/PIM/Conversion/SpatialToPim/SpatialToPim.td index 88eadbc..de37037 100644 --- a/src/PIM/Conversion/SpatialToPim/SpatialToPim.td +++ b/src/PIM/Conversion/SpatialToPim/SpatialToPim.td @@ -9,41 +9,46 @@ include "src/Accelerators/PIM/Dialect/Spatial/Spatial.td" include "src/Accelerators/PIM/Dialect/Pim/Pim.td" #endif // OP_BASE -def onnxToPimTransposeOp : Pat< +def onnxToPimTranspose : Pat< (ONNXTransposeOp:$srcOpRes $data, $perms), (PimTransposeOp $data, $perms, (NativeCodeCall<"onnx_mlir::getBestOutputTensorFromOperandsOrAllocate($_builder, $0.getDefiningOp())"> $srcOpRes)) >; -def spatToPimVMMOp : Pat< +def spatToPimVMM : Pat< (SpatWeightedVMMOp:$srcOpRes $weightIndex, $vector), (PimVMMOp $weightIndex, $vector, (NativeCodeCall<"onnx_mlir::getBestOutputTensorFromOperandsOrAllocate($_builder, $0.getDefiningOp())"> $srcOpRes)) >; -def spatToPimMVMOp : Pat< +def spatToPimMVM : Pat< (SpatWeightedMVMOp:$srcOpRes $weightIndex, $vector), (PimMVMOp $weightIndex, $vector, (NativeCodeCall<"onnx_mlir::getBestOutputTensorFromOperandsOrAllocate($_builder, $0.getDefiningOp())"> $srcOpRes)) >; -def spatToPimVVAddOp : Pat< +def spatToPimVVAdd : Pat< (SpatVAddOp:$srcOpRes $a, $b), (PimVVAddOp $a, $b, (NativeCodeCall<"onnx_mlir::getBestOutputTensorFromOperandsOrAllocate($_builder, $0.getDefiningOp())"> $srcOpRes)) >; -def spatToPimVVMulOp : Pat< +def spatToPimVVMul : Pat< (SpatVMulOp:$srcOpRes $a, $b), (PimVVMulOp $a, $b, (NativeCodeCall<"onnx_mlir::getBestOutputTensorFromOperandsOrAllocate($_builder, $0.getDefiningOp())"> $srcOpRes)) >; -def spatToPimVVMaxOp : Pat< +def spatToPimVVMax : Pat< (SpatVMaxOp:$srcOpRes $a, $b), (PimVVMaxOp $a, $b, (NativeCodeCall<"onnx_mlir::getBestOutputTensorFromOperandsOrAllocate($_builder, $0.getDefiningOp())"> $srcOpRes)) >; +def spatToPimVRelu : Pat< + (SpatReluOp:$srcOpRes $input), + (PimVReluOp $input, + (NativeCodeCall<"onnx_mlir::getBestOutputTensorFromOperandsOrAllocate($_builder, $0.getDefiningOp())"> $srcOpRes)) +>; #endif // SPATIAL_TO_PIM diff --git a/src/PIM/Dialect/Pim/Pim.td b/src/PIM/Dialect/Pim/Pim.td index 616da86..300ddd1 100644 --- a/src/PIM/Dialect/Pim/Pim.td +++ b/src/PIM/Dialect/Pim/Pim.td @@ -363,29 +363,6 @@ def PimVVDMulOp : PimOp<"vvdmul", [DestinationStyleOpInterface]> { }]; } -def PimSumOp : PimOp<"sum", [DestinationStyleOpInterface]> { - let summary = "Reduce all elements to a single value"; - - let arguments = (ins - PimTensor:$input, - PimTensor:$outputBuffer - ); - - let results = (outs - PimTensor:$output - ); - - let extraClassDeclaration = [{ - mlir::MutableOperandRange getDpsInitsMutable() { - return getOutputBufferMutable(); - } - }]; - - let assemblyFormat = [{ - `(` $input `,` $outputBuffer `)` attr-dict `:` `(` type($input) `,` type($outputBuffer) `)` `->` type($output) - }]; -} - def PimVAvgOp : PimOp<"vavg", [DestinationStyleOpInterface]> { let summary = "Average all elements into a single value"; diff --git a/src/PIM/Dialect/Pim/Transforms/Bufferization/OpBufferizationInterfaces.cpp b/src/PIM/Dialect/Pim/Transforms/Bufferization/OpBufferizationInterfaces.cpp index 780d1e2..5d29208 100644 --- a/src/PIM/Dialect/Pim/Transforms/Bufferization/OpBufferizationInterfaces.cpp +++ b/src/PIM/Dialect/Pim/Transforms/Bufferization/OpBufferizationInterfaces.cpp @@ -97,8 +97,7 @@ struct MemCopyDevToHostOpInterface } }; -struct TransposeOpBufferizeInterface -: DstBufferizableOpInterfaceExternalModel { +struct TransposeOpInterface : DstBufferizableOpInterfaceExternalModel { bool bufferizesToMemoryRead(Operation* op, OpOperand& opOperand, const AnalysisState& state) const { return !cast(op).isDpsInit(&opOperand); } @@ -123,7 +122,7 @@ struct TransposeOpBufferizeInterface } }; -struct VMMOpBufferizeInterface : DstBufferizableOpInterfaceExternalModel { +struct VMMOpInterface : DstBufferizableOpInterfaceExternalModel { bool bufferizesToMemoryRead(Operation* op, OpOperand& opOperand, const AnalysisState& state) const { return !cast(op).isDpsInit(&opOperand); } @@ -160,7 +159,7 @@ struct VMMOpBufferizeInterface : DstBufferizableOpInterfaceExternalModel { +struct MVMOpInterface : DstBufferizableOpInterfaceExternalModel { bool bufferizesToMemoryRead(Operation* op, OpOperand& opOperand, const AnalysisState& state) const { return !cast(op).isDpsInit(&opOperand); } @@ -186,8 +185,7 @@ struct MVMOpBufferizeInterface : DstBufferizableOpInterfaceExternalModel -struct BinaryDstOpBufferizeInterface -: DstBufferizableOpInterfaceExternalModel, OpTy> { +struct BinaryDstOpInterface : DstBufferizableOpInterfaceExternalModel, OpTy> { bool bufferizesToMemoryRead(Operation* op, OpOperand& opOperand, const AnalysisState& state) const { return !cast(op).isDpsInit(&opOperand); } @@ -225,17 +223,56 @@ struct BinaryDstOpBufferizeInterface } }; +template +struct UnaryDstOpInterface : DstBufferizableOpInterfaceExternalModel, OpTy> { + bool bufferizesToMemoryRead(Operation* op, OpOperand& opOperand, const AnalysisState& state) const { + return !cast(op).isDpsInit(&opOperand); + } + + bool bufferizesToElementwiseAccess(Operation* tablegen_opaque_val, + const AnalysisState& state, + ArrayRef opOperands) const { + return true; + } + + LogicalResult bufferize(Operation* op, + RewriterBase& rewriter, + const BufferizationOptions& options, + BufferizationState& state) const { + auto unaryOp = cast(op); + + auto inputOpt = getBuffer(rewriter, unaryOp.getInput(), options, state); + if (failed(inputOpt)) + return failure(); + + auto outputBufferOpt = getBuffer(rewriter, unaryOp.getOutputBuffer(), options, state); + if (failed(outputBufferOpt)) + return failure(); + + Value contiguousInput = materializeContiguousMemRef(*inputOpt, op->getLoc(), rewriter); + + replaceOpWithNewBufferizedOp(rewriter, op, outputBufferOpt->getType(), contiguousInput, *outputBufferOpt); + return success(); + } +}; + void registerOpBufferizationInterfaces(DialectRegistry& registry) { registry.addExtension(+[](MLIRContext* ctx, PimDialect* dialect) { PimMemCopyHostToDevOp::attachInterface(*ctx); PimMemCopyDevToHostOp::attachInterface(*ctx); - PimTransposeOp::attachInterface(*ctx); - PimVMMOp::attachInterface(*ctx); - PimMVMOp::attachInterface(*ctx); - PimVVAddOp::attachInterface>(*ctx); - PimVVSubOp::attachInterface>(*ctx); - PimVVMulOp::attachInterface>(*ctx); - PimVVMaxOp::attachInterface>(*ctx); + PimTransposeOp::attachInterface(*ctx); + PimVMMOp::attachInterface(*ctx); + PimMVMOp::attachInterface(*ctx); + + PimVVAddOp::attachInterface>(*ctx); + PimVVSubOp::attachInterface>(*ctx); + PimVVMulOp::attachInterface>(*ctx); + PimVVMaxOp::attachInterface>(*ctx); + + PimVAvgOp::attachInterface>(*ctx); + PimVReluOp::attachInterface>(*ctx); + PimVTanhOp::attachInterface>(*ctx); + PimVSigmOp::attachInterface>(*ctx); }); } diff --git a/src/PIM/Dialect/Pim/Transforms/Bufferization/PimBufferization.td b/src/PIM/Dialect/Pim/Transforms/Bufferization/PimBufferization.td index f5040a4..bc920e3 100644 --- a/src/PIM/Dialect/Pim/Transforms/Bufferization/PimBufferization.td +++ b/src/PIM/Dialect/Pim/Transforms/Bufferization/PimBufferization.td @@ -16,5 +16,4 @@ def memrefCopyToPimMemCopyOp : Pat< (returnType $dst)) >; - #endif // PIM_BUFFERIZATION diff --git a/src/PIM/Dialect/Pim/Transforms/Bufferization/PimBufferizationPass.cpp b/src/PIM/Dialect/Pim/Transforms/Bufferization/PimBufferizationPass.cpp index 10da3ab..a2b99a2 100644 --- a/src/PIM/Dialect/Pim/Transforms/Bufferization/PimBufferizationPass.cpp +++ b/src/PIM/Dialect/Pim/Transforms/Bufferization/PimBufferizationPass.cpp @@ -105,6 +105,6 @@ void PimBufferizationPass::annotateWeightsMemrefs(ModuleOp moduleOp, func::FuncO }); } -std::unique_ptr createBufferizePimPass() { return std::make_unique(); } +std::unique_ptr createPimBufferizationPass() { return std::make_unique(); } } // namespace onnx_mlir diff --git a/src/PIM/Dialect/Spatial/Transforms/SpatialBufferizableOpInterface.cpp b/src/PIM/Dialect/Spatial/Transforms/SpatialBufferizableOpInterface.cpp index a27cc89..5d0869f 100644 --- a/src/PIM/Dialect/Spatial/Transforms/SpatialBufferizableOpInterface.cpp +++ b/src/PIM/Dialect/Spatial/Transforms/SpatialBufferizableOpInterface.cpp @@ -486,8 +486,6 @@ struct WVMMOpInterface : WeightedMultiplicationsOpInterface {}; -struct SumOpInterface : VariadicArgumentElementWiseOpInterface {}; - struct VMaxOpInterface : VariadicArgumentElementWiseOpInterface {}; void registerBufferizableOpInterfaceExternalModels(DialectRegistry& registry) { @@ -496,7 +494,6 @@ void registerBufferizableOpInterfaceExternalModels(DialectRegistry& registry) { SpatVAddOp::attachInterface(*ctx); SpatWeightedVMMOp::attachInterface(*ctx); SpatWeightedMVMOp::attachInterface(*ctx); - SpatSumOp::attachInterface(*ctx); SpatVMaxOp::attachInterface(*ctx); SpatChannelReceiveOp::attachInterface(*ctx); SpatChannelSendOp::attachInterface(*ctx); diff --git a/src/PIM/Pass/PIMPasses.h b/src/PIM/Pass/PIMPasses.h index 3285a48..ba0724c 100644 --- a/src/PIM/Pass/PIMPasses.h +++ b/src/PIM/Pass/PIMPasses.h @@ -13,13 +13,13 @@ std::unique_ptr createSpatialToGraphvizPass(); std::unique_ptr createSpatialToPimPass(); -std::unique_ptr createBufferizePimPass(); +std::unique_ptr createPimBufferizationPass(); -std::unique_ptr createConstantFoldingPass(); +std::unique_ptr createPimConstantFoldingPass(); -std::unique_ptr createMaterializeConstantsPass(); +std::unique_ptr createPimMaterializeConstantsPass(); -std::unique_ptr createVerificationPass(); +std::unique_ptr createPimVerificationPass(); std::unique_ptr createEmitPimJsonPass(); diff --git a/src/PIM/Pass/Pim/ConstantFolding/ConstantFoldingPass.cpp b/src/PIM/Pass/Pim/ConstantFolding/ConstantFoldingPass.cpp index e237de4..76bd097 100644 --- a/src/PIM/Pass/Pim/ConstantFolding/ConstantFoldingPass.cpp +++ b/src/PIM/Pass/Pim/ConstantFolding/ConstantFoldingPass.cpp @@ -47,6 +47,6 @@ struct ConstantFoldingPass : PassWrapper createConstantFoldingPass() { return std::make_unique(); } +std::unique_ptr createPimConstantFoldingPass() { return std::make_unique(); } } // namespace onnx_mlir diff --git a/src/PIM/Pass/Pim/MaterializeConstantsPass.cpp b/src/PIM/Pass/Pim/MaterializeConstantsPass.cpp index bf9b084..fdb9904 100644 --- a/src/PIM/Pass/Pim/MaterializeConstantsPass.cpp +++ b/src/PIM/Pass/Pim/MaterializeConstantsPass.cpp @@ -126,6 +126,6 @@ struct MaterializeConstantsPass : PassWrapper createMaterializeConstantsPass() { return std::make_unique(); } +std::unique_ptr createPimMaterializeConstantsPass() { return std::make_unique(); } } // namespace onnx_mlir diff --git a/src/PIM/Pass/Pim/VerificationPass.cpp b/src/PIM/Pass/Pim/VerificationPass.cpp index 4661744..809685a 100644 --- a/src/PIM/Pass/Pim/VerificationPass.cpp +++ b/src/PIM/Pass/Pim/VerificationPass.cpp @@ -189,6 +189,6 @@ private: } // namespace -std::unique_ptr createVerificationPass() { return std::make_unique(); } +std::unique_ptr createPimVerificationPass() { return std::make_unique(); } } // namespace onnx_mlir diff --git a/src/PIM/PimAccelerator.cpp b/src/PIM/PimAccelerator.cpp index aaa82c5..4541898 100644 --- a/src/PIM/PimAccelerator.cpp +++ b/src/PIM/PimAccelerator.cpp @@ -19,6 +19,7 @@ #include "src/Accelerators/PIM/Dialect/Spatial/Transforms/SpatialBufferizableOpInterface.hpp" #include "src/Accelerators/PIM/Pass/PIMPasses.h" #include "src/Accelerators/PIM/PimAccelerator.hpp" +#include "src/Compiler/CompilerUtils.hpp" #define DEBUG_TYPE "PimAccelerator" @@ -69,13 +70,14 @@ void PimAccelerator::registerDialects(mlir::DialectRegistry& registry) const { void PimAccelerator::registerPasses(int optLevel) const { LLVM_DEBUG(llvm::dbgs() << "Registering passes for PIM accelerator\n"); + TOTAL_COMPILE_PHASE = 8; registerPass(createONNXToSpatialPass); registerPass(createSpatialToGraphvizPass); registerPass(createSpatialToPimPass); - registerPass(createBufferizePimPass); - registerPass(createConstantFoldingPass); - registerPass(createMaterializeConstantsPass); - registerPass(createVerificationPass); + registerPass(createPimBufferizationPass); + registerPass(createPimConstantFoldingPass); + registerPass(createPimMaterializeConstantsPass); + registerPass(createPimVerificationPass); registerPass(createEmitPimJsonPass); } diff --git a/validation/operations/README.md b/validation/operations/README.md index cd6e88e..6594703 100644 --- a/validation/operations/README.md +++ b/validation/operations/README.md @@ -35,6 +35,15 @@ python3 validation/operations/gen_tests.py | Avg include pad | `pool/avg_include_pad` | [1,2,4,4] | [1,2,2,2] | 3x3 | 2 | [1,1,1,1] | `count_include_pad=1` | | Max after Conv | `pool/max_after_conv` | [1,3,6,6] | [1,4,2,2] | Conv 3x3 then Pool 2x2 | 2 | none | Regression for `pool(conv(...))` | +## Relu + +| Test | Directory | Input | Output | Notes | +|------|-----------|-------|--------|-------| +| Basic | `relu/basic` | [4,8] | [4,8] | Standalone 2D Relu | +| 4D | `relu/4d` | [2,3,4,4] | [2,3,4,4] | Standalone NCHW Relu | +| After Conv | `relu/after_conv` | [1,3,5,5] | [1,2,3,3] | Conv 3x3 + bias, then Relu | +| After Gemm | `relu/after_gemm` | [4,64] | [4,32] | Gemm + bias, then Relu | + ## Gemm | Test | Directory | A (input) | W (weight) | Output | transB | alpha | beta | Bias | Notes | diff --git a/validation/operations/gen_tests.py b/validation/operations/gen_tests.py index 526c4c9..777725b 100644 --- a/validation/operations/gen_tests.py +++ b/validation/operations/gen_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate ONNX test models for validating GEMM, Conv, and Pooling implementations.""" +"""Generate ONNX test models for validating GEMM, Conv, Pooling, and Relu implementations.""" import numpy as np import onnx @@ -327,6 +327,60 @@ def maxpool_after_conv(): save_model(model, "pool/max_after_conv", "maxpool_after_conv.onnx") +# --------------------------------------------------------------------------- +# Relu tests +# --------------------------------------------------------------------------- + +def relu_basic(): + """Standalone Relu on a simple 2D tensor.""" + X = helper.make_tensor_value_info("X", TensorProto.FLOAT, [4, 8]) + Y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [4, 8]) + node = helper.make_node("Relu", ["X"], ["Y"]) + graph = helper.make_graph([node], "relu_basic", [X], [Y]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) + save_model(model, "relu/basic", "relu_basic.onnx") + + +def relu_4d(): + """Standalone Relu on an NCHW tensor.""" + X = helper.make_tensor_value_info("X", TensorProto.FLOAT, [2, 3, 4, 4]) + Y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [2, 3, 4, 4]) + node = helper.make_node("Relu", ["X"], ["Y"]) + graph = helper.make_graph([node], "relu_4d", [X], [Y]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) + save_model(model, "relu/4d", "relu_4d.onnx") + + +def relu_after_conv(): + """Conv followed by Relu.""" + rng = np.random.default_rng(60) + X = helper.make_tensor_value_info("X", TensorProto.FLOAT, [1, 3, 5, 5]) + Y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [1, 2, 3, 3]) + W = numpy_helper.from_array(rng.uniform(-1, 1, (2, 3, 3, 3)).astype(np.float32), name="W") + B = numpy_helper.from_array(rng.uniform(-1, 1, (2,)).astype(np.float32), name="B") + conv = helper.make_node("Conv", ["X", "W", "B"], ["C"], + kernel_shape=[3, 3], strides=[1, 1], pads=[0, 0, 0, 0]) + relu = helper.make_node("Relu", ["C"], ["Y"]) + graph = helper.make_graph([conv, relu], "relu_after_conv", [X], [Y], initializer=[W, B]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) + save_model(model, "relu/after_conv", "relu_after_conv.onnx") + + +def relu_after_gemm(): + """Gemm followed by Relu.""" + B, K, N = 4, 64, 32 + rng = np.random.default_rng(61) + W = numpy_helper.from_array(rng.uniform(-1, 1, (K, N)).astype(np.float32), name="W") + C = numpy_helper.from_array(rng.uniform(-1, 1, (N,)).astype(np.float32), name="C") + A = helper.make_tensor_value_info("A", TensorProto.FLOAT, [B, K]) + Y = helper.make_tensor_value_info("Y", TensorProto.FLOAT, [B, N]) + gemm = helper.make_node("Gemm", ["A", "W", "C"], ["G"]) + relu = helper.make_node("Relu", ["G"], ["Y"]) + graph = helper.make_graph([gemm, relu], "relu_after_gemm", [A], [Y], initializer=[W, C]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) + save_model(model, "relu/after_gemm", "relu_after_gemm.onnx") + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -361,4 +415,10 @@ if __name__ == "__main__": avgpool_include_pad() maxpool_after_conv() + print("\nGenerating Relu tests:") + relu_basic() + relu_4d() + relu_after_conv() + relu_after_gemm() + print("\nDone.") diff --git a/validation/operations/relu/4d/relu_4d.onnx b/validation/operations/relu/4d/relu_4d.onnx new file mode 100644 index 0000000000000000000000000000000000000000..3d62ef7bb385607ad49bb67000cf74f3a1e308e3 GIT binary patch literal 91 zcmd0o;bM#sVvJN`2};c=6=E*}lJO=fQPLn`F)k4fMj-(%CJrVbW(Hyw5KaP0 OM&gomV&P&C-~|9sp$X0a literal 0 HcmV?d00001 diff --git a/validation/operations/relu/after_conv/relu_after_conv.onnx b/validation/operations/relu/after_conv/relu_after_conv.onnx new file mode 100644 index 0000000000000000000000000000000000000000..d2cb581926919a32ccec06d156bb1ff8e0cd96c0 GIT binary patch literal 434 zcmd1<;TSs!NnNP#poo&=&Z!zoS#>wCCbHsMXCcN&C6=JnoKhkFA|M%`m{yWn z6rT(-`UN8g69+R03NSh`hI`#${Pb?~zI&2I`xpF9-oHh$+Fpgx%6{jMbGA>m=G)I$ zRc)Ui^=8$a{a+@A?P-2nZvXCD^S;X${q1dU%I!U< z5n{jgXT81W%{03h|8Uz^-<|e2q5Gqxfq^X~#wEhRC?vqe#K8!}%s|Wv!bw2MNHj?%r~+mu7A^(>UI3U4lg|JE literal 0 HcmV?d00001 diff --git a/validation/operations/relu/after_gemm/relu_after_gemm.onnx b/validation/operations/relu/after_gemm/relu_after_gemm.onnx new file mode 100644 index 0000000000000000000000000000000000000000..0c8091544b8acdf66632ad5461ad18630eb009be GIT binary patch literal 8452 zcmZ9SX*7`C*T&6c3dxj2s1PDWaqn%)l&EADGDb9#P&6k>r8J0CN_v`mq{+Q^(WpTg z8mOm9^Q=-z^|#jh?fr1hhkZVrwbwd(pWk(zqo6Xj+eJ}BQPM?GGFVa4O+(Uiu#9JX zYO12LqNJyWWawa-$?+-a8hz*X&Qa46(&FbvCH5|xuW?pzRxnhPbd?PDS?nyoXEB*N zDxtCS5jZmZ4(vE6Pd(dmP+l$Hf?p| zvg4Ow;pKQ7Jy4nxH3srdctum+$bkI^LoT-rWUr*JYC8c?^&Cm;NO|#H`&}(YE z6NcNaPQktV6#0S0Oz~=PfBYwKCQ@M=t=KRR=S4gK)1tX#sXm*fiX`}uekL#K>&4k} z4rCImhAB0+=vng}qP7I_;k^5>a)A=gelQBO^Pkf)<0w@8tB0+N9?_fCHrQTrkHX~U z^E-Vvav6StlqH{&($z5f@9tArlYIlODCBZ+-vM~)*h4ze6NL*-G{TS4F#a(74+OlB zz%oNc;fq!{-zhMJ+iMf?&UF*K=^%%{KAZB-fJ7W*FaY0gw_>mIC$*kR;c&;I8(#jJ ziJp$#)Nf^}SYo;gB0c}1V8NWzOb2shxB@rd+X*{*^JK|QBcB-M_0zZAmJ9 zm6za{57y!%jX_}9?1_~_hj9MaB&=T$3d;>vlftkWe328Jyfkfi`&=`+K588IWQ|9- zl7lnI4=?`w4W$v!$bGOW-&_0B$#}snu-KNv-b()pGlQy0XJ->lIT(T8R}@2}UnYN< zR4AwY@mgYef@7ruJ`6O@w)=dD`|t5#-WnNu|#`{vK5;)n79Z8Ls%SeyIoGvQzO zlHSzlLD5wUUY|aSgPr%2-tUE+=a-FN5>ASCp_l0T+jN?naF=!RJjhdly*Y94&P`TA9F+Ck~L~J4EaJXvlab!+Ur5;FELx_;Y&~9Z5IBwg?x} zXo$t+R%Kk=Ka77R=8&n@U22^$gzYbnV8cV9kYc6;O$U8p;b%qMaV3Qd6`qjgM=5C2 zQN&kX7vM*h5$*^RFnRMcal~9HmOfM}MC)pCLftF)J0ln~J{_R)&kOL|!4RGj(?BD) z#PAAf4f@wy2iI3*arG_`D#B;LFJoCzMaGafO`60#c1FS?Pb0jrgai|V-=wDgM8K|A zYRT9w4)5c~lbj}U^VOp;`|KY|O&v=*^KwwO^*c;CjeJa19UoY>I1ZXz0bA}XgGpqB zI7;Zt2fAuttydubBQMfVho>NS`VAeHpCa^~Xh|RTJK+YuDyrAZ17+-k7yTqDo-Yc* zvR_V58||@UZx%V!5zp>>})$1Xwe{5;gl=6+J#zlC{bjv1x}X&Rz8zHokvAo7>B2 z-hs`OwJDEY)vTpS4v}0~wg}R{I-y;rJB})HM+JAJKX=!HbAlP!m5k;gJ!{2TorAE# zzQysv#phz?`}g9fo1;J~(2@IZ7|AIXgM|}V-GFBktaO_+&*YPT-G78rg4swT zo^I%^g1bMY@dD|w)n5-N6o=usom#YRrYomDmBpq9zQVpc=G^KegKjGQ!Rg3ka@C%I z`}SGWu0e`iaV>%`ub;~MzN~{y(XxEq$Q=vqM)IF}{D8`go?`DY19mdB3hqIfI$QMJG!EI>F7^RVve5QuRzpLsAv6=a?TFGVCQF0t$nFh{Z=|3X;tOPmlwdb z>qGI^xFMJ}BO4PYSdsVaVfaE=Lz+Y5QM$H*tBlevtl+)jGYR$=YNYaSNffup=d|vWkzs@dmMP) zTu3IrZit>%04s0z)J_n_z>3_nH2b6)UV7diYOA97{-l4Pa9{zIbXc=acnIygvy1HS zC9~0fRV=r+!;0U_h2*$uIP(8ut}?>@xTDDrRvYjn4Z;i>X|V$#V; zsFD%Dy3WJ!WaK*NYkM5FtTe|SnQGBMkPLPd6yFZ~_+U<1d22L+0>h zBTeCV_dMA7h+%upSbQO;iE7eEXy}*`jyIf4@Kf7qi2M7Jbd}X{$X!V+E>)qV;v?d3 z8G;G9i|EUskJNwsAsSk`leP|5#YP+j5%Y9#hjcI(l;}WOfi*Ynvct>ou0z3y81B=u zj*>-pp4oHG$ydsr|M?lf7c{eZ#V%hl0o5_ zW*x_g*gZmu3r+fQk(C|l{f)<6Qafp4*-bk8vOm9n-2zYhH`CkT2q&{(2X-&(AT?7P z_;e%;`-Dr7{!eR^nB|OZ@3zp;6({LbR|s8Sx`7#BFBpju{mRVe!H4>-4(qcW`8|1Y7DD6Ha zi>23`Aa#Wgx+T9O_1_2J(umijzch#>HwJK3wI!;4A!^C@#OxocJZ%<{?V#CIzN$B$ z@Jz1oG~^L;)NpT$GWH#`pAJ}9a%Nu*&ReW3L{*-Et(7V`=Jb4i87NJkv;^$@SP9by z#G#wC4IbLI33iUJr?%8Ga%=L$C)!dVy|RI1PA2htH*La~>U{5L4nEBPKxZ#51Lsx& zb9am6|79g?96BA1A&ckRGzlAY*HYu|p*&X694lR?W6ZeucyI6={97-HOC$AhSBor* z89!n3#~e(bs|@AYuPD7k7A1Y3!Abog!irb}7UY63Gp7ftErSG|g&H_VeG^qL3?sd& zxhNjq3H3_v;hst}bsY@AR|SqZ`tSfeT}h%;fhtFZl)_QZQ)2M1-IPvH!M<&8 zDAe*jxoOVA@aK9MJxY)4MMpS#NEvIrep17vWZbOf%gU4sS<0hvgkL@+9Es-|Jx^@P z_Q1Bu`aD_Ep5qK7MDMk8p<~E6vEFAjg+`u)qT+GfHtRCXxmF0?YK}ZNL5g+iC-c7t zx50(C0UWQL4_b@oqgCKdsQeisswvcyU)CNnwggt%v==nLMbnPn-qAQtmaDwIdE_Dh z9d!llT5gPOAFNpKgW=iXXHu|xJy750zP$8?0T1j4qSf(tu;!E^zgw@xmgeJORdpm> zx;LFu?j?bIM=mZ`m*;UAp1kpTHXisg5M6cR_}ZXdAYm~J9T%2CpQ#K}KY8Mw$M(E( zjx9@E$%lHiQ)Ko)8~#|kq5U~|$ew6|3V(GmHozPm&-djcCy!9>Yad)|`%B#ZRe`1Y z8}a(mQm`Ur9`Un=^180o+S&)>z4Q|Cs@ZBG@@a`!yf%lk|0c5k2Y(2ZUnl%5=_>>r zk7bAab>O>UJ1jmP#2x2HV*f-5Nbi&uWF}a1$E9%eS=SH##wF5@YqKFKFa)L^+h2Qx zMhcB1j3M;71-@JBKHu{W$!U9QkQ2gWa!|LZd`Otz5%+ zoV!e&b;l{=l|5Qmu_BxGJ}1xuFKagKc`P_aic~O9nrjyph<87Y6>gnQ#r9bt+oUBMP3dUS&H=Z+$m&25;N5;N9M!N4FD&^zu|3 zggFjmbM?mrJyl?Jd=7MGIfI;;8D3PeCC^XFq+TA49#u_r@0l}Sk#}HyvrDk-?E>!m zN*)jTsA7u8IC>IVAp}+w!>GGgsd`Bul)Q_>k=`(mt<7>( z^1SBnOfrj^j5{XB;u_t(aNuf{m><##Rhd#eTwRsD9~Hs+#hLW?B0coDzMj>LF)I zH>GR)0bienKd(QcHkIAbw6Y%-&kp3*YW+A$$CTVf4fLAVM|>8njrF$&z|nV>oF*o4 zwbe9!a!*f)u$_W6U5Q*2Fp7PG$KdH}Wu$yPhfQsBXsNRSNZtKJ`*!D`bD|w~80+Dz zcV3WD**rGYVIr>6oXSmCU1;}z$6=wrGJiZ;tr;BT8GzYqjo|C9Rkf=m zRdA-`Dsp-ET3q$@5siLj2(SEN_#f5VQr0iO2R z24@(&p=BybcqzXN`VV!5`?`)8b$7Wa@%^^&dSRWY*e%89<@GQ`H4qPG%>k`Ba;)oW zkGE7`lWp2`yr6mFte$NhY#btktqp##^OHH>@EwbLPc9W-?TdjP6F0v5ayD35C&9C2 z*|@jEn|0FWL5Rs~7_+BF+`IP zN!wOiJMa-*_%nsi>pGH-lny35EP+81uIOqw0oN~7LfeaSTyby;e>-f$PA(l}VjIe_ z)-T0%sS8*l+>Aedor}to!&tkhij-CuVXfs-$ZlOsig~VJ{X7I$FZl$egO|{t;zLkw z`T){h4^z&BkuXAX6wf}dO#hZhu)&&}!c-R%GT(fiT=+M=2pquPsY6&z`Xc2OSwQl& z(}H448I26w4>5B(g#&Yv#fr2rmVB}u9xO1Y_HXmJZ+0HlmZflmeg>!h8^U4Bwg_&+ ziiEuN7QA2hN&XR?lyy6cqvlHqE9BaHx#2h2R4TLY_bfrk3Z%dMns%}o+n>>)Ltb&X zai*n#-&255%rat?`?V|908QibR3SR~mkVEreuJHRvF7kHx>cD7T;ZjR* zxkE7j;?H@f&cc>H*XW2$5nSA50~4|eKu5_Rel3aPu?=n?*7o@^2Gk0$fox_vOM zyf3diH<5!HbWpqI3c1b*fz6SA7@cH-k#CD&-A)JGYq1Z+-8D2}cp!dv2o|n)-4U%N z9*N3U(pYd%S8Pd`%>jc)vS#yS_H9gM7sn6K;$g=sd-8?2iUs)bj1jgBc`u|*kmE6R z?NAr$15eF%k<{#MVwBJyZzT+3+aP~#){SAmOD?Dt_6D-o4uwwXbS}Gk4oXZNxK6^9 z3$K3z=LrkgC^=Mc$V}i>{qBpWO6PF=@5M0oh#%_Q{!E`V$MgD06Jb}xX-XX&%G2)T z@awo3+;VybUQCd{{FEw4RT$0d{YHS~+PR#%W|L^6cvM^(It9nINYyH@SOLwMsTkuh z4Z^E>>sr$tzwGnJAJR*qY106bS&%Opzip%sz24LTPZON|a}3&EE}+pe$EaVZ4Nq#( z7m|twV6*HAm~%r1Y0q@v9rhNk?v7o6|Ty8mlbuNy@ zA;YaH!#0_94m<50A%tuaI~2{~j8KKR=0*k!sD>Z}Y15_D1TLo}Z|<*Gq|h28oR&6%zkW@id$L0y>scqwxIdl$wp#P1dTqA7y_yzx$fJ6D8rNlS z5YAjWMVos0=Ed)wU~gf8HqlLV@`MFGGH(){uQiK&T#Bx565)b>8M*6+(32YjIq~~3 zvAjeAifcny?)n{S&TfV5TT|#kcs;yOUn>5%X^MX3tu$w{BIRfvrnreaV5_q$cJEg~ z`!n&ps=1w9uAYFrEds7uJd8`m$#Fp4QIK4pAw;Z>gNMKG!Lx*WFk#?eymRcV=(T4a zYW|rswj*;5B?hg7-1Wc2jh05>`>U6;PkQ0aSTEi(dITn^U7>5z1K7cECDeU3M=znS zxA%u&iJ~s*1a-sGr2_6fGldJ&qj}lnS=bm_Df)kxz?2b+oaZnYqpECpj!F)Ef04kC zE%u9feVG_fr1N~X=`;e?0#J%WE>idQ=K!h zt>0j}UntG#s``-VwGE0b+KInMv2)I5(A{=iH1Aa9UEjyxeyKlnDkxs?dZCF`rhn+i z^9ZgxJCY_Gk-?9h!(r>0;XG~7ER2(t<3T_war)z{E^7 z^si6{=a~ZbUz8=xur%PWN3Bq^A|0#DmeqcOgjG)Ctb;4PrGeo zxXp$QQu2g&a^#*&I|OZK{OaqEt9?TGO^F-(j9g61HaO6L@h0dJ)C?ZZ-K3VvFt&A} zFw<8DCB)_Ma@=p~I^e|zkM0qV|7%2l%4JbC+8-~!4WngwLwIO)30zvF1-q67;P7$7 zdG?AWFlTob>e|P0_zXF`P}>i!Ct2eF=_0r^eInlcCB?Cd4KVV%B!B6a!|1Me@c6+l z=p3NLUvn4Y@e>ZPbBhJ@(FfolbrHsRFQBY7@?7m-P62_YT>ZIC`2KApjGnbZ=zGi? z&)+^ng=a={W(QGo=~OgO(BM+f8<5a`SWKU?gW`HU5Y3;ngzx$B9BS>sTS~oQvhTmp z@wpJ@1pgA^dYM0N$9_6{WgaW_dKQ*LRGG?VVMAG@_&8!1o?ZD(w90Y9b%UJns=f!G z*BOZF)B~R_Gf=yI5nO)J1Dl2lm^f+y9`d%}-rbyBBP-#6S07xac)YfCv={ygOA$;O zZ$s-qb?n>H((6gd;imYpToM)A>y_RWf&33GF=gTi-N6dS_9d+i&zC8Nbv^J5|UW$)Wc|5#3`*X8qXvv7FnWY!>Wyi=Wn z{#EXnn%9Tj+l{&HfG(#MPKU7unw;wLkI;HE8=~^YIxTT5?A55(q5fe%v`i2Gg$N#Z-*S;^$sB04M z44c9lc8}<5yb|9ja)s#Sa{Ru35b7u>v2lkK41GC=SNSP%hN(ZKjdtxVI(;^Ch@pPx zIv~Sur*KNo0@u{0@Qo#jtohG$Hi(oE@C$(5yD)y|V~tT8p1?h2FPs*4ipC$BjmTJ~4OXV$_|ciDlIMUkPrMNeCS}mc@-y^VbvWDTucH|ak16xl z_+CD|Mdt>(vPNYd4Kj?yRf;7btsc#}b4~EV&H|`?nT%m=3H0hebumCIkPkbr6t3*9 zqBj}K;O7SkYJ4~XD$3hL^&z2@7&Qpr_whn3F-7yS7e)JAQ>-YCcoV+=ZFs$Ft(qaloAa=ntlHo!VL`a8T#rx5o2|0j==1%avr~_X29Z zq34r(xi{Jd9<6d@9d9LkJ9QjxYLkLTD$-aFD`DdEzI^RVBlw*urtL?yY4e&;96Udh z=Pe(>bM-gSlVWL%`*J}Te>VkgA5BA}$8z-cU3-|hkLx=SwHh$@j${)-)?{WxS zEs~{AF(!0t%N}a>-3+tj7htH&LE&DAH_ocF;?w^s;)s=2%$i%_l~4?~TeZ0HcqWhO z_Qr=REqTIxGde1_fU}$;_*{bvMeo++>-VZ@NZbs3o^e3M$Q4-?!)JS$@m` z*?gh;3)D=rt&0_FA_ zhJi=klZny-v8aA1kFsij9Z5^*ir-T(&zJna&j4;dizVIDa(SxORUuDU3x{8Z@_&Q+ zAr3Ak4i+HhNCFB* Nq6<2)a4`t*0ss=U3Y-7{ literal 0 HcmV?d00001