diff --git a/docs/compiler/options.md b/docs/compiler/options.md index 0e06ca1..da6fe4a 100644 --- a/docs/compiler/options.md +++ b/docs/compiler/options.md @@ -17,7 +17,7 @@ Boolean switches default to off unless stated otherwise. | `--pim-only-codegen` | Run the codegen tail on bufferized Pim IR. | | `--pim-emit-json` | Also emit per-core JSON instructions. | | `--pim-dump-ir` | Write diagnostic IR snapshots with elided large constants; default: off. | -| `--pim-periodic-diagnostics` | Write the optional pre-codegen recurrence report; default: off. | +| `--pim-periodic-diagnostics` | Write optional recurrence and Spatial execution-work reports; default: off. | | `--pim-export-spatial-dataflow=none\|spatial1\|spatial2\|spatial3\|spatial4\|all` | Export graph, trivially merged, scheduled, and realized CSV snapshots under `reports/`; default: `none`. | | `--pim-conv-lowering=STRATEGY` | `auto` (default), `legacy`, `depthwise`, `packed-im2col`, `streamed-patch`, `streamed-packed`, `output-channel-tiled`, `input-k-tiled`, or `tiled-2d`. | | `--pim-conv-im2col-max-elements=N` | Global im2col limit before streaming; default: 1048576. | diff --git a/docs/compiler/scheduling-implementation.md b/docs/compiler/scheduling-implementation.md index 4155ecc..f05ef69 100644 --- a/docs/compiler/scheduling-implementation.md +++ b/docs/compiler/scheduling-implementation.md @@ -72,6 +72,59 @@ byte-sized elements. Blueprint assembly charges per fragment. A full fragment receive directly into assembly storage omits duplicate endpoint-memory edge cost; later copy elimination can leave this estimate conservative. +`CopyAccessPlan` shares ordered contiguous-run geometry between projection +costing and PIM copy normalization. Its inputs are a logical copy shape and +proven source and destination strides, in elements. The immutable result keeps +outer dimensions, run count, run length, and relative offsets compact. Building +the plan takes linear time and space in rank, independent of loop trip counts. +It creates no IR and does not prove aliasing, storage ownership, or copy legality. +PIM bufferization still owns those decisions and the mapping to byte accesses. +The plan alone does not describe a complete scheduled memory-request stream. +Construction rejects an unrepresentable maximum relative element offset, +including the last element of each run. Callers still check byte scaling and +base-address addition. +Projection costing uses checked row-major strides. Unsupported flattened +offsets or strides produce at most one warning per graph cost query. The query +uses one separately rounded request per element as a conservative service bound. +It does not reject otherwise valid Spatial IR. Target lowering retains its own +address and layout legality checks. Unsupported estimates and malformed IR are +different conditions. + +The PIM adapter's `planComputeInputMaterialization` reads scheduled Spatial +regions without changing them. It returns `Unused`, `ForwardConstant`, +`ForwardFunctionInput`, or `CopyAtEntry`. Function-input normalization forwards +host inputs into compute bodies before entry-copy materialization. +Single-core lowering forwards used arithmetic constants. Batch lowering copies +remaining inputs at body entry, including constants. Lowering and costing +consume the same decisions. The query has constant time and space per +input and does not expand lanes or loops. Abstract target work extraction uses +the same query. Subsequent canonicalization can remove uses. +`getReusableOutputTensor` shares read-only destination selection with vector +lowering. It identifies a tensor operand or requires fresh storage. It does not +prove a physical alias, assign an address, or replace one-shot bufferization. +The result is valid only for the current SSA graph. Changes to uses, operation +order, loop yields, or operands require recomputation. Lowering consumes this +decision immediately; it does not cache values across rewrites. Current PIM +bufferization consumes the explicit destination ties introduced by target +lowering. Output-copy forwarding also consumes the resulting allocation, +subview, copy, and dominance facts. An early destination choice alone cannot +describe those physical aliases or the final storage lifetimes. +Logical storage identity and physical address equality are different facts. +Local-memory planning can assign the same address to distinct allocations whose +lifetimes do not overlap. Scalar register setup therefore needs placement and +instruction-order information in addition to logical alias relationships. +Abstract cost estimates need not reconstruct these addresses. + +This input plan does not count physical constant loads. A forwarded host +constant can need a new local buffer at each use. Bufferization, host folding, +alias analysis, and memory normalization determine the resulting requests. +`isZeroBitPatternSplat` shares constant-content classification between graph +accounting and code generation. PIM replaces eligible positive-zero and integer-zero +loads with local fills. Negative floating zero retains its distinct bit pattern. +Constant content is candidate-independent; input placement and buffer reuse +depend on the realized candidate. Production uses these shared decisions when +extracting Spatial execution events for the queue estimate. + Execution transpose costs share `getTransposeRun` with lowering: round memory reads/writes per run and add vector execution when source stride is not one. The [Pimsim vector unit](../../backend-simulators/pim/pimsim-nn/src/core/component/VectorUnit.cpp) @@ -118,3 +171,89 @@ scheduling overview. Cached PEFT center distances avoid a topology scan per task projection and priority analyses share/cache work rather than revisiting every template body per chunk. Ready-transfer signature searches remain another potential compile-time hotspot. + +### Spatial execution costs + +`SchedulingTarget::estimateRealizedScheduleCost` is a read-only query on realized Spatial IR. +The PIM adapter extracts compact ordered execution events without constructing PIM operations. +It uses physical core assignments, communication, input-materialization decisions, and shared copy geometry. +No physical buffer addresses are required. +The shared engine models controller arbitration, source FIFOs, synchronization, and repeated generations. +Both arbitration policies are sensitivity checks, not guaranteed bounds. +The engine returns typed intervals and recurrence evidence before diagnostic serialization. +The target adapter adds static lowering support. Spatial owns the selection policy margin. +JSON fields are report output and do not control availability or eligibility. +Typed evaluation completes before reporting. Serialization never runs queue analysis. +Only optional diagnostics construct fingerprints, traces, and per-core resource reports. + +Normal compilation extracts costs for multiple candidates and lowers only the selected schedule. +A single candidate bypasses the cost query entirely. +Candidate summaries distinguish available timing, static lowering support, and proved modeled recurrence. +Unknown support remains explicit. All candidates must satisfy these checks before comparison. +`Unsupported` identifies a known static violation, including invalid or repeated physical-core assignments. +`Unknown` identifies missing evidence or an unmodeled form, rather than a proven backend rejection. +`Supported` covers checked static forms and does not prove final allocation success. +A finite interval alone does not establish eligibility. + +Preflight does not repeat bufferization or allocation. +The backend owns final legality and signed 32-bit local-memory address validation. +Resource failures marked by that owner permit an explicit retry from the original checkpoint. +Unclassified failures stop compilation. Every attempted schedule and cause appears in the selection report. + +### Complete-state recurrence + +Queue analysis checks translated states at one fixed core's body entry. +Snapshots contain completion counts relative to core entry, ready-input counts, and busy state for every event. +Run-length encoding keeps these records compact. +Snapshots also contain pending events in arbitration order, remaining FIFO tokens, controller availability, and the tie policy. +One pending firing per event makes node IDs sufficient to resolve simultaneous-event ties. +The graph and target parameters remain fixed within each query. +Time offsets use checked unsigned differences. +Equal states require equal positive generation progress on every core. +This proves recurrence for the deterministic abstract model, not physical execution-time accuracy. +Reporting counters and analysis budgets are not hardware state. + +Token initialization distances and graph edges remain immutable within a query. +Consumption clears a drained queue and resets its head immediately. +Drained and never-populated queues therefore have the same readiness state. +Consumed prefixes, capacities, and compaction indices cannot affect future timing or FIFO contents. +An idle controller's past completion cannot delay future arrivals. +These normalizations require causal, monotonic transitions, which the proof checks. + +Proof construction uses at most 256 generations and 16 MiB of snapshot storage. +Each checkpoint scans events, dependencies, and remaining tokens, then orders a copy of the pending frontier. +Exact comparisons use bounded history; hashes only filter comparisons. +Unproved recurrence remains explicit and cannot guide multi-candidate selection. +When recurrence is proved, its interval is authoritative for that arbitration policy and its report. +Proof-mode reports retain interval-window results under `legacy_interval_window`. +Normal costing stops once recurrence is proved. Detailed diagnostics can continue to collect interval-window evidence. +A legacy event or generation limit cannot override a later complete-state proof. +Budget limits, lack of ready events, and unproved recurrence remain separate evidence. +Internal diagnostic queries can cap interval-window work without reducing the recurrence proof budget. +The standalone instruction diagnostic keeps its interval-window behavior when recurrence is not requested. + +### Approximation and diagnostics + +Scalar setup, copy forwarding, local reuse, and host-slice folding can change request readiness. +These effects remain prediction limitations. Physical address reuse itself adds no execution work. +Direct returns use the same return-operand test as lowering and preserve output multiplicity. +Captured weights used by vector operations require host loads. +Publication-only slices of compact results use `CopyAccessPlan` runs without an intermediate copy. +Matrix writeback requires complete aligned array groups. +Partial output groups, encoded layouts, unavailable geometry, and indirect output collection produce unavailable costs. + +`--pim-periodic-diagnostics` adds resource summaries and ordered-request fingerprints. +Normal compilation creates no fingerprints, detailed traces, or sensitivity runs. +Optional local-copy sensitivity removes modeled copy work without changing generated code. +It does not describe a legal transformed program or an interval bound. +The standalone instruction diagnostic remains available for offline timing validation. + +Checked arithmetic rejects unrepresentable sizes, loop bounds, repeated work, and service durations. +Work-only repeats remain compact. Immutable summaries are cached within a query. +Ordered memory and communication occurrences expand within the event budget. +For `P` compact records, `R` expanded events, and `D` dependencies, construction costs `O(P+R+D)` time. +With `B` processed events and `T` token propagations, queue evaluation costs `O(B log R+T)` time. +Storage costs `O(P+R+G D)` for bounded generation count `G`, plus bounded recurrence history. +Reports separate extraction, expansion, and queue-analysis times. +Fingerprints compare memory and blocking-operation order, without addresses or work between requests. +Equal fingerprints therefore do not prove equal request readiness or timing. diff --git a/docs/compiler/scheduling.md b/docs/compiler/scheduling.md index 28b42eb..e9ea623 100644 --- a/docs/compiler/scheduling.md +++ b/docs/compiler/scheduling.md @@ -136,7 +136,8 @@ introduced after deferred realization, including final output collection, is also absent. Instruction issue, local contention, network injection queues, and memory arbitration remain unmodeled. Missing effects are listed in the report; `selection_eligible` remains false. This plan-level diagnostic does not rank -production candidates. Production uses the instruction-level queue model below. +production candidates. Production extracts execution events from realized Spatial +and evaluates them with the shared queue engine. For `V` events, `E` edges, and `R` recurrence edges, construction requires `O(T + F + S log S)` time and `O(V + E + S)` storage, for existing compute @@ -236,7 +237,8 @@ generations. The maximum request wait includes bootstrap. Event-index order resolves simultaneous readiness. A second run reverses that order and reports `alternative_tie_interval_ns`. These two orders are sensitivity checks, not bounds over every possible tie order. Clock phases and frontend overlap remain -approximate. The standalone report remains read-only. Production selection uses the same queue model through its C++ API. +approximate. The standalone report remains read-only. +Production evaluates Spatial-derived events with the shared queue engine. For `R` global requests, the queue mode adds `O(R)` events and dependencies. For `V` resulting events, `E` dependencies, and generation bound `K`, execution @@ -250,120 +252,88 @@ The standalone frontend retains the recurrence witness. Neither analysis changes ## Default throughput selection -Full PIM code generation performs these steps for `--pipeline > 1`. -`--pim-only-codegen` retains the physical schedule already present in its input. +Full PIM code generation uses Spatial-only queue-aware selection for `--pipeline > 1`. +Latency PEFT and `--pim-only-codegen` retain their existing behavior. +Intermediate emission retains the original schedule and stops at its requested stage. +Its schedule can differ from the final program. -The normal compilation workflow is: +The compiler performs these steps: -1. Run the ONNX frontend, layout planning, common lowering, and graph preparation once. -2. Generate at most eleven verified physical schedules from the existing searches. -3. Remove duplicate placements, including their core order and stage identities. -4. Clone the prepared module for each candidate using MLIR's lossless `IRMapping`. -5. Remap each schedule's compute identities to the clone. -6. Realize communication and lower the clone through verification and instruction selection. -7. Generate binary instruction records in memory, without weights, exports, or candidate files. -8. Run both static controller-arbitration tie policies on that instruction program. -9. Select an eligible candidate and emit only its normal program. +1. Run the frontend, layout planning, and graph preparation once. +2. Generate at most eleven legal schedules from the existing searches. +3. Remove duplicate physical placements, including stage identities and core order. +4. Realize each candidate in an independent clone of the prepared checkpoint. +5. Query the target adapter over realized Spatial IR, then release the clone. +6. Require supported static forms, available costs, and proved modeled recurrence for every candidate. +7. Select a candidate with the existing 2.1% selection margin. +8. Realize the selected schedule again from the unchanged checkpoint. +9. Lower, allocate, verify, and emit only the selected program. -Immutable attributes, including constants and weights, remain shared in the MLIR -context. Candidate operations, values, communication plans, and synchronization -state remain independent. Trials never mutate the prepared checkpoint. Candidates -run sequentially. At most one best lowered module survives between trials. -The retained module follows the conservative selection rule throughout evaluation. -A close alternative does not evict candidate zero. Unsupported timing contracts -and synchronization ablation generate only the original schedule and lower it -once. The compiler checks this before collecting alternatives. The frontend is -not repeated. +A single candidate skips event extraction, queue analysis, recurrence proof, and comparison. +It is realized and lowered once, even when optional timing is unavailable. +No rejected candidate enters PIM IR or instruction generation during cost evaluation. +Constants and weights remain shared immutable attributes in the MLIR context. +Candidate operations, communication plans, and synchronization state remain independent. +No full candidate compilation directories are created. -Intermediate targets (`--EmitSpatial`, `--EmitPim`, and `--EmitPimBufferized`) -retain the original schedule. They do not perform queue-aware selection or run -transformations beyond the requested stage. Their schedule can therefore differ -from final code generation. Full PIM code generation uses queue-aware selection. +`SchedulingTarget::estimateRealizedScheduleCost` is the read-only whole-schedule query. +Its typed result carries both intervals, recurrence evidence, static lowering support, +failure reasons, and the selection margin. Unknown support remains explicit. +The target supplies model evidence. Spatial applies the 2.1% selection policy margin. +Selection does not read JSON diagnostics. Reports serialize the typed results. +Spatial owns candidate generation, legality, placement, and target-neutral comparison. +The PIM adapter owns execution-event extraction, instruction timing, and resource contention. +The shared queue engine models controller arrival arbitration and cross-generation dependencies. +The compiler does not execute candidate programs or invoke a simulator. -The PIM compiler owns instruction timing and controller behavior. Spatial owns -legal candidate generation and schedule identity remapping. `PimPeriodicAnalysis` -is the single instruction-level timing implementation. `raptor-periodic` is its -read-only diagnostic frontend. The compiler invokes neither that executable nor -an external selection script. +Both arbitration policies must establish complete-state recurrence within bounded analysis budgets. +Their estimates are sensitivity checks, not formal bounds. +The candidate with the lowest larger estimate must separate from the original schedule. +Comparison uses `[min(ties)/1.021, max(ties)/0.979]`. +Close alternatives retain candidate zero. The 2.1% margin is a policy heuristic. +Spatial timing predictions have no formal accuracy bound. +The search finds the best distinguishable candidate in its bounded set, without global optimality. -`SchedulingTarget::prepareScheduleCost` is an optional whole-schedule cost query. -It accepts a verified target-lowered module and returns an opaque program identity -and a deferred static evaluation. Spatial compares the resulting interval -estimates and applies the target's reported uncertainty. Missing uncertainty -makes an estimate ineligible. The target owns instruction extraction, -protocol timing, contention, and unavailable-result reasons. Preparation and -evaluation do not change the candidate. Equal program identities avoid repeated -queue analysis. An absent query retains the original schedule. -If generation yields one distinct schedule, no comparison is required. The -compiler realizes and lowers it once, without a trial clone, duplicate -instruction generation, or queue analysis. The selection report records this -case separately from unavailable estimates. Final code generation is unchanged. +Unavailable timing, unknown static support, or unproved recurrence stops a multi-candidate comparison. +The diagnostic identifies each affected candidate and property. +The compiler does not silently exclude unknown candidates or use instruction-level fallback selection. -PIM transport details reside in `PimInstructionTiming`. The optional plan-level -diagnostic also belongs to the PIM compiler. Spatial supplies its physical plans -through an injected reporting callback; it does not encode the PIM handshake. -The instruction emitter remains the source of register setup, address-dependent -instruction suppression, loop expansion, and physical array-group identity. -Candidate evaluation uses this emitter in memory after local-memory planning. -It does not reconstruct these rules from scalar Spatial operation costs. +Static preflight does not prove final backend legality. +The selected candidate runs through normal bufferization, allocation, and backend verification. +A backend owner can mark a resource-limit diagnostic as candidate-specific. +Currently, the local-memory allocator marks signed 32-bit address-space exhaustion. +The compiler discards that failed clone, explicitly records its rejection, and selects again from the remaining estimated schedules. +The first remaining schedule becomes the reference for the same selection policy. +Each schedule is attempted at most once. No graph or placement constraints are relaxed. +Unclassified or systemic backend failures stop compilation immediately. +Code-generation failures also stop compilation when recovery cannot be justified. +If every candidate fails, one diagnostic lists all attempted candidates and causes. -Production projects native fp32 transfer sizes and vector widths to the existing -oracle int8 timing contract. It preserves instruction counts, matrix precision, -vector element counts, synchronization, and transfer identities. This projection -does not change the emitted native program. Unsupported widths, targets without -complete timing, and unsupported instruction semantics produce explicit fallback -reasons. Synchronization ablation retains the original schedule. - -Both arbitration tie runs must converge within the existing generation and event -budgets. Their estimates are sensitivity checks, not formal bounds. Duplicate -instruction programs are analyzed once. The candidate with the lowest larger -estimate must separate from the original schedule. Close alternatives do not -cancel a gain that separates from the original. Comparison uses `[min(ties)/1.021, max(ties)/0.979]`. The fixed 2.1% prediction uncertainty -covers offline validation error with upward rounding. It is a selection margin, -not a fitted instruction latency or a guaranteed error bound. Overlapping ranges -with the original retain candidate zero. Missing original estimates also retain candidate zero. -Unavailable alternatives cannot displace an eligible original. This search finds -the best distinguishable candidate in its bounded set; it does not establish -global optimality. - -`reports/pipeline_selection.json` records candidate identities, estimates, -selection reasons, and generation, lowering, instruction-generation, queue-analysis, -and finalization times. The selected verified lowered module and its captured -schedule report are reused without repeating materialization or communication -realization. Memory reports come from normal final code generation. -With `--enable-timing`, candidate summaries also contain per-pass lowering times -and operation/value counts before bufferization and instruction generation. -Instruction-generation summaries separate preparation from per-core emission. -Explicit Spatial exports, IR snapshots, -and plan-level periodic diagnostics reconstruct the selected Spatial schedule -because those outputs require pre-lowering operations. This optional work does -not change the retained instruction program. Candidate instructions and rejected -IR are released immediately. No temporary compilation directories are created. -`--pim-dump-ir` explicitly enables diagnostic IR snapshots; -these snapshots elide large constants and cannot serve as lossless checkpoints. +`reports/pipeline_selection.json` records Spatial costs, eligibility, selection, and backend attempts. +It also records candidate generation, realization, extraction, expansion, queue, lowering, and finalization times. +`--enable-timing` adds per-pass lowering times and IR operation/value counts. +Final instruction generation uses the normal compiler timing report. +Explicit Spatial exports and IR dumps reconstruct pre-lowering artifacts for the selected schedule. +They do not change the emitted program. Diagnostic MLIR dumps elide large constants and are not lossless checkpoints. ### Static analysis, diagnostics, and external validation -Production decisions use only the input program, compiler options, candidate -instructions and plans, and configured target resources and timing. The static -model executes deterministic abstract events. It does not execute the generated -program. It reads no benchmark CSV, profiling trace, measured latency cache, or -reference artifact. Identical static inputs produce the same decision regardless -of historical measurements. Throughput deferred-demand ordering uses IR order, -rather than allocation addresses. Latency PEFT and its demand ordering are unchanged. +Production decisions use the input graph, compiler options, physical plans, and configured target parameters only. +Identical static inputs produce the same decision regardless of historical measurements. +There are no measured-latency caches, workload lookup tables, or fitted timing corrections. -Optional diagnostics expose recurrence witnesses, instruction timing, and resource -service. Offline Pimsim fast-mode measurements validate these predictions and -candidate rankings. Simulator results never enter production selection. Timing -corrections must follow verified hardware behavior or configured parameters. +`raptor-periodic` retains the instruction-derived analyzer as a standalone validation frontend. +It shares the same timing definitions and queue engine with Spatial event evaluation. +Optional diagnostics expose resource work, recurrence, and ordered-request fingerprints. +Offline Pimsim fast-mode runs validate predictions and emitted programs. +Simulator measurements never enter production selection. -For `B <= 11` schedules, `T` compute instances, and `P` cores, captured schedules -use `O(B(T+P))` storage. Placement deduplication takes `O(B²T)` time. Each distinct -finalist requires downstream lowering and one instruction scan. Queue execution -uses the bounded complexity stated above. Peak storage includes the prepared -module, one best lowered module, one trial, and one queue graph. It does not scale -with retained complete candidate programs. Maximum-cycle-ratio diagnostics do not -run in the candidate evaluation loop. +For `C <= 11` candidates, scheduling metadata uses `O(C(T+P))` storage for `T` compute instances and `P` cores. +Placement deduplication costs `O(C²T)` time. +Candidate realization and extraction are sequential; complete candidate modules are not retained. +Normal successful compilation performs one candidate-specific backend lowering. +Recovery performs at most `C` lowerings. Retry selection costs `O(C²)` time and `O(C)` metadata. +The bounded queue-analysis costs are described in the implementation notes. ## Capacity and recovery diff --git a/src/PIM/Common/CMakeLists.txt b/src/PIM/Common/CMakeLists.txt index cc5252f..3c41a9c 100644 --- a/src/PIM/Common/CMakeLists.txt +++ b/src/PIM/Common/CMakeLists.txt @@ -3,6 +3,7 @@ add_pim_library(OMPimCommon IR/AddressAnalysis.cpp IR/BatchCoreUtils.cpp IR/ConstantUtils.cpp + IR/CopyAccessPlan.cpp IR/CoreBlockUtils.cpp IR/EntryPointUtils.cpp IR/IndexingUtils.cpp diff --git a/src/PIM/Common/IR/AddressAnalysis.cpp b/src/PIM/Common/IR/AddressAnalysis.cpp index 0e3eb92..c77e639 100644 --- a/src/PIM/Common/IR/AddressAnalysis.cpp +++ b/src/PIM/Common/IR/AddressAnalysis.cpp @@ -171,7 +171,9 @@ static llvm::FailureOr resolveConstantGlobalLoad(mlir::memref::LoadOp l if (indices.size() != static_cast(globalType.getRank())) return mlir::failure(); - auto strides = computeRowMajorStrides(globalType.getShape()); + auto checkedStrides = computeRowMajorStridesChecked(globalType.getShape()); + if (failed(checkedStrides)) return mlir::failure(); + auto strides = std::move(*checkedStrides); int64_t linearIndex = linearizeIndex(indices, strides); if (linearIndex < 0 || linearIndex >= globalType.getNumElements()) return mlir::failure(); @@ -308,7 +310,9 @@ llvm::FailureOr compileConstantGlobalLoad(mlir::memref::LoadO CompiledIndexExprNode expr; expr.kind = CompiledIndexExprNode::Kind::ConstantGlobalLoad; expr.globalOp = globalOp; - expr.globalStrides = computeRowMajorStrides(globalType.getShape()); + auto checkedStrides = computeRowMajorStridesChecked(globalType.getShape()); + if (failed(checkedStrides)) return mlir::failure(); + expr.globalStrides = std::move(*checkedStrides); expr.operands.reserve(loadOp.getIndices().size()); for (mlir::Value index : loadOp.getIndices()) { auto compiledIndex = compileIndexValueImpl(index); diff --git a/src/PIM/Common/IR/ConstantUtils.cpp b/src/PIM/Common/IR/ConstantUtils.cpp index c1dc1eb..2bdd9b5 100644 --- a/src/PIM/Common/IR/ConstantUtils.cpp +++ b/src/PIM/Common/IR/ConstantUtils.cpp @@ -10,6 +10,18 @@ using namespace mlir; namespace onnx_mlir { +bool isZeroBitPatternSplat(Attribute value) { + auto dense = dyn_cast_or_null(value); + if (!dense || !dense.isSplat()) + return false; + Attribute element = dense.getSplatValue(); + if (auto floating = dyn_cast(element)) + return floating.getValue().isZero() && !floating.getValue().isNegative(); + if (auto integer = dyn_cast(element)) + return integer.getValue().isZero(); + return false; +} + ConstantPool::ConstantPool(Operation *constantAnchor, OpBuilder &builder) : anchor(constantAnchor), block(getConstantInsertionBlock(constantAnchor)), builder(builder) { diff --git a/src/PIM/Common/IR/ConstantUtils.hpp b/src/PIM/Common/IR/ConstantUtils.hpp index 119ce2d..b89a2fa 100644 --- a/src/PIM/Common/IR/ConstantUtils.hpp +++ b/src/PIM/Common/IR/ConstantUtils.hpp @@ -12,6 +12,9 @@ namespace onnx_mlir { +// Negative floating zero has a distinct representation and must not match. +bool isZeroBitPatternSplat(mlir::Attribute value); + class ConstantPool { public: ConstantPool(mlir::Operation *constantAnchor, mlir::OpBuilder &builder); diff --git a/src/PIM/Common/IR/CopyAccessPlan.cpp b/src/PIM/Common/IR/CopyAccessPlan.cpp new file mode 100644 index 0000000..629ffd8 --- /dev/null +++ b/src/PIM/Common/IR/CopyAccessPlan.cpp @@ -0,0 +1,74 @@ +#include "CopyAccessPlan.hpp" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/CheckedArithmetic.h" +#include + +namespace onnx_mlir { + +mlir::FailureOr getContiguousSuffixRank( + llvm::ArrayRef shape, llvm::ArrayRef elementStrides) { + if (shape.size() != elementStrides.size() + || llvm::any_of(shape, [](int64_t size) { return size <= 0; })) + return mlir::failure(); + int64_t expectedStride = 1, rank = 0; + for (size_t dim = shape.size(); dim-- > 0;) { + if (shape[dim] != 1 && elementStrides[dim] != expectedStride) + break; + ++rank; + auto next = llvm::checkedMul(expectedStride, shape[dim]); + if (!next) return mlir::failure(); + expectedStride = *next; + } + return rank; +} + +mlir::FailureOr CopyAccessPlan::build( + llvm::ArrayRef shape, llvm::ArrayRef sourceElementStrides, + llvm::ArrayRef targetElementStrides) { + auto sourceSuffix = getContiguousSuffixRank(shape, sourceElementStrides); + auto targetSuffix = getContiguousSuffixRank(shape, targetElementStrides); + if (mlir::failed(sourceSuffix) || mlir::failed(targetSuffix)) + return mlir::failure(); + size_t outerRank = shape.size() - std::min(*sourceSuffix, *targetSuffix); + CopyAccessPlan plan; + plan.outerShape.assign(shape.begin(), shape.begin() + outerRank); + plan.sourceOuterStrides.assign(sourceElementStrides.begin(), sourceElementStrides.begin() + outerRank); + plan.targetOuterStrides.assign(targetElementStrides.begin(), targetElementStrides.begin() + outerRank); + int64_t elements = 1, maximumSource = 0, maximumTarget = 0; + for (size_t dim = 0; dim < shape.size(); ++dim) { + if (dim < outerRank && (sourceElementStrides[dim] < 0 || targetElementStrides[dim] < 0)) + return mlir::failure(); + int64_t &product = dim < outerRank ? plan.runCount : plan.runElements; + auto next = llvm::checkedMul(product, shape[dim]); + auto total = llvm::checkedMul(elements, shape[dim]); + if (!next || !total) return mlir::failure(); + product = *next; + elements = *total; + // Unit dimensions never access their stride, including negative strides. + if (shape[dim] != 1) { + auto sourceEnd = llvm::checkedMulAdd(shape[dim] - 1, sourceElementStrides[dim], maximumSource); + auto targetEnd = llvm::checkedMulAdd(shape[dim] - 1, targetElementStrides[dim], maximumTarget); + if (!sourceEnd || !targetEnd) return mlir::failure(); + maximumSource = *sourceEnd; + maximumTarget = *targetEnd; + } + } + return plan; +} + +mlir::FailureOr> CopyAccessPlan::getRunOffsets(int64_t run) const { + if (run < 0 || run >= runCount) return mlir::failure(); + int64_t source = 0, target = 0; + for (size_t dim = outerShape.size(); dim-- > 0;) { + int64_t coordinate = run % outerShape[dim]; + run /= outerShape[dim]; + auto nextSource = llvm::checkedMulAdd(coordinate, sourceOuterStrides[dim], source); + auto nextTarget = llvm::checkedMulAdd(coordinate, targetOuterStrides[dim], target); + if (!nextSource || !nextTarget) return mlir::failure(); + source = *nextSource; + target = *nextTarget; + } + return std::pair{source, target}; +} + +} // namespace onnx_mlir diff --git a/src/PIM/Common/IR/CopyAccessPlan.hpp b/src/PIM/Common/IR/CopyAccessPlan.hpp new file mode 100644 index 0000000..4580e8c --- /dev/null +++ b/src/PIM/Common/IR/CopyAccessPlan.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include "mlir/Support/LogicalResult.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include +#include + +namespace onnx_mlir { + +// Geometry only. Callers must prove storage layout, aliasing, and copy legality. +mlir::FailureOr getContiguousSuffixRank( + llvm::ArrayRef shape, llvm::ArrayRef elementStrides); + +class CopyAccessPlan { +public: + // All accessed relative element offsets must fit int64_t, including run tails. + // Byte scaling and addition of base addresses remain caller responsibilities. + static mlir::FailureOr build(llvm::ArrayRef shape, + llvm::ArrayRef sourceElementStrides, + llvm::ArrayRef targetElementStrides); + + int64_t getRunCount() const { return runCount; } + int64_t getRunElements() const { return runElements; } + llvm::ArrayRef getOuterShape() const { return outerShape; } + llvm::ArrayRef getSourceOuterStrides() const { return sourceOuterStrides; } + llvm::ArrayRef getTargetOuterStrides() const { return targetOuterStrides; } + // Source and target element offsets, relative to their separately proven bases. + mlir::FailureOr> getRunOffsets(int64_t run) const; + +private: + int64_t runCount = 1; + int64_t runElements = 1; + llvm::SmallVector outerShape, sourceOuterStrides, targetOuterStrides; +}; + +} // namespace onnx_mlir diff --git a/src/PIM/Common/IR/ShapeUtils.cpp b/src/PIM/Common/IR/ShapeUtils.cpp index 5e0fd55..9cc396c 100644 --- a/src/PIM/Common/IR/ShapeUtils.cpp +++ b/src/PIM/Common/IR/ShapeUtils.cpp @@ -1,6 +1,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/CheckedArithmetic.h" #include @@ -9,9 +10,20 @@ namespace onnx_mlir { llvm::SmallVector computeRowMajorStrides(llvm::ArrayRef shape) { + auto strides = computeRowMajorStridesChecked(shape); + if (mlir::failed(strides)) + llvm::report_fatal_error("row-major strides are not representable in int64_t"); + return std::move(*strides); +} + +mlir::FailureOr> +computeRowMajorStridesChecked(llvm::ArrayRef shape) { llvm::SmallVector strides(shape.size(), 1); - for (int64_t dim = static_cast(shape.size()) - 2; dim >= 0; --dim) - strides[dim] = strides[dim + 1] * shape[dim + 1]; + for (int64_t dim = static_cast(shape.size()) - 2; dim >= 0; --dim) { + auto stride = llvm::checkedMul(strides[dim + 1], shape[dim + 1]); + if (!stride) return mlir::failure(); + strides[dim] = *stride; + } return strides; } diff --git a/src/PIM/Common/IR/ShapeUtils.hpp b/src/PIM/Common/IR/ShapeUtils.hpp index ce1be65..b097c37 100644 --- a/src/PIM/Common/IR/ShapeUtils.hpp +++ b/src/PIM/Common/IR/ShapeUtils.hpp @@ -19,7 +19,11 @@ namespace onnx_mlir { using HSliceId = size_t; using CoreId = size_t; +// Invariant form fails explicitly on overflow, including assertion-free builds. +// Optional analyses and rewrites must use the checked form. llvm::SmallVector computeRowMajorStrides(llvm::ArrayRef shape); +mlir::FailureOr> +computeRowMajorStridesChecked(llvm::ArrayRef shape); struct TransposeRun { size_t start; diff --git a/src/PIM/Common/IR/WeightUtils.cpp b/src/PIM/Common/IR/WeightUtils.cpp index ed7b775..96e1f5d 100644 --- a/src/PIM/Common/IR/WeightUtils.cpp +++ b/src/PIM/Common/IR/WeightUtils.cpp @@ -191,7 +191,9 @@ resolveWeightView(mlir::Operation* coreLikeOp, mlir::Value weight, const StaticV ResolvedWeightView view; view.globalOp = globalOp; view.shape.assign(denseAttr.getType().getShape().begin(), denseAttr.getType().getShape().end()); - view.strides = computeRowMajorStrides(view.shape); + auto checkedStrides = computeRowMajorStridesChecked(view.shape); + if (failed(checkedStrides)) return mlir::failure(); + view.strides = std::move(*checkedStrides); CompiledIndexExpr offsetExpr = makeConstantExpr(0); for (mlir::Operation* viewOp : llvm::reverse(viewOps)) { diff --git a/src/PIM/Common/Support/Diagnostics.cpp b/src/PIM/Common/Support/Diagnostics.cpp index 3a5d0f3..e00e40f 100644 --- a/src/PIM/Common/Support/Diagnostics.cpp +++ b/src/PIM/Common/Support/Diagnostics.cpp @@ -4,6 +4,21 @@ namespace onnx_mlir::pim { +static constexpr llvm::StringLiteral candidateResourceFailure = "pim.candidate-resource-limit"; + +mlir::InFlightDiagnostic emitCandidateResourceError(mlir::Operation *op, llvm::StringRef message) { + auto diagnostic = op->emitError(message); + diagnostic.getUnderlyingDiagnostic()->getMetadata().emplace_back(llvm::StringRef(candidateResourceFailure)); + return diagnostic; +} + +bool isCandidateResourceError(mlir::Diagnostic &diagnostic) { + return llvm::any_of(diagnostic.getMetadata(), [](const mlir::DiagnosticArgument &argument) { + return argument.getKind() == mlir::DiagnosticArgument::DiagnosticArgumentKind::String + && argument.getAsString() == candidateResourceFailure; + }); +} + mlir::InFlightDiagnostic emitUnsupportedStaticShapeDiagnostic(mlir::Operation* op, llvm::StringRef valueDescription) { return op->emitOpError() << "requires statically shaped " << valueDescription; } diff --git a/src/PIM/Common/Support/Diagnostics.hpp b/src/PIM/Common/Support/Diagnostics.hpp index 2eeebf7..a405636 100644 --- a/src/PIM/Common/Support/Diagnostics.hpp +++ b/src/PIM/Common/Support/Diagnostics.hpp @@ -12,6 +12,10 @@ namespace onnx_mlir::pim { +/// Only a backend owner can classify a failed schedule as resource-infeasible. +mlir::InFlightDiagnostic emitCandidateResourceError(mlir::Operation *op, llvm::StringRef message); +bool isCandidateResourceError(mlir::Diagnostic &diagnostic); + struct CappedDiagnosticReporter { explicit CappedDiagnosticReporter(int64_t maxReportedFailures = 8) : maxReportedFailures(maxReportedFailures) {} diff --git a/src/PIM/Compiler/CMakeLists.txt b/src/PIM/Compiler/CMakeLists.txt index c84fe34..53cb9a6 100644 --- a/src/PIM/Compiler/CMakeLists.txt +++ b/src/PIM/Compiler/CMakeLists.txt @@ -17,6 +17,8 @@ add_pim_library(OMPimCompilerUtils PimCompilerUtils.cpp PimScheduleDiagnostic.cpp PimPeriodicAnalysis.cpp + PimQueueReporting.cpp + PimSpatialCost.cpp PimPipelineSelection.cpp PimArtifactWriter.cpp PimCodeGen.cpp diff --git a/src/PIM/Compiler/PimBinaryFormat.hpp b/src/PIM/Compiler/PimBinaryFormat.hpp index bff1209..ea4278c 100644 --- a/src/PIM/Compiler/PimBinaryFormat.hpp +++ b/src/PIM/Compiler/PimBinaryFormat.hpp @@ -14,6 +14,7 @@ namespace onnx_mlir::pim_binary { inline constexpr char kMagic[4] = {'P', 'I', 'M', 'B'}; inline constexpr uint32_t kVersion = 1; +inline constexpr int32_t kMatrixWeightPrecisionBits = 8; inline constexpr uint64_t kCountOffset = 8; inline constexpr size_t kHeaderSize = 12; inline constexpr size_t kRecordSize = 20; diff --git a/src/PIM/Compiler/PimCodeGen.cpp b/src/PIM/Compiler/PimCodeGen.cpp index 0539ac8..92dbf8f 100644 --- a/src/PIM/Compiler/PimCodeGen.cpp +++ b/src/PIM/Compiler/PimCodeGen.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -32,6 +31,7 @@ #include #include "Common/IR/CompactAsmUtils.hpp" +#include "Common/IR/ConstantUtils.hpp" #include "Common/PimCommon.hpp" #include "Common/Support/CheckedArithmetic.hpp" #include "Common/Support/Diagnostics.hpp" @@ -42,7 +42,6 @@ #include "src/Accelerators/PIM/Common/Support/FileSystemUtils.hpp" #include "src/Accelerators/PIM/Compiler/PimArtifactWriter.hpp" #include "src/Accelerators/PIM/Compiler/PimBinaryFormat.hpp" -#include "src/Accelerators/PIM/Compiler/PimPeriodicAnalysis.hpp" #include "src/Accelerators/PIM/Compiler/PimCodeGen.hpp" #include "src/Accelerators/PIM/Compiler/PimCompilerOptions.hpp" #include "src/Accelerators/PIM/Compiler/PimCoreProgram.hpp" @@ -116,15 +115,7 @@ static bool isZeroSplatGlobal(mlir::Value value) { auto globalOp = moduleOp ? lookupGlobalForGetGlobal(moduleOp, getGlobalOp) : memref::GlobalOp(); if (!globalOp || !globalOp.getConstant() || !globalOp.getInitialValue()) return false; - auto denseAttr = dyn_cast(*globalOp.getInitialValue()); - if (!denseAttr || !denseAttr.isSplat()) - return false; - Attribute valueAttr = denseAttr.getSplatValue(); - if (auto floatAttr = dyn_cast(valueAttr)) - return floatAttr.getValue().isZero() && !floatAttr.getValue().isNegative(); - if (auto integerAttr = dyn_cast(valueAttr)) - return integerAttr.getValue().isZero(); - return false; + return isZeroBitPatternSplat(*globalOp.getInitialValue()); } // Pim instruction immediates are serialized as signed int32_t fields today @@ -613,7 +604,7 @@ void PimCodeGen::emitMvmOp(size_t groupId, size_t rdAddr, size_t rdOffset, size_ instruction.opcode = pim_binary::Opcode::mvmul; instruction.rd = 0; instruction.r1 = 1; - instruction.r2OrImm = 8; + instruction.r2OrImm = pim_binary::kMatrixWeightPrecisionBits; instruction.generic1 = 0; instruction.generic2 = pim::checkedI32OrCrash(groupId, "mvm group id"); emitInstruction(instruction); @@ -895,7 +886,6 @@ struct CoreEmissionResult { OnnxMlirCompilerErrorCodes status = CompilerSuccess; MemoryReportRow reportRow; - std::string instructions; llvm::SmallVector usedWeights; llvm::SmallVector diagnostics; size_t diagnosticCount = 0; @@ -1150,10 +1140,8 @@ static OnnxMlirCompilerErrorCodes emitEmptyCoreArtifacts(StringRef outputDirPath return CompilerSuccess; } -OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::string& outputDirPath, - PimInstructionProgram *instructions, PimCodegenTiming *timing) { - auto start = std::chrono::steady_clock::now(); - if (!instructions && !outputDirPath.empty()) { +OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std::string& outputDirPath) { + if (!outputDirPath.empty()) { if (auto error = sys::fs::create_directory(outputDirPath)) { errs() << "Error creating output directory: " << outputDirPath << ": " << error.message() << '\n'; return InvalidOutputFileAccess; @@ -1169,9 +1157,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std:: memory.hostMem.allocateHost(moduleOp, funcOp); memory.reportHost(); - if (!instructions) - if (auto err = writeMemoryBinary(moduleOp, funcOp, memory, outputDirPath)) - return err; + if (auto err = writeMemoryBinary(moduleOp, funcOp, memory, outputDirPath)) + return err; json::Object xbarsPerArrayGroup; size_t maxCoreId = 0; @@ -1301,22 +1288,16 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std:: std::error_code errorCode; auto outputCorePath = outputDirPath + "/core_" + std::to_string(job.physicalCoreId) + ".pim"; - llvm::SmallVector binary; - raw_svector_ostream memoryStream(binary); - std::unique_ptr fileStream; - if (!instructions) { - fileStream = std::make_unique(outputCorePath, errorCode, sys::fs::OF_None); - if (errorCode) { - errs() << "Error opening core file: " << outputCorePath << ": " << errorCode.message() << '\n'; - result.status = InvalidOutputFileAccess; - return result; - } + auto fileStream = std::make_unique(outputCorePath, errorCode, sys::fs::OF_None); + if (errorCode) { + errs() << "Error opening core file: " << outputCorePath << ": " << errorCode.message() << '\n'; + result.status = InvalidOutputFileAccess; + return result; } - raw_pwrite_stream &coreBinaryStream = instructions - ? static_cast(memoryStream) : *fileStream; + raw_pwrite_stream &coreBinaryStream = *fileStream; std::unique_ptr coreJsonStream; - if (!instructions && pimEmitJson.getValue()) { + if (pimEmitJson.getValue()) { std::string outputCoreJsonPath = outputDirPath + "/core_" + std::to_string(job.physicalCoreId) + ".json"; errorCode = std::error_code(); coreJsonStream = std::make_unique(outputCoreJsonPath, errorCode); @@ -1332,12 +1313,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std:: PimCodeGen coreCodeGen(jobMemory, instructionWriter, coreJsonStream.get()); auto finalizeInstructions = [&]() { bool succeeded = mlir::succeeded(instructionWriter.finalize()); - if (fileStream) { - fileStream->close(); - if (fileStream->has_error()) { fileStream->clear_error(); succeeded = false; } - } else { - result.instructions.assign(binary.begin(), binary.end()); - } + fileStream->close(); + if (fileStream->has_error()) { fileStream->clear_error(); succeeded = false; } return succeeded; }; @@ -1390,14 +1367,8 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std:: }; std::vector jobResults(jobs.size()); - auto emissionStart = std::chrono::steady_clock::now(); - if (timing) - timing->preparationSeconds = std::chrono::duration(emissionStart - start).count(); mlir::parallelFor( moduleOp.getContext(), 0, jobs.size(), [&](size_t index) { jobResults[index] = emitJob(jobs[index]); }); - if (timing) - timing->emissionSeconds = std::chrono::duration( - std::chrono::steady_clock::now() - emissionStart).count(); pim::CappedDiagnosticReporter diagnostics; Operation* summaryAnchor = nullptr; @@ -1427,33 +1398,13 @@ OnnxMlirCompilerErrorCodes onnx_mlir::compileToPimCode(ModuleOp& moduleOp, std:: physicalCoreId < activePhysicalCores.size(); ++physicalCoreId) { if (activePhysicalCores[physicalCoreId]) continue; - if (!instructions) - if (auto err = emitEmptyCoreArtifacts(outputDirPath, physicalCoreId)) - return err; + if (auto err = emitEmptyCoreArtifacts(outputDirPath, physicalCoreId)) + return err; xbarsPerArrayGroup["core" + std::to_string(physicalCoreId)] = json::Array {}; memory.recordCoreReport(physicalCoreId, MemoryReportRow {}); } - if (instructions) { - instructions->instructions.resize(maxPhysicalCoreId + 1); - instructions->arrayGroups.resize(maxPhysicalCoreId + 1); - for (size_t core = 0; core <= maxPhysicalCoreId; ++core) { - instructions->arrayGroups[core].clear(); - llvm::SmallVector empty; - raw_svector_ostream stream(empty); - pim_binary::writeHeader(stream); - instructions->instructions[core].assign(empty.begin(), empty.end()); - } - for (size_t i = 0; i < jobs.size(); ++i) { - size_t core = jobs[i].physicalCoreId; - instructions->instructions[core] = std::move(jobResults[i].instructions); - for (const auto &weight : jobResults[i].usedWeights) - instructions->arrayGroups[core].push_back(weight.shape[1] / crossbarSize.getValue()); - } - return CompilerSuccess; - } - llvm::SmallVector weightRequests; weightRequests.reserve(jobs.size()); for (size_t jobIndex = 0; jobIndex < jobs.size(); ++jobIndex) { diff --git a/src/PIM/Compiler/PimCodeGen.hpp b/src/PIM/Compiler/PimCodeGen.hpp index 85bf13a..2186a2f 100644 --- a/src/PIM/Compiler/PimCodeGen.hpp +++ b/src/PIM/Compiler/PimCodeGen.hpp @@ -25,7 +25,6 @@ namespace onnx_mlir { struct CompiledCoreProgram; -struct PimInstructionProgram; class PimInstructionWriter; struct MemEntry { @@ -227,13 +226,7 @@ public: }; -struct PimCodegenTiming { - double preparationSeconds = 0; - double emissionSeconds = 0; -}; - -OnnxMlirCompilerErrorCodes compileToPimCode(mlir::ModuleOp& moduleOpRef, std::string& outputDirName, - PimInstructionProgram *instructions = nullptr, PimCodegenTiming *timing = nullptr); +OnnxMlirCompilerErrorCodes compileToPimCode(mlir::ModuleOp& moduleOpRef, std::string& outputDirName); } // namespace onnx_mlir diff --git a/src/PIM/Compiler/PimCompilerOptions.cpp b/src/PIM/Compiler/PimCompilerOptions.cpp index cdcbdb1..cc8cf5c 100644 --- a/src/PIM/Compiler/PimCompilerOptions.cpp +++ b/src/PIM/Compiler/PimCompilerOptions.cpp @@ -90,7 +90,7 @@ llvm::cl::opt pimReportConvLowering("pim-report-conv-lowering", llvm::cl::cat(OnnxMlirOptions)); llvm::cl::opt pimPeriodicDiagnostics("pim-periodic-diagnostics", - llvm::cl::desc("Emit the optional pre-codegen recurrence timing report"), + llvm::cl::desc("Emit optional Spatial recurrence and execution-work reports"), llvm::cl::init(false), llvm::cl::cat(OnnxMlirOptions)); llvm::cl::opt pimEmitJson("pim-emit-json", diff --git a/src/PIM/Compiler/PimCompilerUtils.cpp b/src/PIM/Compiler/PimCompilerUtils.cpp index 45beba5..28b53c9 100644 --- a/src/PIM/Compiler/PimCompilerUtils.cpp +++ b/src/PIM/Compiler/PimCompilerUtils.cpp @@ -16,7 +16,7 @@ #include "src/Accelerators/PIM/Compiler/PimCompilerOptions.hpp" #include "PimPipelineSelection.hpp" #include "PimInstructionTiming.hpp" -#include "PimPeriodicAnalysis.hpp" +#include "PimSpatialCost.hpp" #include "src/Accelerators/PIM/Compiler/PimCompilerUtils.hpp" #include "src/Accelerators/PIM/Conversion/ONNXToSpatial/ONNXToSpatialOptions.hpp" #include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp" diff --git a/src/PIM/Compiler/PimDependencyTokens.hpp b/src/PIM/Compiler/PimDependencyTokens.hpp new file mode 100644 index 0000000..aa7d0e2 --- /dev/null +++ b/src/PIM/Compiler/PimDependencyTokens.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include +#include + +namespace onnx_mlir { + +// Emptiness is normalized after consumption. Capacity and consumed prefixes +// cannot affect virtual timing, readiness, or the remaining FIFO sequence. +class PimDependencyTokens { + llvm::SmallVector values; + size_t head = 0; +public: + void initialize(size_t distance) { values.assign(distance, 0); head = 0; } + bool empty() const { return values.empty(); } + void push(uint64_t time) { values.push_back(time); } + uint64_t consume() { + assert(!empty()); + uint64_t time = values[head++]; + if (head == values.size()) { values.clear(); head = 0; } + else if (head >= values.size() - head) { + values.erase(values.begin(), values.begin() + head); + head = 0; + } + return time; + } + llvm::ArrayRef remaining() const { return llvm::ArrayRef(values).drop_front(head); } +}; + +} // namespace onnx_mlir diff --git a/src/PIM/Compiler/PimInstructionTiming.hpp b/src/PIM/Compiler/PimInstructionTiming.hpp index e6c40ad..be3e173 100644 --- a/src/PIM/Compiler/PimInstructionTiming.hpp +++ b/src/PIM/Compiler/PimInstructionTiming.hpp @@ -6,6 +6,7 @@ namespace onnx_mlir { // PIM transport and instruction timing. Spatial consumes only the base costs. struct PimInstructionTiming : spatial::SchedulingTarget { + Cost matrixCellBits = 0; bool peerPermissionThenSourceRead = false; Cost signalIssueNs = 0; Cost controlMessageBytes = 0; @@ -22,15 +23,17 @@ struct PimInstructionTiming : spatial::SchedulingTarget { && localMemoryWidthBytes && transferWidthBytes; } - Cost getLinkServiceNs(Cost bytes, size_t source, size_t destination) const { - Cost flits = bytes / transferWidthBytes + (bytes % transferWidthBytes != 0); - return checkedMultiply(flits, + template + Cost getLinkServiceNs(Cost bytes, size_t source, size_t destination, Arithmetic math = {}) const { + Cost flits = math.divideUp(bytes, transferWidthBytes); + return math.multiply(flits, getInterProcessorLatencyNs(source, destination)); } - Cost getSharedMemoryLinkServiceNs(Cost bytes, size_t core, bool response) const { - Cost flits = bytes / transferWidthBytes + (bytes % transferWidthBytes != 0); - return checkedMultiply(flits, response ? sharedMemoryResponseLatencyNs[core] + template + Cost getSharedMemoryLinkServiceNs(Cost bytes, size_t core, bool response, Arithmetic math = {}) const { + Cost flits = math.divideUp(bytes, transferWidthBytes); + return math.multiply(flits, response ? sharedMemoryResponseLatencyNs[core] : sharedMemoryRequestLatencyNs[core]); } diff --git a/src/PIM/Compiler/PimPeriodicAnalysis.cpp b/src/PIM/Compiler/PimPeriodicAnalysis.cpp index 7caa22a..e14c843 100644 --- a/src/PIM/Compiler/PimPeriodicAnalysis.cpp +++ b/src/PIM/Compiler/PimPeriodicAnalysis.cpp @@ -1,9 +1,7 @@ #include "PimPeriodicAnalysis.hpp" +#include "PimDependencyTokens.hpp" #include "PimInstructionTiming.hpp" -#include "PimCodeGen.hpp" -#include "PimCompilerUtils.hpp" -#include "PimCompilerOptions.hpp" -#include "PimScheduleDiagnostic.hpp" +#include "PimTimingArithmetic.hpp" #include "llvm/Support/SHA256.h" #include "llvm/ADT/StringExtras.h" #include @@ -12,6 +10,7 @@ #include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/DenseMap.h" #include #include #include @@ -21,6 +20,11 @@ using namespace onnx_mlir; using namespace onnx_mlir::spatial; using Opcode = pim_binary::Opcode; +using pim_timing::add; +using pim_timing::mul; +using pim_timing::sub; +using pim_timing::ceil; +using pim_timing::arithmetic; namespace { using Key = std::pair; struct Event { int64_t core; uint32_t pc; llvm::StringRef kind; }; @@ -33,14 +37,65 @@ struct Core { size_t entry, tail, firstNetwork, lastNetwork; Cost pending = 0, work = 0, instructions = 0; std::array service{}; + std::array traceWork{}; + llvm::SHA256 requests{}, blockingOrder{}; + struct RequestRun { bool write; Cost bytes, count; }; + std::vector requestRuns{}; bool blockingNetworkInChunk = false; }; constexpr size_t absent = std::numeric_limits::max(); -llvm::json::Object estimateQueuedPeriod(const std::vector &events, +// The state excludes counters used only for reporting and analysis budgets. +// Equal translated states and equal progress on every core prove a periodic +// execution of this deterministic model, not a hardware prediction bound. +class PeriodicStateHistory { + struct Snapshot { + std::vector state; + std::vector generations; + Cost time; + uint64_t hash; + }; + std::vector history; + size_t bytes = 0; +public: + bool exhausted = false; + ScheduleRecurrenceEvidence evidence{ScheduleRecurrenceEvidence::Unproved, 0, std::nullopt, {}}; + size_t stateWords = 0; + size_t checkpointCount() const { return history.size(); } + size_t historyBytes() const { return bytes; } + bool observe(std::vector state, std::vector generations, Cost time) { + uint64_t hash = 14695981039346656037ULL; + for (uint64_t item : state) hash = (hash ^ item) * 1099511628211ULL; + for (const auto &previous : history) { + if (hash != previous.hash || state != previous.state || time <= previous.time) continue; + size_t period = generations.front() - previous.generations.front(); + if (!period) continue; + bool uniform = true; + for (size_t i = 0; i < generations.size(); ++i) + uniform &= generations[i] - previous.generations[i] == period; + if (!uniform) continue; + evidence.status = ScheduleRecurrenceEvidence::Proved; + evidence.periodGenerations = period; + evidence.intervalNs = (time - previous.time) / static_cast(period); + stateWords = state.size(); + return true; + } + size_t added = add(mul(add(state.capacity(), generations.capacity()), sizeof(uint64_t)), sizeof(Snapshot)); + if (add(bytes, added) > 16 * 1024 * 1024) { + exhausted = true; + evidence.reason = "periodic-state history exceeds 16 MiB"; + return false; + } + bytes += added; + history.push_back({std::move(state), std::move(generations), time, hash}); + return false; + } +}; + +PimQueuePolicyResult estimateQueuedPeriod(const std::vector &events, const std::vector &edges, const std::vector> &services, const std::map &cores, - bool reverseTies) { + bool reverseTies, bool verifyState, bool diagnostics, std::optional intervalWindowBudget) { struct Node { llvm::SmallVector incoming, outgoing; size_t completed = 0, readyInputs = 0; @@ -51,16 +106,12 @@ llvm::json::Object estimateQueuedPeriod(const std::vector &events, size_t node; bool arrival; }; - struct Tokens { - llvm::SmallVector values; - size_t head = 0; - }; auto later = [reverseTies](const Pending &a, const Pending &b) { if (a.time != b.time) return a.time > b.time; return reverseTies ? a.node < b.node : a.node > b.node; }; std::vector nodes(events.size()); - std::vector tokens(edges.size()); + std::vector tokens(edges.size()); std::vector service(events.size(), 0); std::vector memoryAccess(events.size(), false); std::vector entries(events.size(), false); @@ -72,7 +123,7 @@ llvm::json::Object estimateQueuedPeriod(const std::vector &events, for (auto [index, edge] : llvm::enumerate(edges)) { nodes[edge.source].outgoing.push_back(index); nodes[edge.target].incoming.push_back(index); - tokens[index].values.assign(edge.distance, 0); + tokens[index].initialize(edge.distance); nodes[edge.target].readyInputs += edge.distance != 0; } std::priority_queue, decltype(later)> pending(later); @@ -82,43 +133,177 @@ llvm::json::Object estimateQueuedPeriod(const std::vector &events, Cost time = 0; for (size_t edge : state.incoming) { auto &queue = tokens[edge]; - time = std::max(time, queue.values[queue.head++]); - if (queue.head == queue.values.size()) { - queue.values.clear(); queue.head = 0; --state.readyInputs; - } else if (queue.head * 2 >= queue.values.size()) { - queue.values.erase(queue.values.begin(), queue.values.begin() + queue.head); - queue.head = 0; - } + Cost ready = queue.consume(); + time = std::max(time, ready); + if (queue.empty()) --state.readyInputs; } state.busy = true; - pending.push({time - service[node], node, memoryAccess[node]}); + pending.push({sub(time, service[node]), node, memoryAccess[node]}); }; for (size_t node = 0; node < nodes.size(); ++node) schedule(node); - constexpr size_t maximumRounds = 64; - std::array roundEnd{}; - std::array roundCount{}; - std::map> queueWait; + const size_t maximumRounds = verifyState ? 256 : 64; + std::vector roundEnd(maximumRounds); + std::vector roundCount(maximumRounds); + std::map> queueWait; Cost available = 0; Cost maximumQueueWait = 0; size_t maximumQueueNode = 0; size_t processed = 0; - const size_t budget = 16 * (events.size() + edges.size()); - llvm::json::Object result{{"status", "unavailable"}}; + const size_t defaultBudget = mul(16, add(events.size(), edges.size())); + const size_t legacyBudget = std::min(defaultBudget, intervalWindowBudget.value_or(defaultBudget)); + const size_t budget = verifyState ? mul(32, defaultBudget) : legacyBudget; + PimQueueIntervalWindow result; + std::optional legacy; + std::optional generationLimit; + PeriodicStateHistory history; + bool proved = false, monotonic = true; + size_t proofProcessedEvents = 0; + Cost previousTime = 0; + auto location = [&](size_t index) { + return PimQueueEventLocation{events[index].core, events[index].pc, events[index].kind.str()}; + }; + auto progress = [&](bool includeIntervals) { + PimQueueProgress value; + value.processedEvents = processed; + size_t generations = maximumRounds; + for (const auto &[core, state] : cores) + generations = std::min(generations, nodes[state.entry].completed); + value.completedGenerations = generations ? generations - 1 : 0; + value.maximumQueueWaitNs = maximumQueueWait; + if (maximumQueueWait) value.maximumQueueRequest = location(maximumQueueNode); + value.generationLimit = generationLimit; + if (diagnostics && value.completedGenerations >= 4) + for (const auto &[core, values] : queueWait) { + Cost wait = 0; + for (size_t generation = value.completedGenerations - 4; + generation < value.completedGenerations; ++generation) + wait = add(wait, values[generation]); + value.perCoreQueueWaitNs.emplace_back(core, wait / 4.0); + } + if (includeIntervals) + for (size_t round = 1; round < maximumRounds && roundCount[round] == cores.size(); ++round) + value.completedIntervalsNs.push_back(sub(roundEnd[round], roundEnd[round - 1])); + return value; + }; + auto finish = [&]() { + PimQueuePolicyResult value; + value.progress = progress(false); + value.reverseTies = reverseTies; + value.legacy = legacy ? std::move(*legacy) : std::move(result); + value.termination = value.legacy.termination; + if (verifyState) { + if (proved) value.termination = PimQueueTermination::RecurrenceProved; + else if (history.exhausted) value.termination = PimQueueTermination::HistoryLimit; + else if (!monotonic) value.termination = PimQueueTermination::NonMonotonic; + else if (generationLimit) value.termination = PimQueueTermination::GenerationLimit; + else value.termination = pending.empty() ? PimQueueTermination::NoReadyEvents + : PimQueueTermination::EventBudget; + if (!proved && history.evidence.reason.empty()) + history.evidence.reason = value.termination == PimQueueTermination::NoReadyEvents + ? "queue execution has no ready events before complete-state recurrence" + : "no complete translated state repeated within the bounded analysis"; + value.recurrence = std::move(history.evidence); + value.checkpoints = history.checkpointCount(); + value.historyBytes = history.historyBytes(); + value.stateWords = history.stateWords; + value.proofProcessedEvents = proofProcessedEvents; + } + return value; + }; + std::vector eventEntries; + if (verifyState) { + eventEntries.reserve(events.size()); + for (const auto &event : events) eventEntries.push_back(cores.at(event.core).entry); + // Removing controller service from readiness must preserve causality. + for (const auto &edge : edges) + if (edge.delayNs < service[edge.target]) { + monotonic = false; + history.evidence.reason = "memory readiness can precede its triggering event"; + } + } + auto checkpoint = [&](Cost time) { + std::vector state{static_cast(reverseTies)}; + auto relative = [&](Cost value) { + state.push_back(value < time); + state.push_back(value < time ? sub(time, value) : sub(value, time)); + }; + // RLE preserves completion counters, ready counts, and busy state for + // every node. Only the common inference-generation offset is removed. + size_t previousRun = 0; + for (size_t i = 0; i < nodes.size(); ++i) { + const auto &node = nodes[i]; + size_t entry = nodes[eventEntries[i]].completed; + uint64_t sign = node.completed < entry; + uint64_t offset = sign ? entry - node.completed : node.completed - entry; + if (i && state[previousRun + 1] == sign && state[previousRun + 2] == offset + && state[previousRun + 3] == node.readyInputs && state[previousRun + 4] == node.busy) + ++state[previousRun]; + else { + previousRun = state.size(); + state.insert(state.end(), {1, sign, offset, node.readyInputs, node.busy}); + } + } + state.push_back(0); // End of the positive-length node-state runs. + // An idle controller's past completion time cannot delay a future arrival. + relative(std::max(available, time)); + auto frontier = pending; + state.push_back(frontier.size()); + while (!frontier.empty()) { + auto event = frontier.top(); frontier.pop(); + state.insert(state.end(), {event.node, event.arrival}); + relative(event.time); + } + // Consumed prefixes and head indices only control vector compaction. + // Remaining FIFO values completely determine future token consumption. + for (size_t edge = 0; edge < tokens.size(); ++edge) { + auto values = tokens[edge].remaining(); + if (values.empty()) continue; + state.insert(state.end(), {edge, values.size()}); + for (auto time : values) relative(time); + } + std::vector generations; + for (const auto &[core, entry] : cores) generations.push_back(nodes[entry.entry].completed); + return history.observe(std::move(state), std::move(generations), time); + }; while (!pending.empty() && processed++ < budget) { auto next = pending.top(); pending.pop(); + if (verifyState && next.time < previousTime) { + monotonic = false; + history.evidence.reason = "event processing is not monotonic; translated-state proof unavailable"; + } + previousTime = next.time; auto &node = nodes[next.node]; + if (verifyState && !legacy && processed > legacyBudget) { + legacy.emplace(); + legacy->reason = "queue diagnostic did not converge within its event budget"; + legacy->termination = PimQueueTermination::EventBudget; + legacy->progress = progress(true); + } + if (verifyState && !legacy && node.completed >= 64) { + legacy.emplace(); + legacy->reason = "queue diagnostic exceeded its per-event generation bound"; + legacy->termination = PimQueueTermination::GenerationLimit; + legacy->progress = progress(false); + legacy->progress.generationLimit = location(next.node); + } if (node.completed >= maximumRounds) { - result["reason"] = "queue diagnostic exceeded its per-event generation bound"; - return result; + result.reason = "queue diagnostic exceeded its per-event generation bound"; + result.termination = PimQueueTermination::GenerationLimit; + generationLimit = location(next.node); + result.progress = progress(false); + return finish(); } if (next.arrival) { Cost begin = std::max(available, next.time); - if (begin - next.time > maximumQueueWait) { - maximumQueueWait = begin - next.time; + if (sub(begin, next.time) > maximumQueueWait) { + maximumQueueWait = sub(begin, next.time); maximumQueueNode = next.node; } - queueWait[events[next.node].core][node.completed] += begin - next.time; - available = begin + service[next.node]; + if (diagnostics) { + auto &values = queueWait.try_emplace(events[next.node].core, maximumRounds, 0).first->second; + values[node.completed] = add(values[node.completed], sub(begin, next.time)); + } + available = add(begin, service[next.node]); pending.push({available, next.node, false}); continue; } @@ -131,41 +316,43 @@ llvm::json::Object estimateQueuedPeriod(const std::vector &events, Cost first = roundEnd[round - 2] - roundEnd[round - 4]; Cost second = roundEnd[round] - roundEnd[round - 2]; Cost maximum = std::max(first, second); - if (maximum && std::abs(static_cast(first) - second) <= maximum * 0.01) { - result["status"] = "estimated"; - result["interval_ns"] = (roundEnd[round] - roundEnd[round - 4]) / 4.0; - result["completed_generations"] = static_cast(round); - result["processed_events"] = static_cast(processed); - result["maximum_queue_wait_ns_including_bootstrap"] = static_cast(maximumQueueWait); - if (maximumQueueWait) { - result["maximum_queue_core"] = events[maximumQueueNode].core; - result["maximum_queue_pc"] = events[maximumQueueNode].pc; - } - llvm::json::Array waits; - for (const auto &[core, values] : queueWait) { - Cost wait = 0; - for (size_t generation = round - 4; generation < round; ++generation) - wait += values[generation]; - waits.push_back(llvm::json::Object{{"core", core}, {"queue_wait_ns", wait / 4.0}}); - } - result["per_core"] = std::move(waits); - result["tie_policy"] = reverseTies ? "reverse event-index order for simultaneous readiness" - : "event-index order for simultaneous readiness"; - return result; + if (!legacy && processed <= legacyBudget && maximum && std::abs(static_cast(first) - second) <= maximum * 0.01) { + result.intervalNs = (roundEnd[round] - roundEnd[round - 4]) / 4.0; + result.termination = PimQueueTermination::IntervalConverged; + result.progress = progress(false); + if (!verifyState) return finish(); + legacy = std::move(result); + result = PimQueueIntervalWindow{}; } } } for (size_t edge : node.outgoing) { - if (tokens[edge].values.empty()) ++nodes[edges[edge].target].readyInputs; - tokens[edge].values.push_back(next.time + edges[edge].delayNs); + if (tokens[edge].empty()) ++nodes[edges[edge].target].readyInputs; + tokens[edge].push(add(next.time, edges[edge].delayNs)); schedule(edges[edge].target); } schedule(next.node); + if (verifyState && monotonic && !proved && next.node == cores.begin()->second.entry) { + proved = checkpoint(next.time); + if (proved) proofProcessedEvents = processed; + } + if (proved && !diagnostics) return finish(); + if (legacy && (proved || history.exhausted || !monotonic)) return finish(); } - result["reason"] = "queue diagnostic did not converge within its event budget"; - return result; + result.termination = pending.empty() ? PimQueueTermination::NoReadyEvents : PimQueueTermination::EventBudget; + result.reason = pending.empty() ? "queue execution has no ready events before interval convergence" + : "queue diagnostic did not converge within its event budget"; + result.progress = progress(true); + return finish(); } +struct EvaluatedAnalysis { + WholeScheduleCost cost; + std::optional queue; + std::optional recurrence; + std::string error; +}; + class Analysis { public: PimInstructionTiming target; @@ -179,11 +366,14 @@ public: std::vector> memoryServices; std::array opcodeService{}, opcodeCounts{}; std::string error; + size_t abstractBudget = 1000000; - bool readCore(int64_t core, llvm::StringRef data, const llvm::json::Array &groups); - llvm::json::Object queueReport(); + bool readCore(int64_t core, llvm::StringRef data, llvm::ArrayRef groups); + bool readExecution(int64_t core, const std::vector &program); + EvaluatedAnalysis evaluate(); bool connect(); - llvm::json::Object report(); + llvm::json::Object report(const EvaluatedAnalysis &evaluation) const; + void recordWork(llvm::json::Object &result) const; private: std::map> sends; @@ -205,7 +395,7 @@ private: Core &state = cores.at(core); if (state.pending) { append(core, pc, "local-work", state.pending); - state.work += state.pending; + state.work = add(state.work, state.pending); state.pending = 0; } if (state.blockingNetworkInChunk) @@ -228,14 +418,51 @@ private: state.lastNetwork = end; return {begin, arrival}; } + void traceWork(int64_t core, Cost cost, PimExecutionEvent::WorkClass resource) { + if (options.executionTrace) + cores.at(core).traceWork[resource] = add(cores.at(core).traceWork[resource], cost); + } + void traceBoundary(int64_t core) { + if (!options.executionTrace) return; + auto &trace = (*options.executionTrace)[core]; + auto &work = cores.at(core).traceWork; + for (size_t resource = 0; resource < work.size(); ++resource) + if (work[resource]) { + trace.push_back({PimExecutionEvent::Work, work[resource]}); + trace.back().workClass = static_cast(resource); + work[resource] = 0; + } + } + void recordBlocking(int64_t core, Opcode op, Cost first, Cost second = 0) { + if (options.executionTrace) { + using E = PimExecutionEvent; + traceBoundary(core); + E event{op == Opcode::ld ? E::Load : op == Opcode::st ? E::Store + : op == Opcode::send ? E::Send : op == Opcode::recv ? E::Receive + : op == Opcode::sync ? E::Signal : E::Wait, 0}; + if (op == Opcode::ld || op == Opcode::st) event.amount = first; + else if (op == Opcode::send || op == Opcode::recv) { + event.peer = first; event.amount = second; + } else if (op == Opcode::sync) { event.peer = first; event.reg = second; } + else { event.reg = first; event.count = second; } + (*options.executionTrace)[core].push_back(std::move(event)); + } + if (!options.workBreakdown) return; + std::array event{}; + event[0] = static_cast(op); + llvm::support::endian::write64le(event.data() + 1, first); + llvm::support::endian::write64le(event.data() + 9, second); + cores.at(core).blockingOrder.update(llvm::StringRef(event.data(), event.size())); + } Cost local(Cost bytes, bool write) const { - return target.getLocalMemoryServiceNs(bytes, write); + return target.getLocalMemoryServiceNs(bytes, write, arithmetic); } Cost retirement(bool matrix = false) const { // Completion can join the current clock edge before ClockDomain runs. // ROB retirement needs another edge; matrix also has AG/commit registers. - return issue ? (matrix ? 3 : 1) * target.processorPeriodNs : 0; + return issue ? mul(matrix ? 3 : 1, target.processorPeriodNs) : 0; } + void memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes); bool communication(int64_t core, uint32_t pc, Opcode op, int32_t peer, int32_t reg, int32_t count, Cost bytes); }; @@ -250,13 +477,14 @@ bool Analysis::communication(int64_t core, uint32_t pc, Opcode op, error = "SYNC has an invalid core or event register"; return false; } + recordBlocking(core, op, peer, reg); auto request = network(core, pc, begin, - target.getLinkServiceNs(target.controlMessageBytes, core, peer)); + target.getLinkServiceNs(target.controlMessageBytes, core, peer, arithmetic)); signals[{peer, reg}].push_back({request.arrival, core}); - append(core, pc, "sync-complete", target.signalIssueNs + retirement()); - cores.at(core).work += target.signalIssueNs + retirement(); - cores.at(core).service[6] += target.signalIssueNs + retirement(); - opcodeService[static_cast(op)] += target.signalIssueNs + retirement(); + append(core, pc, "sync-complete", add(target.signalIssueNs, retirement())); + cores.at(core).work = add(cores.at(core).work, add(target.signalIssueNs, retirement())); + cores.at(core).service[6] = add(cores.at(core).service[6], add(target.signalIssueNs, retirement())); + opcodeService[static_cast(op)] = add(opcodeService[static_cast(op)], add(target.signalIssueNs, retirement())); } else if (op == Opcode::wait) { size_t end = append(core, pc, "wait-complete", retirement()); if (reg < 0 || reg >= static_cast(target.synchronizationRegisterCount) @@ -264,38 +492,108 @@ bool Analysis::communication(int64_t core, uint32_t pc, Opcode op, error = "WAIT requires one positive exact count per register per generation"; return false; } - cores.at(core).work += retirement(); - cores.at(core).service[6] += retirement(); - opcodeService[static_cast(op)] += retirement(); + recordBlocking(core, op, reg, count); + cores.at(core).work = add(cores.at(core).work, retirement()); + cores.at(core).service[6] = add(cores.at(core).service[6], retirement()); + opcodeService[static_cast(op)] = add(opcodeService[static_cast(op)], retirement()); } else { if (peer < 0 || peer >= static_cast(target.processorCount)) { error = "direct transfer has an invalid peer"; return false; } + recordBlocking(core, op, peer, bytes); size_t complete = event(core, pc, op == Opcode::send ? "send-complete" : "receive-complete"); edges.push_back({begin, complete, 0, 0}); cores.at(core).tail = complete; if (op == Opcode::send) { auto request = network(core, pc, begin, - target.getLinkServiceNs(target.controlMessageBytes, core, peer), 1); + target.getLinkServiceNs(target.controlMessageBytes, core, peer, arithmetic), 1); // SwitchSocket::send enqueues immediately in an unbounded FIFO. size_t enqueue = event(core, pc, "payload-enqueue"); edges.push_back({enqueue, complete, retirement(), 0}); - auto payload = network(core, pc, enqueue, target.getLinkServiceNs(bytes, core, peer)); + auto payload = network(core, pc, enqueue, target.getLinkServiceNs(bytes, core, peer, arithmetic)); sends[{core, peer}].push_back({enqueue, request.arrival, payload.arrival, bytes}); } else { auto permission = network(core, pc, begin, - target.getLinkServiceNs(target.controlMessageBytes, core, peer)); + target.getLinkServiceNs(target.controlMessageBytes, core, peer, arithmetic)); receives[{peer, core}].push_back({complete, permission.arrival, permission.begin, bytes}); } - cores.at(core).work += local(bytes, op == Opcode::recv) + retirement(); - cores.at(core).service[2] += local(bytes, op == Opcode::recv) + retirement(); - opcodeService[static_cast(op)] += local(bytes, op == Opcode::recv) + retirement(); + cores.at(core).work = add(cores.at(core).work, add(local(bytes, op == Opcode::recv), retirement())); + cores.at(core).service[2] = add(cores.at(core).service[2], add(local(bytes, op == Opcode::recv), retirement())); + opcodeService[static_cast(op)] = add(opcodeService[static_cast(op)], add(local(bytes, op == Opcode::recv), retirement())); } return true; } -bool Analysis::readCore(int64_t core, llvm::StringRef data, const llvm::json::Array &groups) { +void Analysis::memoryAccess(int64_t core, uint32_t pc, bool write, Cost bytes) { + Core &state = cores.at(core); + recordBlocking(core, write ? Opcode::st : Opcode::ld, bytes); + if (options.workBreakdown) { + std::array request{}; + request[0] = write; + llvm::support::endian::write64le(request.data() + 1, bytes); + state.requests.update(llvm::StringRef(request.data(), request.size())); + if (!state.requestRuns.empty() && state.requestRuns.back().write == write + && state.requestRuns.back().bytes == bytes) + state.requestRuns.back().count = add(state.requestRuns.back().count, 1); + else state.requestRuns.push_back({write, bytes, 1}); + } + auto demand = target.getSharedMemoryDemand(bytes, 1, write, arithmetic); + memory.readBytes = add(memory.readBytes, demand.readBytes); + memory.writeBytes = add(memory.writeBytes, demand.writeBytes); + memory.readRequests = add(memory.readRequests, demand.readRequests); + memory.writeRequests = add(memory.writeRequests, demand.writeRequests); + memory.serviceNs = add(memory.serviceNs, demand.serviceNs); + Cost completion = add(write ? 0 : local(bytes, true), retirement()); + Cost cost = add(add(add(write ? local(bytes, false) : 0, demand.serviceNs), completion), + add(target.getSharedMemoryLinkServiceNs(write ? bytes : target.controlMessageBytes, core, false, arithmetic), + target.getSharedMemoryLinkServiceNs(!write && target.sharedMemoryReadReturnsPayload ? bytes : 0, core, true, arithmetic))); + state.service[write ? 4 : 3] = add(state.service[write ? 4 : 3], cost); + opcodeService[static_cast(write ? Opcode::st : Opcode::ld)] = add(opcodeService[static_cast(write ? Opcode::st : Opcode::ld)], cost); + if (memoryQueue) { + flush(core, pc); + if (write) append(core, pc, "store-source-read", local(bytes, false)); + auto request = network(core, pc, state.tail, + target.getSharedMemoryLinkServiceNs(write ? bytes : target.controlMessageBytes, core, false, arithmetic)); + size_t done = event(core, pc, "memory-service-complete"); + edges.push_back({request.arrival, done, demand.serviceNs, 0}); + memoryServices.emplace_back(done, demand.serviceNs); + Cost reply = target.getSharedMemoryLinkServiceNs( + !write && target.sharedMemoryReadReturnsPayload ? bytes : 0, core, true, arithmetic); + size_t response = done; + if (reply) { + response = event(core, pc, "memory-response"); + edges.push_back({done, response, reply, 0}); + } + state.lastNetwork = response; + size_t complete = event(core, pc, "memory-complete"); + edges.push_back({response, complete, completion, 0}); + state.tail = complete; + state.work = add(state.work, cost); + return; + } + if (!state.blockingNetworkInChunk) { + flush(core, pc); + if (write) { + Cost read = local(bytes, false); + append(core, pc, "store-source-read", read); + state.work = add(state.work, read); + cost = sub(cost, read); + } + size_t begin = append(core, pc, "memory-queue-join"); + if (queues && state.lastNetwork != absent) + edges.push_back({state.lastNetwork, begin, 0, 0}); + if (state.firstNetwork == absent) + state.firstNetwork = begin; + state.blockingNetworkInChunk = true; + } + // ROB=1 serializes these blocking requests. No external dependency joins + // inside a chunk; retain their exact service sum without per-request nodes. + state.pending = add(state.pending, cost); + +} + +bool Analysis::readCore(int64_t core, llvm::StringRef data, llvm::ArrayRef groups) { if (data.size() < 12 || data.take_front(4) != "PIMB" || llvm::support::endian::read32le(data.data() + 4) != 1) { error = "invalid binary instruction header"; return false; @@ -331,9 +629,10 @@ bool Analysis::readCore(int64_t core, llvm::StringRef data, const llvm::json::Ar outputBits = options.oraclePrecision ? 8 : b; } // shortcut: serialize scalar issue; model frontend buffering before selection. - Cost cost = issue ? target.processorPeriodNs : 0; - state.pending += cost; state.service[5] += cost; - opcodeService[static_cast(op)] += cost; + Cost cost = issue && options.scalarIssue ? target.processorPeriodNs : 0; + traceWork(core, cost, PimExecutionEvent::ScalarIssue); + state.pending = add(state.pending, cost); state.service[5] = add(state.service[5], cost); + opcodeService[static_cast(op)] = add(opcodeService[static_cast(op)], cost); continue; } if (length < 0 || op > Opcode::sync || op == Opcode::vsoftmax) { @@ -345,7 +644,7 @@ bool Analysis::readCore(int64_t core, llvm::StringRef data, const llvm::json::Ar if (length % 4) { error = "oracle timing projection requires element-aligned transfers"; return false; } bytes /= 4; } - Cost in = (inputBits + 7) / 8, out = (outputBits + 7) / 8; + Cost in = ceil(inputBits, 8), out = ceil(outputBits, 8); if (op <= Opcode::vrsl && (!inputBits || !outputBits)) { error = "compute instruction precedes SETBW"; return false; } @@ -354,84 +653,39 @@ bool Analysis::readCore(int64_t core, llvm::StringRef data, const llvm::json::Ar continue; } if (op == Opcode::ld || op == Opcode::st) { - bool write = op == Opcode::st; - auto demand = target.getSharedMemoryDemand(bytes, 1, write); - memory += demand; - Cost completion = (write ? 0 : local(bytes, true)) + retirement(); - Cost cost = (write ? local(bytes, false) : 0) + demand.serviceNs + completion - + target.getSharedMemoryLinkServiceNs(write ? bytes : target.controlMessageBytes, core, false) - + target.getSharedMemoryLinkServiceNs(!write && target.sharedMemoryReadReturnsPayload ? bytes : 0, core, true); - state.service[write ? 4 : 3] += cost; - opcodeService[static_cast(op)] += cost; - if (memoryQueue) { - flush(core, pc); - if (write) append(core, pc, "store-source-read", local(bytes, false)); - auto request = network(core, pc, state.tail, - target.getSharedMemoryLinkServiceNs(write ? bytes : target.controlMessageBytes, core, false)); - size_t done = event(core, pc, "memory-service-complete"); - edges.push_back({request.arrival, done, demand.serviceNs, 0}); - memoryServices.emplace_back(done, demand.serviceNs); - Cost reply = target.getSharedMemoryLinkServiceNs( - !write && target.sharedMemoryReadReturnsPayload ? bytes : 0, core, true); - size_t response = done; - if (reply) { - response = event(core, pc, "memory-response"); - edges.push_back({done, response, reply, 0}); - } - state.lastNetwork = response; - size_t complete = event(core, pc, "memory-complete"); - edges.push_back({response, complete, completion, 0}); - state.tail = complete; - state.work += cost; - continue; - } - if (!state.blockingNetworkInChunk) { - flush(core, pc); - if (write) { - Cost read = local(bytes, false); - append(core, pc, "store-source-read", read); - state.work += read; - cost -= read; - } - size_t begin = append(core, pc, "memory-queue-join"); - if (queues && state.lastNetwork != absent) - edges.push_back({state.lastNetwork, begin, 0, 0}); - if (state.firstNetwork == absent) - state.firstNetwork = begin; - state.blockingNetworkInChunk = true; - } - // ROB=1 serializes these blocking requests. No external dependency joins - // inside a chunk; retain their exact service sum without per-request nodes. - state.pending += cost; + memoryAccess(core, pc, op == Opcode::st, bytes); continue; } + Cost cost = 0; unsigned resource = 2; if (op == Opcode::mvmul) { if (arg <= 0 || length < 0 || static_cast(length) >= groups.size() - || !groups[length].getAsInteger() || *groups[length].getAsInteger() <= 0) { + || groups[length] <= 0) { error = "matrix instruction references an invalid array group"; return false; } - Cost output = (target.matrixColumns * cellBits / arg) * *groups[length].getAsInteger() * out; - cost = local(target.matrixRows * in * target.matrixInputReadMultiplier, false) - + target.getMatrixComputeLatencyNs(inputBits) + local(output, true); - cost += retirement(true); resource = 0; + Cost output = getPimMatrixOutputBytes(target, cellBits, arg, groups[length], out); + cost = getPimMatrixWorkNs(target, inputBits, output, issue); resource = 0; } else if (op <= Opcode::vrsl) { bool binary = op >= Opcode::vvadd && op <= Opcode::vvsra; bool outputWidth = op == Opcode::vvmul || op == Opcode::vvsll || op == Opcode::vvsra || op == Opcode::vtanh || op == Opcode::vsigm || op == Opcode::vrsu || op == Opcode::vrsl; // The oracle's VectorUnit uses input width for VAVG/VVDMUL writeback. // It waits vector_latency_cycle nanoseconds, independently of core period. - cost = local(bytes * in, false) * (binary ? 2 : 1) - + ((bytes + target.vectorWidth - 1) / target.vectorWidth) * target.vectorLatencyCycles - + local(bytes * (outputWidth ? out : in), true) + retirement(); + cost = getPimVectorWorkNs(target, bytes, in, outputWidth ? out : in, + binary, issue); resource = 1; } else { - cost = local(bytes, true) + (op == Opcode::lmv ? local(bytes, false) : 0) + retirement(); + cost = getPimLocalTransferWorkNs(target, bytes, op == Opcode::lmv, true, issue); } - state.pending += cost; - state.service[resource] += cost; - opcodeService[static_cast(op)] += cost; + traceWork(core, cost, op == Opcode::mvmul ? PimExecutionEvent::Matrix + : op == Opcode::vmv ? PimExecutionEvent::VectorCopy + : resource == 1 ? PimExecutionEvent::Vector + : op == Opcode::lldi ? PimExecutionEvent::Fill : PimExecutionEvent::LocalCopy); + state.pending = add(state.pending, cost); + state.service[resource] = add(state.service[resource], cost); + opcodeService[static_cast(op)] = add(opcodeService[static_cast(op)], cost); } + traceBoundary(core); flush(core, count); edges.push_back({state.tail, state.entry, 0, 1}); if (queues && state.firstNetwork != absent) @@ -439,6 +693,118 @@ bool Analysis::readCore(int64_t core, llvm::StringRef data, const llvm::json::Ar return true; } +using LocalWorkSummary = std::optional>; +using LocalWorkCache = llvm::DenseMap *, LocalWorkSummary>; + +static LocalWorkSummary summarizeLocalWork( + const std::vector &program, LocalWorkCache &cache) { + if (auto found = cache.find(&program); found != cache.end()) return found->second; + std::array result{}; + for (const auto &item : program) { + if (item.kind == PimExecutionEvent::Work) { + if (item.workClass < PimExecutionEvent::Matrix || item.workClass > PimExecutionEvent::ScalarIssue) + throw pim_timing::Unrepresentable{"unknown abstract work resource"}; + result[item.workClass] = add(result[item.workClass], item.amount); + } else if (item.kind == PimExecutionEvent::Repeat) { + auto nested = summarizeLocalWork(item.body, cache); + if (!nested) return cache[&program] = std::nullopt; + for (size_t resource = 0; resource < result.size(); ++resource) + result[resource] = add(result[resource], mul((*nested)[resource], item.amount)); + } else return cache[&program] = std::nullopt; + } + return cache[&program] = result; +} + +bool Analysis::readExecution(int64_t core, const std::vector &program) { + if (program.empty()) return true; + size_t entry = event(core, 0, "body-entry"); + cores.emplace(core, Core{entry, entry, absent, absent}); + Core &state = cores.at(core); + auto work = [&](const PimExecutionEvent &item, Cost multiplier) { + using E = PimExecutionEvent; + if (item.workClass < E::Matrix || item.workClass > E::ScalarIssue) + throw pim_timing::Unrepresentable{"unknown abstract work resource"}; + if (item.workClass == E::ScalarIssue && !options.scalarIssue) return; + if (options.ignoreLocalCopies && (item.workClass == E::LocalCopy || item.workClass == E::VectorCopy)) return; + Cost cost = mul(item.amount, multiplier); + traceWork(core, cost, item.workClass); + state.pending = add(state.pending, cost); + unsigned resource = item.workClass == E::Matrix ? 0 + : item.workClass == E::Vector || item.workClass == E::VectorCopy ? 1 + : item.workClass == E::ScalarIssue ? 5 : 2; + state.service[resource] = add(state.service[resource], cost); + }; + LocalWorkCache localWorkCache; + struct WorkRun { size_t end; std::array service{}; }; + llvm::DenseMap workRuns; + size_t &budget = abstractBudget; + uint32_t position = 0; + std::function &)> consume = + [&](const std::vector &sequence) { + for (size_t index = 0; index < sequence.size(); ++index) { + const auto &item = sequence[index]; + if (item.kind != PimExecutionEvent::Work && !budget--) { + error = "abstract queue-event budget exceeded"; return false; + } + if (item.kind < PimExecutionEvent::Work || item.kind > PimExecutionEvent::Repeat) { + error = "unknown abstract event kind"; return false; + } + ++position; + if (item.kind == PimExecutionEvent::Repeat) { + if (auto summary = summarizeLocalWork(item.body, localWorkCache)) { + for (size_t resource = 0; resource < summary->size(); ++resource) { + PimExecutionEvent event{PimExecutionEvent::Work, (*summary)[resource]}; + event.workClass = static_cast(resource); + work(event, item.amount); + } + continue; + } + if (item.amount > budget) { error = "abstract repetition budget exceeded"; return false; } + for (uint64_t i = 0; i < item.amount; ++i) + if (!consume(item.body)) return false; + } else if (item.kind == PimExecutionEvent::Work) { + auto found = workRuns.find(&item); + if (found == workRuns.end()) { + WorkRun run{index}; + while (run.end < sequence.size() && sequence[run.end].kind == PimExecutionEvent::Work) { + const auto &event = sequence[run.end++]; + if (event.workClass < PimExecutionEvent::Matrix || event.workClass > PimExecutionEvent::ScalarIssue) + throw pim_timing::Unrepresentable{"unknown abstract work resource"}; + run.service[event.workClass] = add(run.service[event.workClass], event.amount); + } + found = workRuns.try_emplace(&item, run).first; + } + for (size_t resource = 0; resource < found->second.service.size(); ++resource) { + PimExecutionEvent event{PimExecutionEvent::Work, found->second.service[resource]}; + event.workClass = static_cast(resource); + work(event, 1); + } + index = found->second.end - 1; + } else if (item.kind == PimExecutionEvent::Load || item.kind == PimExecutionEvent::Store) { + memoryAccess(core, position, item.kind == PimExecutionEvent::Store, item.amount); + } else { + for (int64_t value : {item.peer, item.reg, item.count}) + if (value < 0 || value > std::numeric_limits::max()) { + error = "unrepresentable abstract communication operand"; return false; + } + Opcode op = item.kind == PimExecutionEvent::Send ? Opcode::send + : item.kind == PimExecutionEvent::Receive ? Opcode::recv + : item.kind == PimExecutionEvent::Signal ? Opcode::sync : Opcode::wait; + if (!communication(core, position, op, item.peer, item.reg, item.count, item.amount)) + return false; + } + } + return true; + }; + if (!consume(program)) return false; + traceBoundary(core); + flush(core, position); + edges.push_back({state.tail, state.entry, 0, 1}); + if (queues && state.firstNetwork != absent) + edges.push_back({state.lastNetwork, state.firstNetwork, 0, 1}); + return true; +} + bool Analysis::connect() { if (sends.size() != receives.size() || signals.size() != waits.size()) { error = "communication endpoint sets differ"; return false; @@ -454,7 +820,7 @@ bool Analysis::connect() { // Its outgoing FIFO may still contain earlier SYNC or transport requests. edges.push_back({send.requestArrival, receive.permissionDispatch, 0, 0}); edges.push_back({receive.permissionArrival, send.enqueue, local(send.bytes, false), 0}); - edges.push_back({send.payloadArrival, receive.complete, local(send.bytes, true) + retirement(), 0}); + edges.push_back({send.payloadArrival, receive.complete, add(local(send.bytes, true), retirement()), 0}); } } for (const auto &[key, pending] : waits) { @@ -468,31 +834,75 @@ bool Analysis::connect() { return true; } -llvm::json::Object Analysis::report() { - llvm::json::Object result; - if (cores.empty()) { - error = "artifact has no active repeating instruction stream"; - result["status"] = "unavailable"; result["reason"] = error; return result; +EvaluatedAnalysis Analysis::evaluate() { + EvaluatedAnalysis result; + if (cores.empty()) result.error = "artifact has no active repeating instruction stream"; + else if (!connect()) result.error = error; + if (!result.error.empty()) { result.cost.reason = result.error; return result; } + if (options.detailed) { + auto recurrence = analyzePeriodicDependencies(events.size(), edges, result.error); + if (mlir::failed(recurrence)) { result.cost.reason = result.error; return result; } + result.recurrence = std::move(*recurrence); + } + if (!options.detailed || memoryQueue) { + auto start = std::chrono::steady_clock::now(); + result.queue.emplace(); + auto &queue = *result.queue; + queue.recurrenceRequired = options.verifyPeriodicState; + queue.details = options.workBreakdown || options.detailed; + queue.measureTime = options.workBreakdown || options.measureTime; + queue.memoryServiceNs = memory.serviceNs; + for (size_t policy = 0; policy < queue.policies.size(); ++policy) + queue.policies[policy] = estimateQueuedPeriod(events, edges, memoryServices, cores, + policy != 0, queue.recurrenceRequired, queue.details, options.intervalWindowEventBudget); + queue.evaluationSeconds = std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + result.cost = queue.cost(); + } + return result; +} + +llvm::json::Object Analysis::report(const EvaluatedAnalysis &evaluation) const { + llvm::json::Object result; + if (!evaluation.error.empty()) { + result["status"] = "unavailable"; result["reason"] = evaluation.error; return result; } - if (!connect()) { result["status"] = "unavailable"; result["reason"] = error; return result; } result["global_memory_service_ns"] = static_cast(memory.serviceNs); result["global_read_bytes"] = static_cast(memory.readBytes); result["global_write_bytes"] = static_cast(memory.writeBytes); result["global_read_requests"] = static_cast(memory.readRequests); result["global_write_requests"] = static_cast(memory.writeRequests); + if (options.workBreakdown) { + llvm::json::Array orders; + for (const auto &[core, state] : cores) { + auto hash = state.requests, blocking = state.blockingOrder; + llvm::json::Array runs; + for (const auto &run : state.requestRuns) + runs.push_back(llvm::json::Array{run.write ? "write" : "read", run.bytes, run.count}); + orders.push_back(llvm::json::Object{{"core", core}, {"identity", llvm::toHex(hash.final())}, + {"blocking_identity", llvm::toHex(blocking.final())}, {"runs", std::move(runs)}}); + } + result["request_order"] = std::move(orders); + } + Cost directBytes = 0, directRequests = 0; + for (const auto &[key, transfers] : sends) + for (const auto &transfer : transfers) { directBytes = add(directBytes, transfer.bytes); directRequests = add(directRequests, 1); } + result["direct_requests"] = static_cast(directRequests); + result["direct_bytes"] = static_cast(directBytes); + bool available = !evaluation.queue || evaluation.cost.available(); + result["status"] = available ? "estimated" : "unavailable"; + if (!available) result["reason"] = evaluation.cost.reason; if (!options.detailed) { - result["status"] = "estimated"; - result["memory_controller"] = queueReport(); + result["memory_controller"] = serializePimQueueEvaluation(*evaluation.queue); + if (options.workBreakdown) recordWork(result); return result; } - auto estimate = analyzePeriodicDependencies(events.size(), edges, error); - if (mlir::failed(estimate)) { result["status"] = "unavailable"; result["reason"] = error; return result; } - result["status"] = "estimated"; - result["recurrence_lower_ns"] = estimate->recurrenceLowerNs; - result["recurrence_upper_ns"] = estimate->recurrenceUpperNs; - result["within_generation_span_ns"] = static_cast(estimate->withinGenerationSpanNs); - result["interval_estimate_ns"] = std::max(estimate->recurrenceUpperNs, static_cast(memory.serviceNs)); - result["limiting_bound"] = memory.serviceNs > estimate->recurrenceUpperNs ? "global-memory" : "recurrence"; + const auto &estimate = *evaluation.recurrence; + result["recurrence_lower_ns"] = estimate.recurrenceLowerNs; + result["recurrence_upper_ns"] = estimate.recurrenceUpperNs; + result["within_generation_span_ns"] = static_cast(estimate.withinGenerationSpanNs); + result["interval_estimate_ns"] = std::max(estimate.recurrenceUpperNs, static_cast(memory.serviceNs)); + result["limiting_bound"] = memory.serviceNs > estimate.recurrenceUpperNs ? "global-memory" : "recurrence"; result["events"] = static_cast(events.size()); result["edges"] = static_cast(edges.size()); // Keep small timing fixtures inspectable without dumping network-size graphs. @@ -507,22 +917,10 @@ llvm::json::Object Analysis::report() { result["timing_graph"] = llvm::json::Object{{"events", std::move(nodes)}, {"dependencies", std::move(dependencies)}}; } result["network_queues"] = queues; result["instruction_issue"] = issue; - if (memoryQueue) result["memory_controller"] = queueReport(); + if (memoryQueue) result["memory_controller"] = serializePimQueueEvaluation(*evaluation.queue); result["resource_service_columns"] = llvm::json::Array{"matrix", "vector", "local_transfer", "global_load", "global_store", "scalar_issue", "synchronization"}; - Cost directBytes = 0, directRequests = 0; - for (const auto &[key, transfers] : sends) - for (const auto &transfer : transfers) { directBytes += transfer.bytes; ++directRequests; } - result["direct_requests"] = static_cast(directRequests); - result["direct_bytes"] = static_cast(directBytes); - llvm::json::Object opcodeDemand, instructionCounts; - for (size_t op = 0; op < opcodeCounts.size(); ++op) - if (opcodeCounts[op]) { - opcodeDemand[pim_binary::kOpcodeNames[op]] = static_cast(opcodeService[static_cast(op)]); - instructionCounts[pim_binary::kOpcodeNames[op]] = static_cast(opcodeCounts[op]); - } - result["opcode_service_ns"] = std::move(opcodeDemand); - result["instruction_counts"] = std::move(instructionCounts); + recordWork(result); llvm::json::Array perCore, cycle; Cost maximumCoreWork = 0; for (const auto &[core, state] : cores) { @@ -534,11 +932,11 @@ llvm::json::Object Analysis::report() { } std::map cycleCores; Cost cycleDelay = 0, cycleDistance = 0; - for (size_t index : estimate->criticalCycle) { + for (size_t index : estimate.criticalCycle) { const auto &edge = edges[index]; const auto &from = events[edge.source], &to = events[edge.target]; - cycleCores[to.core] += edge.delayNs; - cycleDelay += edge.delayNs; cycleDistance += edge.distance; + cycleCores[to.core] = add(cycleCores[to.core], edge.delayNs); + cycleDelay = add(cycleDelay, edge.delayNs); cycleDistance = add(cycleDistance, edge.distance); if (cycle.size() < 128) cycle.push_back(llvm::json::Object{{"source_core", from.core}, {"target_core", to.core}, {"source_pc", from.pc}, {"target_pc", to.pc}, {"kind", to.kind}, @@ -548,7 +946,7 @@ llvm::json::Object Analysis::report() { for (auto [core, delay] : cycleCores) critical.push_back(llvm::json::Object{{"core", core}, {"delay_ns", static_cast(delay)}}); result["critical_cycle"] = std::move(cycle); result["critical_cores"] = std::move(critical); - result["critical_cycle_edges"] = static_cast(estimate->criticalCycle.size()); + result["critical_cycle_edges"] = static_cast(estimate.criticalCycle.size()); result["critical_cycle_delay_ns"] = static_cast(cycleDelay); result["critical_cycle_distance"] = static_cast(cycleDistance); result["per_core"] = std::move(perCore); @@ -558,24 +956,56 @@ llvm::json::Object Analysis::report() { return result; } -llvm::json::Object Analysis::queueReport() { - auto queued = estimateQueuedPeriod(events, edges, memoryServices, cores, false); - auto alternative = estimateQueuedPeriod(events, edges, memoryServices, cores, true); - queued["alternative_tie_status"] = alternative.getString("status").value(); - if (auto reason = alternative.getString("reason")) - queued["alternative_tie_reason"] = reason->str(); - if (auto interval = alternative.getNumber("interval_ns")) - queued["alternative_tie_interval_ns"] = *interval; - if (auto interval = queued.getNumber("interval_ns")) - queued["memory_service_fraction"] = memory.serviceNs / *interval; - queued["arbitration"] = "FIFO by predicted controller arrival"; - return queued; +void Analysis::recordWork(llvm::json::Object &result) const { + llvm::json::Object opcodeDemand, instructionCounts; + for (size_t op = 0; op < opcodeCounts.size(); ++op) + if (opcodeCounts[op]) { + opcodeDemand[pim_binary::kOpcodeNames[op]] = static_cast(opcodeService[static_cast(op)]); + instructionCounts[pim_binary::kOpcodeNames[op]] = static_cast(opcodeCounts[op]); + } + llvm::json::Array perCore; + std::array totals{}; + for (const auto &[core, state] : cores) { + llvm::json::Array resource; + for (size_t i = 0; i < totals.size(); ++i) { + totals[i] = add(totals[i], state.service[i]); + resource.push_back(state.service[i]); + } + perCore.push_back(llvm::json::Object{{"core", core}, {"resource_service_ns", std::move(resource)}}); + } + llvm::json::Array resource; + for (Cost value : totals) resource.push_back(value); + result["resource_service_ns"] = std::move(resource); + result["resource_service_columns"] = llvm::json::Array{"matrix", "vector", "local_transfer", + "global_load", "global_store", "scalar_issue", "synchronization"}; + result["per_core_service"] = std::move(perCore); + result["opcode_service_ns"] = std::move(opcodeDemand); + result["instruction_counts"] = std::move(instructionCounts); } + } // namespace +std::optional PimQueuePolicyResult::intervalNs() const { + return recurrence.proved() ? recurrence.intervalNs : legacy.intervalNs; +} + +WholeScheduleCost PimQueueEvaluation::cost() const { + WholeScheduleCost result; + result.primary = policies[0].intervalNs(); + result.alternative = policies[1].intervalNs(); + result.recurrence = {policies[0].recurrence, policies[1].recurrence}; + result.timing.evaluationSeconds = evaluationSeconds; + if (!result.available()) { + const auto &failed = result.primary ? policies[1] : policies[0]; + result.reason = !failed.recurrence.reason.empty() ? failed.recurrence.reason : failed.legacy.reason; + } + return result; +} + llvm::json::Object onnx_mlir::analyzePimInstructions(const PimInstructionProgram &program, - const PimInstructionTiming &target, int64_t cellBits, PimPeriodicOptions options) { + const PimInstructionTiming &target, int64_t cellBits, PimPeriodicOptions options) try { + if (options.executionTrace) options.executionTrace->assign(target.processorCount, {}); Analysis analysis; analysis.target = target; analysis.cellBits = cellBits; @@ -587,85 +1017,94 @@ llvm::json::Object onnx_mlir::analyzePimInstructions(const PimInstructionProgram return llvm::json::Object{{"status", "unavailable"}, {"reason", reason.str()}}; }; if (!target.hasPeriodicTransferTiming() || cellBits <= 0) - return unavailable("target has no supported instruction timing contract"); + return unavailable("target has no supported queue timing contract"); if (program.instructions.empty() || program.instructions.size() > target.processorCount || program.instructions.size() != program.arrayGroups.size()) return unavailable("instruction program core count or array groups do not fit the target"); for (size_t id = 0; id < program.instructions.size(); ++id) { - llvm::json::Array groups; - for (int64_t count : program.arrayGroups[id]) groups.push_back(count); - if (!analysis.readCore(id, program.instructions[id], groups)) + if (!analysis.readCore(id, program.instructions[id], program.arrayGroups[id])) return unavailable(analysis.error); } - return analysis.report(); + auto evaluation = analysis.evaluate(); + return analysis.report(evaluation); +} catch (const pim_timing::Unrepresentable &failure) { + return llvm::json::Object{{"status", "unavailable"}, {"reason", failure.reason}}; } -void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target, - PimInstructionTiming timing) { - auto hardware = std::make_shared(std::move(timing)); - target.reportPeriodicSchedule = [hardware](const spatial::ComputeGraph &graph, - const spatial::ScheduledComputeMaterializationResult &materialization, - const spatial::DeferredTransferPlan &transfers, - const spatial::DeferredBoundaryPlan &boundaries) { - dumpPeriodicScheduleEstimate(graph, materialization, transfers, boundaries, *hardware); - }; - auto contract = getPimPeriodicTimingContract(); - if (!contract) target.wholeScheduleCostUnavailable = llvm::toString(contract.takeError()); - if (!hardware->hasPeriodicTransferTiming()) - target.wholeScheduleCostUnavailable = "target has no supported instruction timing contract"; - if (pimDisableSynchronization) - target.wholeScheduleCostUnavailable = "synchronization ablation has no periodic queue timing contract"; - if (!target.wholeScheduleCostUnavailable.empty()) return; - target.prepareScheduleCost = [hardware, cellBits = *contract](mlir::ModuleOp module, - llvm::json::Object &summary) -> mlir::FailureOr { - auto start = std::chrono::steady_clock::now(); - PimInstructionProgram program; - PimCodegenTiming timing; - std::string unused; - if (compileToPimCode(module, unused, &program, &timing) != CompilerSuccess) - return mlir::failure(); - summary["codegen_preparation_seconds"] = timing.preparationSeconds; - summary["codegen_emission_seconds"] = timing.emissionSeconds; - summary["instruction_generation_seconds"] = std::chrono::duration( - std::chrono::steady_clock::now() - start).count(); - llvm::SHA256 hash; - int64_t instructionBytes = 0; - for (size_t core = 0; core < program.instructions.size(); ++core) { - instructionBytes += program.instructions[core].size(); - hash.update(program.instructions[core]); - for (int64_t group : program.arrayGroups[core]) hash.update(std::to_string(group) + ","); - hash.update(";"); - } - summary["instruction_bytes"] = instructionBytes; - summary["instruction_count"] = (instructionBytes - - pim_binary::kHeaderSize * program.instructions.size()) / pim_binary::kRecordSize; - spatial::ScheduleCostQuery query; - query.identity = llvm::toHex(hash.final()); - query.evaluate = [program = std::move(program), hardware, cellBits](llvm::json::Object &summary) { - PimPeriodicOptions options; - options.oraclePrecision = true; - auto report = analyzePimInstructions(program, *hardware, cellBits, options); - for (const char *field : {"global_memory_service_ns", "global_read_bytes", "global_write_bytes", - "global_read_requests", "global_write_requests"}) - if (auto value = report.getInteger(field)) summary[field] = *value; - spatial::WholeScheduleCost cost; - // Conservative selection margin; not a timing correction or formal bound. - cost.relativeUncertainty = 0.021; - if (auto *queue = report.getObject("memory_controller")) { - if (queue->getString("status") == "estimated" - && queue->getString("alternative_tie_status") == "estimated") { - cost.primary = queue->getNumber("interval_ns"); - cost.alternative = queue->getNumber("alternative_tie_interval_ns"); - } - if (auto reason = queue->getString("reason")) cost.reason = reason->str(); - if (!cost.eligible() && cost.reason.empty()) - if (auto reason = queue->getString("alternative_tie_reason")) - cost.reason = "alternative arbitration: " + reason->str(); - } else if (auto reason = report.getString("reason")) cost.reason = reason->str(); - if (!cost.eligible() && cost.reason.empty()) - cost.reason = "arbitration sensitivity estimate unavailable or nonconverged"; - return cost; - }; - return query; - }; +uint64_t onnx_mlir::getPimLocalTransferWorkNs(const PimInstructionTiming &target, + uint64_t bytes, bool read, bool write, bool instructionIssue) { + return add(add(read ? target.getLocalMemoryServiceNs(bytes, false, arithmetic) : 0, + write ? target.getLocalMemoryServiceNs(bytes, true, arithmetic) : 0), + instructionIssue ? target.processorPeriodNs : 0); +} + +uint64_t onnx_mlir::getPimVectorWorkNs(const PimInstructionTiming &target, + uint64_t elements, uint64_t inputBytes, uint64_t outputBytes, + bool binary, bool instructionIssue) { + return add(add(mul(target.getLocalMemoryServiceNs(mul(elements, inputBytes), false, arithmetic), binary ? 2 : 1), + mul(ceil(elements, target.vectorWidth), target.vectorLatencyCycles)), + add(target.getLocalMemoryServiceNs(mul(elements, outputBytes), true, arithmetic), + instructionIssue ? target.processorPeriodNs : 0)); +} + +uint64_t onnx_mlir::getPimMatrixOutputBytes(const PimInstructionTiming &target, + uint64_t cellBits, uint64_t weightBits, uint64_t groups, uint64_t outputBytes) { + if (!weightBits || !cellBits || !groups) + throw pim_timing::Unrepresentable{"unavailable physical array-group geometry"}; + return mul(mul(target.matrixColumns, cellBits) / weightBits, mul(groups, outputBytes)); +} + +uint64_t onnx_mlir::getPimMatrixWorkNs(const PimInstructionTiming &target, + uint64_t inputBits, uint64_t outputBytes, bool instructionIssue) { + return add(add(target.getLocalMemoryServiceNs(mul(mul(target.matrixRows, ceil(inputBits, 8)), + target.matrixInputReadMultiplier), false, arithmetic), + target.getMatrixComputeLatencyNs(inputBits, arithmetic)), + add(target.getLocalMemoryServiceNs(outputBytes, true, arithmetic), + instructionIssue ? mul(3, target.processorPeriodNs) : 0)); +} + +static PimPeriodicEstimate unavailableExecution(llvm::StringRef reason, bool verifyState) { + PimPeriodicEstimate result; + result.cost.reason = reason.str(); + if (verifyState) + for (auto &recurrence : result.cost.recurrence) + recurrence.status = ScheduleRecurrenceEvidence::Unproved; + result.report = llvm::json::Object{{"status", "unavailable"}, {"reason", reason.str()}}; + return result; +} + +PimPeriodicEstimate onnx_mlir::evaluatePimExecution(const PimExecutionProgram &program, + const PimInstructionTiming &target, PimPeriodicOptions options) try { + if (options.executionTrace) options.executionTrace->assign(target.processorCount, {}); + Analysis analysis; + analysis.target = target; + analysis.options = options; + analysis.queues = options.networkQueues; + analysis.issue = options.instructionIssue; + analysis.memoryQueue = options.memoryController; + if (!target.hasPeriodicTransferTiming() || program.size() > target.processorCount) + return unavailableExecution("unsupported abstract timing contract", options.verifyPeriodicState); + auto start = std::chrono::steady_clock::now(); + for (size_t core = 0; core < program.size(); ++core) + if (!analysis.readExecution(core, program[core])) + return unavailableExecution(analysis.error, options.verifyPeriodicState); + double expansionSeconds = std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + auto evaluation = analysis.evaluate(); + evaluation.cost.timing.expansionSeconds = expansionSeconds; + if (options.verifyPeriodicState && !evaluation.cost.convergenceProved().has_value()) + for (auto &recurrence : evaluation.cost.recurrence) + recurrence.status = ScheduleRecurrenceEvidence::Unproved; + auto report = analysis.report(evaluation); + report["event_expansion_ms"] = expansionSeconds * 1000; + report["expanded_graph_events"] = static_cast(analysis.events.size()); + report["expanded_graph_edges"] = static_cast(analysis.edges.size()); + return {std::move(evaluation.cost), std::move(report), std::move(evaluation.queue)}; +} catch (const pim_timing::Unrepresentable &failure) { + return unavailableExecution(failure.reason, options.verifyPeriodicState); +} + +llvm::json::Object onnx_mlir::analyzePimExecution(const PimExecutionProgram &program, + const PimInstructionTiming &target, PimPeriodicOptions options) { + return evaluatePimExecution(program, target, options).report; } diff --git a/src/PIM/Compiler/PimPeriodicAnalysis.hpp b/src/PIM/Compiler/PimPeriodicAnalysis.hpp index 118fcbd..4838f79 100644 --- a/src/PIM/Compiler/PimPeriodicAnalysis.hpp +++ b/src/PIM/Compiler/PimPeriodicAnalysis.hpp @@ -1,14 +1,65 @@ #pragma once #include "llvm/Support/JSON.h" +#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp" #include #include #include namespace onnx_mlir { -namespace spatial { struct SchedulingTarget; } struct PimInstructionTiming; -void configurePimScheduleCosts(spatial::SchedulingTarget &target, PimInstructionTiming timing); + +enum class PimQueueTermination { + NotRun, IntervalConverged, RecurrenceProved, EventBudget, GenerationLimit, + HistoryLimit, NonMonotonic, NoReadyEvents +}; + +struct PimQueueEventLocation { + int64_t core = 0; + uint32_t pc = 0; + std::string kind; +}; + +struct PimQueueProgress { + size_t processedEvents = 0, completedGenerations = 0; + uint64_t maximumQueueWaitNs = 0; + std::optional maximumQueueRequest, generationLimit; + std::vector> perCoreQueueWaitNs; + std::vector completedIntervalsNs; +}; + +struct PimQueueIntervalWindow { + std::optional intervalNs; + std::string reason; + PimQueueTermination termination = PimQueueTermination::NotRun; + PimQueueProgress progress; +}; + +struct PimQueuePolicyResult { + PimQueueIntervalWindow legacy; + spatial::ScheduleRecurrenceEvidence recurrence; + PimQueueProgress progress; + PimQueueTermination termination = PimQueueTermination::NotRun; + size_t checkpoints = 0, historyBytes = 0, stateWords = 0, proofProcessedEvents = 0; + bool reverseTies = false; + std::optional intervalNs() const; +}; + +struct PimQueueEvaluation { + std::array policies; + uint64_t memoryServiceNs = 0; + double evaluationSeconds = 0; + bool recurrenceRequired = false, details = false, measureTime = false; + spatial::WholeScheduleCost cost() const; +}; + +llvm::json::Object serializePimQueueEvaluation(const PimQueueEvaluation &evaluation); + +struct PimPeriodicEstimate { + spatial::WholeScheduleCost cost; + llvm::json::Object report; + std::optional queue; +}; // Binary instructions, indexed by physical core. No executable or measured data. struct PimInstructionProgram { @@ -16,15 +67,58 @@ struct PimInstructionProgram { std::vector> arrayGroups; }; +// Hardware work only. Bytes are already in the target timing precision. +// Repeat keeps ordered request patterns compact until queue construction. +struct PimExecutionEvent { + enum Kind { Work, Load, Store, Send, Receive, Signal, Wait, Repeat }; + enum WorkClass { Matrix, Vector, LocalCopy, Fill, VectorCopy, ScalarIssue }; + Kind kind = Work; + uint64_t amount = 0; + int64_t peer = 0, reg = 0, count = 0; + std::vector body; + bool constant = false; + WorkClass workClass = Fill; + PimExecutionEvent(Kind kind, uint64_t amount, int64_t peer = 0, + int64_t reg = 0, int64_t count = 0, std::vector body = {}) + : kind(kind), amount(amount), peer(peer), reg(reg), count(count), body(std::move(body)) {} +}; +using PimExecutionProgram = std::vector>; + struct PimPeriodicOptions { bool networkQueues = true; bool instructionIssue = true; + bool scalarIssue = true; + bool ignoreLocalCopies = false; + bool workBreakdown = false; + bool measureTime = false; + // Optional ordered work/IO extraction for timing validation; no execution. + PimExecutionProgram *executionTrace = nullptr; + // Prove recurrence of the modeled state. Instruction diagnostics retain + // their legacy interval-window result separately. + bool verifyPeriodicState = false; + // Optional diagnostic cap. The complete-state proof keeps its original budget. + std::optional intervalWindowEventBudget; bool memoryController = true; bool detailed = false; // Apply the oracle's fp32-to-int8 timing projection without changing the program. bool oraclePrecision = false; }; +llvm::json::Object analyzePimExecution(const PimExecutionProgram &program, + const PimInstructionTiming &target, PimPeriodicOptions options = {}); +PimPeriodicEstimate evaluatePimExecution(const PimExecutionProgram &program, + const PimInstructionTiming &target, PimPeriodicOptions options = {}); + +uint64_t getPimLocalTransferWorkNs(const PimInstructionTiming &target, + uint64_t bytes, bool read, bool write, bool instructionIssue = true); +uint64_t getPimVectorWorkNs(const PimInstructionTiming &target, + uint64_t elements, uint64_t inputBytes, uint64_t outputBytes, + bool binary, bool instructionIssue = true); +uint64_t getPimMatrixOutputBytes(const PimInstructionTiming &target, + uint64_t cellBits, uint64_t weightBits, uint64_t groups, uint64_t outputBytes); +uint64_t getPimMatrixWorkNs(const PimInstructionTiming &target, + uint64_t inputBits, uint64_t outputBytes, bool instructionIssue = true); + llvm::json::Object analyzePimInstructions(const PimInstructionProgram &program, const PimInstructionTiming &target, int64_t cellBits, PimPeriodicOptions options = {}); diff --git a/src/PIM/Compiler/PimPipelineSelection.cpp b/src/PIM/Compiler/PimPipelineSelection.cpp index 111651e..408e580 100644 --- a/src/PIM/Compiler/PimPipelineSelection.cpp +++ b/src/PIM/Compiler/PimPipelineSelection.cpp @@ -13,7 +13,7 @@ #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/JSON.h" #include "llvm/Support/raw_os_ostream.h" -#include +#include "src/Accelerators/PIM/Common/Support/Diagnostics.hpp" #include using namespace mlir; @@ -57,6 +57,14 @@ struct CandidateTiming final : PassInstrumentation { } }; +void writeSelectionReport(llvm::json::Object report) { + auto file = openReportFileWithExtension("pipeline_selection", "json"); + if (file) { + llvm::raw_os_ostream out(file); + out << llvm::formatv("{0:2}\n", llvm::json::Value(std::move(report))); + } +} + struct PimPipelineSelectionPass final : PassWrapper> { MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(PimPipelineSelectionPass) PimPipelineSelectionPass(const SchedulingTarget &target, const SpatialTargetResources &resources, @@ -68,156 +76,256 @@ struct PimPipelineSelectionPass final : PassWrapper(*summary)); - return pm.run(module); - } - - LogicalResult evaluate(ModuleOp checkpoint, const MergeScheduleResult &schedule, - OwningOpRef &candidate, - std::string &scheduleReport, llvm::json::Object &summary) { + LogicalResult realizeCandidate(ModuleOp checkpoint, const MergeScheduleResult &schedule, + OwningOpRef &candidate, std::string &scheduleReport, + llvm::json::Object &summary) { auto begin = Clock::now(); IRMapping mapping; candidate = cast(checkpoint->clone(mapping)); auto remapped = remapSpatialSchedule(schedule, mapping); if (failed(realizeSpatialSchedule(*candidate, remapped, target, stages, - SpatialDataflowExportStage::None, nullptr, &scheduleReport)) - || failed(lower(*candidate, &summary))) return failure(); - if (enableTiming) summary["codegen_input_ir"] = countIR(candidate->getOperation()); - summary["lowering_seconds"] = elapsed(begin); + SpatialDataflowExportStage::None, nullptr, &scheduleReport))) return failure(); + summary["realization_seconds"] = elapsed(begin); + return success(); + } + + WholeScheduleCost spatialCost(ModuleOp candidate, llvm::json::Object &summary) { + WholeScheduleCost cost; + auto begin = Clock::now(); + llvm::json::Object estimate; + if (target.estimateRealizedScheduleCost) + cost = target.estimateRealizedScheduleCost(candidate, estimate); + else cost.reason = target.wholeScheduleCostUnavailable.empty() + ? "target has no realized-Spatial cost query" : target.wholeScheduleCostUnavailable; + cost.selectionMargin = kPipelineSelectionMargin; + estimate["safe_for_selection"] = cost.eligible(); + estimate["diagnostic_only"] = false; + summary["spatial_cost"] = std::move(estimate); + summary["spatial_cost_seconds"] = elapsed(begin); + return cost; + } + + LogicalResult evaluateSpatialCandidates(ModuleOp checkpoint, + const PreparedSpatialSchedules &prepared, std::vector &costs, + llvm::json::Object &report) { + llvm::json::Array summaries; + double realization = 0, extraction = 0, expansion = 0, queue = 0; + std::string errors; + for (size_t i = 0; i < costs.size(); ++i) { + llvm::json::Object summary{{"index", static_cast(i)}}; + OwningOpRef candidate; + std::string scheduleReport; + { + ScopedCompilerOutputSuppression silence; + if (failed(realizeCandidate(checkpoint, prepared.candidates[i], candidate, scheduleReport, summary))) + costs[i].reason = "Spatial schedule realization failed verification"; + else costs[i] = spatialCost(*candidate, summary); + } + realization += summary.getNumber("realization_seconds").value_or(0); + extraction += costs[i].timing.extractionSeconds; + expansion += costs[i].timing.expansionSeconds; + queue += costs[i].timing.evaluationSeconds; + bool supported = costs[i].loweringSupport == StaticLoweringSupport::Supported; + auto convergence = costs[i].convergenceProved(); + auto support = costs[i].loweringSupported(); + bool converged = convergence == true; + bool safe = costs[i].eligible(); + summary["cost_available"] = costs[i].available(); + summary["convergence_proved"] = convergence + ? llvm::json::Value(*convergence) : llvm::json::Value(nullptr); + summary["lowering_supported"] = support + ? llvm::json::Value(*support) : llvm::json::Value(nullptr); + summary["eligible"] = safe; + if (costs[i].primary) summary["interval_ns"] = *costs[i].primary; + if (costs[i].alternative) summary["alternative_tie_interval_ns"] = *costs[i].alternative; + if (!safe) { + std::string property; + if (!costs[i].available()) property += "unavailable timing; "; + if (!supported) property += "unsupported or unknown static lowering; "; + if (!converged) property += "unproved modeled-state convergence; "; + if (property.empty()) property = "unavailable selection margin"; + summary["reason"] = property + ": " + costs[i].reason; + errors += "\ncandidate " + std::to_string(i) + ": " + property + ": " + costs[i].reason; + } + summaries.push_back(std::move(summary)); + } + report["candidates"] = std::move(summaries); + report["candidate_realization_seconds"] = realization; + report["event_extraction_seconds"] = extraction; + report["event_expansion_seconds"] = expansion; + report["queue_seconds"] = queue; + if (!errors.empty()) { + report["status"] = "incomplete-costs"; + report["reason"] = errors; + checkpoint.emitError("Spatial pipeline cannot compare the complete candidate set:") << errors; + return failure(); + } return success(); } void runOnOperation() override { - ModuleOp module = getOperation(); + ModuleOp checkpoint = getOperation(); auto start = Clock::now(); - std::string unsupported; - if (!target.prepareScheduleCost) - unsupported = target.wholeScheduleCostUnavailable.empty() - ? "target has no whole-schedule cost query" : target.wholeScheduleCostUnavailable; - auto prepared = prepareSpatialSchedules(module, target, stages, unsupported.empty()); + auto prepared = prepareSpatialSchedules(checkpoint, target, stages, /*collectCandidates=*/true); if (failed(prepared)) { signalPassFailure(); return; } - bool singleCandidate = unsupported.empty() && prepared->candidates.size() == 1; - double generationSeconds = elapsed(start); - std::vector scores(prepared->candidates.size()); - llvm::json::Array summaries; - OwningOpRef bestModule; - std::string bestScheduleReport; - size_t bestIndex = 0; - double loweringSeconds = 0, instructionSeconds = 0, queueSeconds = 0; - size_t lowered = 0, analyzed = 0, unavailable = 0, duplicates = 0; - for (size_t i = 0; i < prepared->candidates.size(); ++i) { - llvm::json::Object summary{{"index", static_cast(i)}}; - if (!unsupported.empty()) { - scores[i].reason = unsupported; - } else if (singleCandidate) { - scores[i].reason = "one distinct generated schedule; no cost comparison required"; - } else { - ScopedCompilerOutputSuppression silence; - OwningOpRef candidate; - std::string scheduleReport; - if (failed(evaluate(module, prepared->candidates[i], candidate, scheduleReport, summary))) { - if (!i) { signalPassFailure(); return; } - scores[i].reason = "candidate lowering failed verification"; - } else { - ++lowered; - loweringSeconds += *summary.getNumber("lowering_seconds"); - auto query = target.prepareScheduleCost(*candidate, summary); - if (failed(query)) { - if (!i) { signalPassFailure(); return; } - scores[i].reason = "candidate cost preparation failed verification"; - ++unavailable; - summary["eligible"] = false; - summary["reason"] = scores[i].reason; - summaries.push_back(std::move(summary)); - continue; - } - instructionSeconds += summary.getNumber("instruction_generation_seconds").value_or(0); - scores[i].identity = query->identity; - auto duplicate = std::find_if(scores.begin(), scores.begin() + i, - [&](const WholeScheduleCost &other) { return other.identity == scores[i].identity; }); - if (duplicate != scores.begin() + i) { - ++duplicates; - scores[i].reason = "duplicate emitted instruction program"; - summary["duplicate_of"] = static_cast(duplicate - scores.begin()); - } else { - auto begin = Clock::now(); - scores[i] = query->evaluate(summary); - scores[i].identity = query->identity; - double seconds = elapsed(begin); - queueSeconds += seconds; - ++analyzed; - summary["queue_seconds"] = seconds; - } - std::string partialReason; - size_t partialSelection = selectPipelineCandidate( - llvm::ArrayRef(scores.data(), i + 1), partialReason); - if (!bestModule || partialSelection == i) { - bestIndex = i; - bestModule = std::move(candidate); - bestScheduleReport = std::move(scheduleReport); - } - } - } - if (!singleCandidate && !scores[i].eligible() - && scores[i].reason != "duplicate emitted instruction program") ++unavailable; - summary["eligible"] = scores[i].eligible(); - summary["identity"] = scores[i].identity; - if (scores[i].primary) summary["interval_ns"] = *scores[i].primary; - if (scores[i].alternative) summary["alternative_tie_interval_ns"] = *scores[i].alternative; - if (scores[i].relativeUncertainty) - summary["relative_prediction_uncertainty"] = *scores[i].relativeUncertainty; - if (!scores[i].reason.empty()) summary["reason"] = scores[i].reason; - summaries.push_back(std::move(summary)); + if (prepared->candidates.empty()) { + checkpoint.emitError("Spatial pipeline requires at least one candidate schedule"); + signalPassFailure(); return; } + bool single = prepared->candidates.size() == 1; + llvm::json::Object report{{"strategy", "spatial-queue-aware"}, {"static_analysis", true}, + {"candidate_generation_seconds", elapsed(start)}, + {"generated_candidates", static_cast(prepared->candidates.size())}, + {"single_candidate_analysis_skipped", single}, {"selection_margin", kPipelineSelectionMargin}, + {"prediction_error_bound_available", false}, {"temporary_disk_bytes", 0}}; + std::vector costs(prepared->candidates.size()); + if (!single && failed(evaluateSpatialCandidates(checkpoint, *prepared, costs, report))) { + writeSelectionReport(std::move(report)); signalPassFailure(); return; + } + if (single) report["candidates"] = llvm::json::Array{llvm::json::Object{{"index", 0}}}; + OwningOpRef winner; + std::string scheduleReport; + llvm::json::Array attempts; + double realizationSeconds = 0, backendSeconds = 0; + size_t lowered = 0; + auto attempt = [&](size_t index) { + llvm::json::Object summary{{"candidate", static_cast(index)}}; + std::string capturedReport; + OwningOpRef candidate; + LogicalResult realized = failure(); + { + ScopedCompilerOutputSuppression silence; + realized = realizeCandidate(checkpoint, prepared->candidates[index], candidate, capturedReport, summary); + } + PimCandidateResult result = PimCandidateResult::Failure; + if (failed(realized)) { + summary["reason"] = "selected Spatial schedule failed deterministic realization"; + summary["status"] = "realization-failed"; + } else { + ++lowered; + result = lowerPimPipelineCandidate(*candidate, resources, summary); + } + realizationSeconds += summary.getNumber("realization_seconds").value_or(0); + backendSeconds += summary.getNumber("backend_seconds").value_or(0); + if (result == PimCandidateResult::Success) { + summary["status"] = "lowered"; + winner = std::move(candidate); + scheduleReport = std::move(capturedReport); + } else if (result == PimCandidateResult::ResourceLimit) + llvm::errs() << "Spatial pipeline candidate " << index + << " rejected by a backend resource limit; rejection recorded in pipeline_selection.json\n"; + attempts.push_back(std::move(summary)); + return result; + }; std::string reason; - size_t selected = selectPipelineCandidate(scores, reason); - if (singleCandidate) reason = scores.front().reason; - else if (!unsupported.empty()) reason += ": " + unsupported; - else if (!scores.front().eligible() && !scores.front().reason.empty()) - reason += ": " + scores.front().reason; - auto finalizeStart = Clock::now(); + std::optional selected; + if (single) { + if (attempt(0) == PimCandidateResult::Success) selected = 0; + reason = selected ? "one distinct schedule; no cost comparison required" + : "the single schedule failed backend lowering; no alternatives are available"; + } else selected = lowerSelectedPimCandidate(costs, attempt, reason); + report["backend_attempts"] = std::move(attempts); + report["lowered_candidates"] = static_cast(lowered); + report["costed_candidates"] = single ? 0 : static_cast(costs.size()); + size_t attemptCount = report.getArray("backend_attempts")->size(); + report["backend_retries"] = static_cast(attemptCount ? attemptCount - 1 : 0); + report["winner_realization_seconds"] = realizationSeconds; + report["backend_lowering_seconds"] = backendSeconds; + if (!selected) { + report["status"] = "backend-failure"; + report["reason"] = reason; + std::string failures; + for (auto &entry : *report.getArray("backend_attempts")) { + auto *failed = entry.getAsObject(); + failures += "\ncandidate " + std::to_string(*failed->getInteger("candidate")) + + ": " + failed->getString("reason").value_or("backend failure").str(); + } + checkpoint.emitError("Spatial pipeline backend selection failed; ") << reason << failures; + writeSelectionReport(std::move(report)); signalPassFailure(); return; + } + auto finalize = Clock::now(); bool spatialArtifacts = exportStage != SpatialDataflowExportStage::None - || pimPeriodicDiagnostics || isPimIrDumpEnabled(); - // Explicit Spatial diagnostics require the pre-lowering representation. - if (!bestModule || spatialArtifacts) { - if (failed(realizeSpatialSchedule(module, prepared->candidates[selected], target, - stages, exportStage, pimPeriodicDiagnostics ? &prepared->graph : nullptr))) { signalPassFailure(); return; } + || pimPeriodicDiagnostics || isPimIrDumpEnabled(); + if (spatialArtifacts) { + if (failed(realizeSpatialSchedule(checkpoint, prepared->candidates[*selected], target, + stages, exportStage, pimPeriodicDiagnostics ? &prepared->graph : nullptr))) { + signalPassFailure(); return; + } } else { auto file = openDialectDumpFileWithExtension("spatial3_scheduled_no_comm", "/reports", "txt"); - if (file) file << bestScheduleReport; + if (file) file << scheduleReport; } - if (bestModule) { - assert(selected == bestIndex && "retained module must be the selected candidate"); - module->setAttrs((*bestModule)->getAttrs()); - module.getBodyRegion().takeBody(bestModule->getBodyRegion()); - } else if (failed(lower(module))) { signalPassFailure(); return; } - llvm::errs() << "Queue-aware pipeline: candidate " << selected << " of " << scores.size() + checkpoint->setAttrs((*winner)->getAttrs()); + checkpoint.getBodyRegion().takeBody(winner->getBodyRegion()); + report["selection"] = static_cast(*selected); + report["status"] = "selected"; + report["reason"] = reason; + report["finalization_seconds"] = elapsed(finalize); + report["total_seconds"] = elapsed(start); + llvm::errs() << "Spatial queue-aware pipeline: candidate " << *selected << " of " << costs.size() << "; " << reason << "\n"; - llvm::json::Object report{{"selection", static_cast(selected)}, {"reason", reason}, - {"static_analysis", true}, {"strategy", "queue-aware"}, {"candidates", std::move(summaries)}, - {"single_candidate_analysis_skipped", singleCandidate}, - {"candidate_generation_seconds", generationSeconds}, {"candidate_lowering_seconds", loweringSeconds}, - {"instruction_generation_seconds", instructionSeconds}, - {"queue_seconds", queueSeconds}, {"finalization_seconds", elapsed(finalizeStart)}, {"total_seconds", elapsed(start)}, - {"generated_candidates", static_cast(scores.size())}, - {"lowered_candidates", static_cast(lowered)}, - {"reused_lowered_candidate", static_cast(bestModule)}, - {"reconstructed_spatial_artifacts", spatialArtifacts && static_cast(bestModule)}, - {"analyzed_candidates", static_cast(analyzed)}, - {"unavailable_estimates", static_cast(unavailable)}, - {"duplicate_programs", static_cast(duplicates)}, {"temporary_disk_bytes", 0}}; - auto file = openReportFileWithExtension("pipeline_selection", "json"); - if (file) { llvm::raw_os_ostream out(file); out << llvm::formatv("{0:2}\n", llvm::json::Value(std::move(report))); } + writeSelectionReport(std::move(report)); } }; } // namespace +PimCandidateResult onnx_mlir::lowerPimPipelineCandidate(ModuleOp module, + const SpatialTargetResources &resources, llvm::json::Object &summary) { + bool classified = true, anyError = false; + std::string diagnostic; + size_t count = 0; + ScopedDiagnosticHandler handler(module.getContext(), [&](Diagnostic &message) { + if (message.getSeverity() == DiagnosticSeverity::Error) { + anyError = true; + classified &= pim::isCandidateResourceError(message); + if (++count <= 8) diagnostic += message.str().substr(0, 2048) + "\n"; + } + return success(); + }); + auto start = Clock::now(); + PassManager pm(module.getContext()); + addPimLoweringPasses(pm, resources, EmitPimCodegen, /*reportProgress=*/false); + if (enableTiming) pm.addInstrumentation(std::make_unique(summary)); + LogicalResult result = pm.run(module); + summary["backend_seconds"] = elapsed(start); + if (succeeded(result)) return PimCandidateResult::Success; + summary["reason"] = diagnostic.empty() ? "backend failed without a classified diagnostic" : diagnostic; + bool resource = anyError && classified; + summary["status"] = resource ? "resource-limit" : "systemic-or-unclassified-error"; + return resource ? PimCandidateResult::ResourceLimit : PimCandidateResult::Failure; +} + +std::optional onnx_mlir::lowerSelectedPimCandidate(llvm::ArrayRef costs, + const std::function &lower, std::string &reason) { + if (costs.empty()) { + reason = "Spatial pipeline requires at least one candidate schedule"; + return std::nullopt; + } + for (const auto &cost : costs) + if (!cost.eligible()) { + reason = "cannot compare the complete candidate set: unavailable cost or unproved eligibility"; + return std::nullopt; + } + std::vector remaining(costs.begin(), costs.end()); + std::vector indices; + for (size_t i = 0; i < costs.size(); ++i) indices.push_back(i); + while (!indices.empty()) { + size_t selected = selectPipelineCandidate(remaining, reason); + auto result = lower(indices[selected]); + if (result == PimCandidateResult::Success) return indices[selected]; + if (result != PimCandidateResult::ResourceLimit) { + reason = "systemic or unclassified backend failure; recovery is not justified"; + return std::nullopt; + } + remaining.erase(remaining.begin() + selected); + indices.erase(indices.begin() + selected); + } + reason = "all candidate schedules failed backend resource constraints"; + return std::nullopt; +} + std::unique_ptr onnx_mlir::createPimPipelineSelectionPass( const SchedulingTarget &target, const SpatialTargetResources &resources, size_t stages, SpatialDataflowExportStage exportStage) { diff --git a/src/PIM/Compiler/PimPipelineSelection.hpp b/src/PIM/Compiler/PimPipelineSelection.hpp index 4c7d972..ffc4d96 100644 --- a/src/PIM/Compiler/PimPipelineSelection.hpp +++ b/src/PIM/Compiler/PimPipelineSelection.hpp @@ -1,8 +1,21 @@ #pragma once #include "mlir/Pass/Pass.h" +#include "llvm/ADT/ArrayRef.h" +#include +#include +#include + +namespace mlir { class ModuleOp; } +namespace llvm::json { class Object; } namespace onnx_mlir { -namespace spatial { struct SchedulingTarget; struct SpatialTargetResources; enum class SpatialDataflowExportStage; } +namespace spatial { struct SchedulingTarget; struct SpatialTargetResources; struct WholeScheduleCost; enum class SpatialDataflowExportStage; } +enum class PimCandidateResult { Success, ResourceLimit, Failure }; +PimCandidateResult lowerPimPipelineCandidate(mlir::ModuleOp module, + const spatial::SpatialTargetResources &resources, llvm::json::Object &summary); +// Callback failures distinguish owner-classified resource limits from other errors. +std::optional lowerSelectedPimCandidate(llvm::ArrayRef costs, + const std::function &lower, std::string &reason); std::unique_ptr createPimPipelineSelectionPass( const spatial::SchedulingTarget &target, const spatial::SpatialTargetResources &resources, size_t stages, spatial::SpatialDataflowExportStage exportStage); diff --git a/src/PIM/Compiler/PimQueueReporting.cpp b/src/PIM/Compiler/PimQueueReporting.cpp new file mode 100644 index 0000000..6a8119d --- /dev/null +++ b/src/PIM/Compiler/PimQueueReporting.cpp @@ -0,0 +1,125 @@ +#include "PimPeriodicAnalysis.hpp" + +using namespace onnx_mlir; + +namespace { +const char *windowStatus(const PimQueuePolicyResult &policy) { + if (policy.legacy.intervalNs) return "estimated"; + return policy.legacy.termination == PimQueueTermination::NotRun && policy.recurrence.proved() + ? "not-required" : "unavailable"; +} + +const char *terminationName(PimQueueTermination termination) { + switch (termination) { + case PimQueueTermination::NotRun: return "not-run"; + case PimQueueTermination::IntervalConverged: return "interval-converged"; + case PimQueueTermination::RecurrenceProved: return "recurrence-proved"; + case PimQueueTermination::EventBudget: return "event-budget"; + case PimQueueTermination::GenerationLimit: return "generation-limit"; + case PimQueueTermination::HistoryLimit: return "history-limit"; + case PimQueueTermination::NonMonotonic: return "non-monotonic"; + case PimQueueTermination::NoReadyEvents: return "no-ready-events"; + } + llvm_unreachable("invalid queue termination"); +} + +void serializeProgress(llvm::json::Object &report, const PimQueueProgress &progress, + bool estimated, bool details, bool intervals) { + if (progress.processedEvents) report["processed_events"] = progress.processedEvents; + if (estimated) { + report["completed_generations"] = progress.completedGenerations; + report["maximum_queue_wait_ns_including_bootstrap"] = progress.maximumQueueWaitNs; + if (progress.maximumQueueRequest) { + report["maximum_queue_core"] = progress.maximumQueueRequest->core; + report["maximum_queue_pc"] = progress.maximumQueueRequest->pc; + } + if (details) { + llvm::json::Array cores; + for (auto [core, wait] : progress.perCoreQueueWaitNs) + cores.push_back(llvm::json::Object{{"core", core}, {"queue_wait_ns", wait}}); + report["per_core"] = std::move(cores); + } + } + if (progress.generationLimit) { + report["generation_limit_core"] = progress.generationLimit->core; + report["generation_limit_pc"] = progress.generationLimit->pc; + report["generation_limit_kind"] = progress.generationLimit->kind; + } + if (intervals) { + llvm::json::Array values; + for (uint64_t value : progress.completedIntervalsNs) values.push_back(value); + report["completed_interval_ns"] = std::move(values); + } +} + +llvm::json::Object serializeWindow(const PimQueuePolicyResult &policy, bool details) { + const auto &window = policy.legacy; + llvm::json::Object result{{"status", windowStatus(policy)}}; + if (window.intervalNs) { + result["interval_ns"] = *window.intervalNs; + result["tie_policy"] = policy.reverseTies ? "reverse event-index order for simultaneous readiness" + : "event-index order for simultaneous readiness"; + } + if (!window.reason.empty()) result["reason"] = window.reason; + serializeProgress(result, window.progress, window.intervalNs.has_value(), details, + window.termination == PimQueueTermination::EventBudget + || window.termination == PimQueueTermination::NoReadyEvents); + return result; +} + +llvm::json::Object serializeRecurrence(const PimQueuePolicyResult &policy) { + const auto &evidence = policy.recurrence; + llvm::json::Object result{{"status", evidence.proved() ? "proved" : "unproved"}, + {"termination", terminationName(policy.termination)}}; + if (policy.checkpoints) { + result["checkpoints"] = policy.checkpoints; + result["history_bytes"] = policy.historyBytes; + } + if (evidence.proved()) { + result["period_generations"] = evidence.periodGenerations; + result["interval_ns"] = *evidence.intervalNs; + result["state_words"] = policy.stateWords; + result["complete_scheduler_state"] = true; + result["proof_processed_events"] = policy.proofProcessedEvents; + } + if (!evidence.reason.empty()) result["reason"] = evidence.reason; + return result; +} + +void addAlternative(llvm::json::Object &report, const PimQueuePolicyResult &policy, bool legacy) { + auto interval = legacy ? policy.legacy.intervalNs : policy.intervalNs(); + report["alternative_tie_status"] = legacy ? windowStatus(policy) : interval ? "estimated" : "unavailable"; + if (interval) report["alternative_tie_interval_ns"] = *interval; + if ((legacy || !policy.recurrence.proved()) && !policy.legacy.reason.empty()) + report["alternative_tie_reason"] = policy.legacy.reason; +} +} // namespace + +llvm::json::Object onnx_mlir::serializePimQueueEvaluation(const PimQueueEvaluation &evaluation) { + const auto &primary = evaluation.policies[0]; + const auto &alternative = evaluation.policies[1]; + llvm::json::Object report; + if (evaluation.recurrenceRequired && primary.recurrence.proved()) { + report = llvm::json::Object{{"status", "estimated"}, {"interval_ns", *primary.intervalNs()}, + {"tie_policy", primary.reverseTies ? "reverse event-index order for simultaneous readiness" + : "event-index order for simultaneous readiness"}}; + serializeProgress(report, primary.progress, true, evaluation.details, false); + } else report = serializeWindow(primary, evaluation.details); + if (evaluation.recurrenceRequired) { + auto legacy = serializeWindow(primary, evaluation.details); + legacy["termination"] = terminationName(primary.legacy.termination); + legacy["alternative_termination"] = terminationName(alternative.legacy.termination); + addAlternative(legacy, alternative, true); + report["interval_source"] = primary.recurrence.proved() ? "complete-state recurrence" : "legacy interval window"; + report["alternative_interval_source"] = alternative.recurrence.proved() ? "complete-state recurrence" : "legacy interval window"; + report["periodic_state"] = serializeRecurrence(primary); + report["alternative_periodic_state"] = serializeRecurrence(alternative); + report["legacy_interval_window"] = std::move(legacy); + } + addAlternative(report, alternative, false); + if (auto interval = primary.intervalNs()) + report["memory_service_fraction"] = evaluation.memoryServiceNs / *interval; + if (evaluation.measureTime) report["analysis_ms"] = evaluation.evaluationSeconds * 1000; + report["arbitration"] = "FIFO by predicted controller arrival"; + return report; +} diff --git a/src/PIM/Compiler/PimScheduleDiagnostic.cpp b/src/PIM/Compiler/PimScheduleDiagnostic.cpp index 09c43ee..44a21dc 100644 --- a/src/PIM/Compiler/PimScheduleDiagnostic.cpp +++ b/src/PIM/Compiler/PimScheduleDiagnostic.cpp @@ -357,6 +357,7 @@ static mlir::FailureOr analyzePlan(PeriodicPlan &pla return analyzePeriodicDependencies(plan.events.size(), plan.edges, plan.error); } + } // namespace llvm::json::Object getPeriodicScheduleReport(const ComputeGraph &graph, diff --git a/src/PIM/Compiler/PimSpatialCost.cpp b/src/PIM/Compiler/PimSpatialCost.cpp new file mode 100644 index 0000000..80ceab6 --- /dev/null +++ b/src/PIM/Compiler/PimSpatialCost.cpp @@ -0,0 +1,623 @@ +#include "PimSpatialCost.hpp" +#include "PimInstructionTiming.hpp" +#include "PimTimingArithmetic.hpp" +#include "PimBinaryFormat.hpp" +#include "PimCompilerUtils.hpp" +#include "PimCompilerOptions.hpp" +#include "PimScheduleDiagnostic.hpp" +#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp" +#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp" +#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp" +#include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp" +#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp" +#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/DeferredProjectionAnalysis.hpp" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/IR/Matchers.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/Support/MathExtras.h" +#include +#include +#include +#include + +using namespace mlir; +using namespace onnx_mlir; +using namespace onnx_mlir::spatial; +using pim_timing::add; +using pim_timing::mul; +using pim_timing::ceil; +namespace { +using Event = PimExecutionEvent; +using Sequence = std::vector; +enum class Residence { Local, Host, Constant, Zero, CapturedWeight }; + +PimPeriodicEstimate unavailableCost(llvm::StringRef reason) { + PimPeriodicEstimate result; + result.cost.reason = reason.str(); + result.report = llvm::json::Object{{"status", "unavailable"}, {"cost_available", false}, + {"convergence_proved", nullptr}, {"lowering_supported", nullptr}, + {"safe_for_selection", false}, {"reason", reason.str()}}; + return result; +} + +class Extraction { +public: + const PimInstructionTiming ⌖ + bool diagnostics; + PimExecutionProgram program; + std::string error, loweringReason; + StaticLoweringSupport loweringSupport = StaticLoweringSupport::Supported; + std::map gaps; + int64_t visited = 0, constantLoads = 0; + explicit Extraction(const PimInstructionTiming &target, bool diagnostics) + : target(target), diagnostics(diagnostics), program(target.processorCount) {} + bool build(ModuleOp module); +private: + llvm::SmallDenseSet assignedCores; + DenseMap residence; + DenseMap publicationAccess; + DenseMap> directOutputs; + llvm::SmallDenseSet compactStorage; + llvm::SmallPtrSet indexed; + StaticIndexEnvironment environment; + Residence storage(Value value); + uint64_t elements(Type type) { + auto shape = dyn_cast(type); + if (!shape || !shape.hasStaticShape()) { + error = "abstract work requires a static shaped payload"; + return 0; + } + if (!shape.getElementType().isIntOrFloat() || shape.getElementTypeBitWidth() != 32) { + error = "abstract precision projection requires 32-bit elements"; + return 0; + } + uint64_t result = 1; + for (int64_t dimension : shape.getShape()) result = mul(result, dimension); + return result; + } + uint64_t bytes(Type type) { + uint64_t count = elements(type); + auto shaped = dyn_cast(type); + if (!shaped || !shaped.getElementType().isIntOrFloat()) { + error = "abstract transfer requires a scalar integer or floating element"; + return 0; + } + uint64_t nativeBytes = ceil(mul(count, shaped.getElementTypeBitWidth()), 8); + if (nativeBytes > std::numeric_limits::max()) { + loweringSupport = StaticLoweringSupport::Unknown; + loweringReason = "native transfer size requires backend normalization and size verification"; + } + return ceil(mul(count, target.computeBitwidth), 8); + } + void work(Sequence &sequence, uint64_t ns, Event::WorkClass resource = Event::LocalCopy) { + if (!ns) return; + if (!sequence.empty() && sequence.back().kind == Event::Work + && sequence.back().workClass == resource) + sequence.back().amount = add(sequence.back().amount, ns); + else { + sequence.push_back({Event::Work, ns}); + sequence.back().workClass = resource; + } + } + uint64_t localWork(uint64_t bytes, bool read, bool write) const { + return getPimLocalTransferWorkNs(target, bytes, read, write); + } + void materialize(Value value, Sequence &sequence); + bool accessSlice(tensor::ExtractSliceOp slice, Sequence &sequence, bool host); + bool block(Block &body, Sequence &sequence); + bool operation(Operation &op, Sequence &sequence); + bool loop(scf::ForOp op, Sequence &sequence); + bool core(Operation *op, int64_t physicalCore, unsigned lane); +}; + +Residence Extraction::storage(Value value) { + if (auto it = residence.find(value); it != residence.end()) return it->second; + Attribute constant; + if (matchPattern(value, m_Constant(&constant))) + return isZeroBitPatternSplat(constant) ? Residence::Zero : Residence::Constant; + if (auto slice = value.getDefiningOp()) return storage(slice.getSource()); + if (auto collapse = value.getDefiningOp()) return storage(collapse.getSrc()); + if (auto expand = value.getDefiningOp()) return storage(expand.getSrc()); + if (auto cast = value.getDefiningOp()) return storage(cast.getSource()); + if (auto branch = value.getDefiningOp()) { + auto condition = evaluateDeferredIndex(branch.getCondition(), environment); + if (succeeded(condition)) { + Region ®ion = *condition ? branch.getThenRegion() : branch.getElseRegion(); + return storage(region.front().getTerminator()->getOperand(cast(value).getResultNumber())); + } + } + if (auto arg = dyn_cast(value)) { + if (auto loop = dyn_cast_or_null(arg.getOwner()->getParentOp())) + if (arg.getArgNumber()) return Residence::Local; + return Residence::Host; + } + if (isa_and_nonnull(value.getDefiningOp())) + return Residence::Local; + error = "abstract storage residence is unavailable for " + + value.getDefiningOp()->getName().getStringRef().str(); + return Residence::Local; // The caller discards the estimate when error is set. +} + +void Extraction::materialize(Value value, Sequence &sequence) { + auto place = storage(value); + if (place == Residence::Host || place == Residence::Constant || place == Residence::CapturedWeight) { + sequence.push_back({Event::Load, bytes(value.getType())}); + // Captured weights remain block arguments; only direct global zero loads fold to fills. + if (place == Residence::Constant || place == Residence::CapturedWeight) { + ++constantLoads; + sequence.back().constant = true; + } + } else if (place == Residence::Zero) { + work(sequence, localWork(bytes(value.getType()), false, true), Event::Fill); + } +} + +bool Extraction::accessSlice(tensor::ExtractSliceOp slice, Sequence &sequence, bool host) { + auto source = computeRowMajorStridesChecked(slice.getSourceType().getShape()); + auto destination = computeRowMajorStridesChecked(slice.getStaticSizes()); + if (failed(source) || failed(destination)) { error = "abstract slice geometry unavailable"; return false; } + SmallVector strides; + for (auto [stride, scale] : llvm::zip(*source, slice.getStaticStrides())) { + int64_t result; + if (scale <= 0 || llvm::MulOverflow(stride, scale, result)) { + error = "abstract slice stride unavailable"; return false; + } + strides.push_back(result); + } + auto copy = CopyAccessPlan::build(slice.getStaticSizes(), strides, *destination); + if (failed(copy)) { error = "abstract slice access plan unavailable"; return false; } + uint64_t size = ceil(mul(copy->getRunElements(), target.computeBitwidth), 8); + if (host) { + sequence.push_back({Event::Repeat, static_cast(copy->getRunCount()), + 0, 0, 0, {{Event::Load, size}}}); + residence[slice.getResult()] = Residence::Local; + compactStorage.insert(slice.getResult()); + } else if (compactStorage.contains(slice.getSource()) + && slice.getSourceType().getRank() == slice.getType().getRank() + && llvm::all_of(slice.getResult().getUsers(), [](Operation *user) { + return isa(user); + })) { + // Publication-only slices retain the compact source's subview geometry. + publicationAccess.insert_or_assign(slice.getResult(), *copy); + } else if (copy->getRunCount() > 1) { + if (diagnostics) ++gaps["local slice materialization and forwarding"]; + work(sequence, mul(copy->getRunCount(), localWork(size, true, true))); + } + if (compactStorage.contains(slice.getSource()) && copy->getRunCount() == 1) + compactStorage.insert(slice.getResult()); + return true; +} + +bool Extraction::loop(scf::ForOp op, Sequence &sequence) { + auto lower = evaluateDeferredIndex(op.getLowerBound(), environment); + auto upper = evaluateDeferredIndex(op.getUpperBound(), environment); + auto step = evaluateDeferredIndex(op.getStep(), environment); + if (failed(lower) || failed(upper) || failed(step) || *step <= 0) { + error = "abstract loop has unavailable static bounds"; return false; + } + for (Value initial : op.getInitArgs()) materialize(initial, sequence); + int64_t distance = 0; + if (*upper > *lower && llvm::SubOverflow(*upper, *lower, distance)) { + error = "unrepresentable abstract loop trip count"; return false; + } + uint64_t count = ceil(distance, *step); + bool indexSensitive = indexed.contains(op); + if (indexSensitive && count > 100000) { + error = "abstract indexed loop exceeds analysis budget"; return false; + } + if (!count) return true; + if (indexSensitive) { + for (uint64_t i = 0; i < count; ++i) { + int64_t induction; + if (llvm::AddOverflow(*lower, static_cast(mul(i, *step)), induction)) { + error = "unrepresentable abstract induction value"; return false; + } + environment.bindings[op.getInductionVar()] = induction; + if (!block(*op.getBody(), sequence)) return false; + } + } else { + environment.bindings[op.getInductionVar()] = *lower; + Event repeat{Event::Repeat, count}; + if (!block(*op.getBody(), repeat.body)) return false; + sequence.push_back(std::move(repeat)); + } + environment.bindings.erase(op.getInductionVar()); + return true; +} + +bool Extraction::operation(Operation &op, Sequence &sequence) { + if (auto loopOp = dyn_cast(op)) return loop(loopOp, sequence); + if (auto branch = dyn_cast(op)) { + auto condition = evaluateDeferredIndex(branch.getCondition(), environment); + if (failed(condition)) { error = "abstract branch condition is unavailable"; return false; } + Region ®ion = *condition ? branch.getThenRegion() : branch.getElseRegion(); + return region.empty() || block(region.front(), sequence); + } + auto index = [&](Value value) -> int64_t { + auto result = evaluateDeferredIndex(value, environment); + if (failed(result)) { error = "abstract communication index is unavailable"; return -1; } + return *result; + }; + if (auto sync = dyn_cast(op)) { + sequence.push_back({Event::Signal, 0, index(sync.getTargetCoreId()), index(sync.getEventRegister())}); + } else if (auto wait = dyn_cast(op)) { + int64_t count = index(wait.getWaitValue()); + if (count) sequence.push_back({Event::Wait, 0, 0, index(wait.getEventRegister()), count}); + } else if (auto send = dyn_cast(op)) { + materialize(send.getInput(), sequence); + sequence.push_back({Event::Send, bytes(send.getInput().getType()), index(send.getTargetCoreId())}); + } else if (auto recv = dyn_cast(op)) { + sequence.push_back({Event::Receive, bytes(recv.getOutput().getType()), index(recv.getSourceCoreId())}); + } else if (auto store = dyn_cast(op)) { + materialize(store.getInput(), sequence); + if (auto plan = publicationAccess.find(store.getInput()); plan != publicationAccess.end()) + sequence.push_back({Event::Repeat, static_cast(plan->second.getRunCount()), + 0, 0, 0, {{Event::Store, ceil(mul(plan->second.getRunElements(), target.computeBitwidth), 8)}}}); + else sequence.push_back({Event::Store, bytes(store.getInput().getType())}); + sequence.push_back({Event::Signal, 0, index(store.getTargetCoreId()), index(store.getEventRegister())}); + } else if (auto load = dyn_cast(op)) { + int64_t count = index(load.getWaitValue()); + if (count) sequence.push_back({Event::Wait, 0, 0, index(load.getEventRegister()), count}); + sequence.push_back({Event::Load, bytes(load.getOutput().getType())}); + } else if (auto vmm = dyn_cast(op)) { + materialize(vmm.getInput(), sequence); + auto weight = dyn_cast(vmm.getWeight().getType()); + if (!weight || weight.getRank() != 2 || !weight.hasStaticShape() || !target.matrixColumns + || weight.getDimSize(1) <= 0 + || static_cast(weight.getDimSize(1)) % target.matrixColumns + || weight.getDimSize(0) <= 0 + || static_cast(weight.getDimSize(0)) > target.matrixRows + || weight.getEncoding() || !weight.getElementType().isIntOrFloat() + || weight.getElementTypeBitWidth() != 32) { + error = "unavailable physical matrix weight geometry"; return false; + } + if (diagnostics && static_cast(weight.getDimSize(0)) < target.matrixRows) + ++gaps["matrix input padding and copy forwarding"]; + uint64_t output = getPimMatrixOutputBytes(target, target.matrixCellBits, + pim_binary::kMatrixWeightPrecisionBits, weight.getDimSize(1) / target.matrixColumns, + ceil(target.computeBitwidth, 8)); + work(sequence, getPimMatrixWorkNs(target, target.computeBitwidth, output), Event::Matrix); + } else if (isa(op)) { + for (Value operand : op.getOperands()) materialize(operand, sequence); + uint64_t width = ceil(target.computeBitwidth, 8); + work(sequence, getPimVectorWorkNs(target, elements(op.getOperand(0).getType()), + width, width, op.getNumOperands() == 2), Event::Vector); + } else if (auto extract = dyn_cast(op)) { + auto place = storage(extract.getSource()); + if (place == Residence::Host || place == Residence::Local) + if (!accessSlice(extract, sequence, place == Residence::Host)) return false; + } else if (auto insert = dyn_cast(op)) { + materialize(insert.getSource(), sequence); + uint64_t size = bytes(insert.getSource().getType()); + work(sequence, localWork(size, true, true)); + if (compactStorage.contains(insert.getDest())) compactStorage.insert(insert.getResult()); + if (diagnostics) ++gaps["output copy forwarding"]; + } else if (auto transpose = dyn_cast(op)) { + materialize(transpose.getInput(), sequence); + auto type = cast(transpose.getInput().getType()); + uint64_t volume = elements(type); + if (!volume) { + if (error.empty()) error = "zero-element transpose has no abstract timing contract"; + return false; + } + if (failed(computeRowMajorStridesChecked(type.getShape()))) { + error = "abstract transpose geometry unavailable"; return false; + } + auto run = getTransposeRun(type.getShape(), transpose.getPermutation()); + uint64_t width = ceil(target.computeBitwidth, 8); + uint64_t runBytes = mul(run.elements, width); + uint64_t cost = run.sourceStride == 1 + ? localWork(runBytes, true, true) + : getPimVectorWorkNs(target, run.elements, width, width, false); + work(sequence, mul(volume / run.elements, cost), + run.sourceStride == 1 ? Event::LocalCopy : Event::VectorCopy); + if (diagnostics) ++gaps["transpose alias and constant folding"]; + } else if (auto pad = dyn_cast(op)) { + materialize(pad.getSource(), sequence); + work(sequence, localWork(bytes(pad.getResult().getType()), false, true), Event::Fill); + work(sequence, localWork(bytes(pad.getSource().getType()), true, true)); + if (diagnostics) ++gaps["padding fill and copy normalization"]; + } else if (auto concat = dyn_cast(op)) { + for (Value input : concat.getInputs()) { + materialize(input, sequence); + uint64_t size = bytes(input.getType()); + work(sequence, localWork(size, true, true)); + } + compactStorage.insert(concat.getOutput()); + if (diagnostics) ++gaps["concat alias and copy elimination"]; + } else if (auto empty = dyn_cast(op)) { + compactStorage.insert(empty.getResult()); + } else if (isa(op) || isPureIndexComputationOp(&op)) { + // Compile-time shape/index work has no device execution cost. + } else if (auto parallel = dyn_cast(op)) { + auto batch = parallel->getParentOfType(); + auto first = batch ? batch.getOutputArgument(0) : std::nullopt; + if (!first) { error = "unavailable abstract batch output mapping"; return false; } + for (Operation &nested : parallel.getRegion().front()) { + auto insert = dyn_cast(nested); + auto destination = insert ? dyn_cast(insert.getDest()) : BlockArgument{}; + if (!destination || destination.getArgNumber() < first->getArgNumber()) { + error = "unsupported abstract batch output"; return false; + } + unsigned result = destination.getArgNumber() - first->getArgNumber(); + if (result >= batch.getNumResults()) { error = "invalid abstract batch output"; return false; } + if (succeeded(getDirectReturnOperandIndex(cast(batch.getResult(result))))) { + materialize(insert.getSource(), sequence); + sequence.push_back({Event::Store, bytes(insert.getSource().getType())}); + } else if (!batch.getResult(result).use_empty()) { + error = "unavailable indirect batch output collection"; return false; + } + } + } else { + error = "unsupported abstract operation: " + op.getName().getStringRef().str(); + return false; + } + return error.empty(); +} + +bool Extraction::block(Block &body, Sequence &sequence) { + for (Operation &op : body) { + if (++visited > 1000000) { error = "abstract extraction budget exceeded"; return false; } + if (!operation(op, sequence)) return false; + for (Value result : op.getResults()) + if (auto outputs = directOutputs.find(result); outputs != directOutputs.end()) + // Lowering inserts each store immediately after the shared producer. + for (unsigned destination : llvm::reverse(outputs->second)) { + (void)destination; // Distinct host destinations require distinct stores. + sequence.push_back({Event::Store, bytes(result.getType())}); + } + } + return true; +} + +bool Extraction::core(Operation *op, int64_t physicalCore, unsigned lane) { + if (physicalCore < 0 || physicalCore >= static_cast(program.size())) { + loweringSupport = StaticLoweringSupport::Unsupported; + error = "physical core ID is outside the target: " + std::to_string(physicalCore); return false; + } + if (!assignedCores.insert(physicalCore).second) { + loweringSupport = StaticLoweringSupport::Unsupported; + error = "duplicate physical core assignment: " + std::to_string(physicalCore); return false; + } + residence.clear(); publicationAccess.clear(); directOutputs.clear(); compactStorage.clear(); + environment.bindings.clear(); + ValueRange inputs, weights; + auto single = dyn_cast(op); + auto batch = dyn_cast(op); + if (single) { inputs = single.getInputs(); weights = single.getWeights(); } + else { inputs = batch.getInputs(); weights = batch.getWeights(); environment.bindings[*batch.getLaneArgument()] = lane; } + Sequence &sequence = program[physicalCore]; + if (single) { + auto yield = cast(single.getBody().front().getTerminator()); + for (auto [result, value] : llvm::zip(single.getResults(), yield.getOperands())) + if (auto destination = getDirectReturnOperandIndex(cast(result)); succeeded(destination)) { + if (!value.getDefiningOp() || value.getDefiningOp()->getBlock() != &single.getBody().front()) { + error = "unavailable direct output insertion position"; return false; + } + directOutputs[value].push_back(*destination); + } else if (!result.use_empty()) { + error = "unavailable indirect compute output collection"; return false; + } + } + for (unsigned i = 0; i < weights.size(); ++i) + residence[single ? *single.getWeightArgument(i) : *batch.getWeightArgument(i)] = Residence::CapturedWeight; + for (unsigned i = 0; i < inputs.size(); ++i) { + Value argument = single ? *single.getInputArgument(i) : *batch.getInputArgument(i); + auto plan = planComputeInputMaterialization(op, i); + if (failed(plan)) { error = "input materialization plan unavailable"; return false; } + residence[argument] = storage(inputs[i]); + // FuncOpArgToGlobalMemoryPattern removes this input and exposes the host + // tensor in the body before entry-copy materialization runs. + if (*plan == ComputeInputMaterialization::ForwardFunctionInput) { + if (diagnostics) ++gaps["function-input whole-load versus slice materialization"]; + continue; + } + if (*plan == ComputeInputMaterialization::CopyAtEntry) { + materialize(inputs[i], sequence); + residence[argument] = Residence::Local; + compactStorage.insert(argument); + if (diagnostics) ++gaps["input host slicing and receive sinking"]; + } + } + return block(op->getRegion(0).front(), sequence); +} + +bool Extraction::build(ModuleOp module) { + bool okay = true; + module.walk([&](Operation *op) { + if (indexed.contains(op) || isa(op)) + if (Operation *parent = op->getParentOp()) indexed.insert(parent); + }); + module.walk([&](Operation *op) { + if (!okay) return WalkResult::interrupt(); + if (auto single = dyn_cast(op)) { + auto coreId = op->getAttrOfType("coreId"); + if (!coreId) { error = "missing physical core"; okay = false; } + else okay = core(op, coreId.getInt(), 0); + return WalkResult::skip(); + } + if (auto batch = dyn_cast(op)) { + auto ids = op->getAttrOfType("coreIds"); + if (!ids || ids.size() != batch.getLaneCount()) { error = "missing physical batch cores"; okay = false; } + else for (auto [lane, id] : llvm::enumerate(ids.asArrayRef())) if (!(okay = core(op, id, lane))) break; + return WalkResult::skip(); + } + return WalkResult::advance(); + }); + return okay && error.empty(); +} +} // namespace + +PimPeriodicEstimate onnx_mlir::evaluatePimSpatialSchedule(ModuleOp module, + const PimInstructionTiming &target, PimPeriodicOptions options) try { + if (!target.hasPeriodicTransferTiming() || !target.computeBitwidth) + return unavailableCost("unsupported abstract timing contract"); + auto start = std::chrono::steady_clock::now(); + Extraction extraction(target, options.workBreakdown); + bool available = extraction.build(module); + double extractionMs = std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + PimPeriodicEstimate estimate; + if (available) estimate = evaluatePimExecution(extraction.program, target, options); + else { + estimate.cost.reason = extraction.error; + estimate.report = llvm::json::Object{{"status", "unavailable"}, {"reason", extraction.error}}; + if (options.verifyPeriodicState) + for (auto &recurrence : estimate.cost.recurrence) + recurrence.status = ScheduleRecurrenceEvidence::Unproved; + } + auto &report = estimate.report; + auto &cost = estimate.cost; + cost.timing.extractionSeconds = extractionMs / 1000; + if (available || extraction.loweringSupport == StaticLoweringSupport::Unsupported) + cost.loweringSupport = extraction.loweringSupport; + if (available && options.workBreakdown) { + PimPeriodicOptions sensitivity; + sensitivity.ignoreLocalCopies = true; + auto withoutCopies = analyzePimExecution(extraction.program, target, sensitivity); + if (auto *queue = withoutCopies.getObject("memory_controller")) + report["without_local_copy_work"] = std::move(*queue); + } + auto convergence = cost.convergenceProved(); + auto support = cost.loweringSupported(); + report["cost_available"] = cost.primary.has_value() && cost.alternative.has_value(); + report["convergence_proved"] = convergence + ? llvm::json::Value(*convergence) : llvm::json::Value(nullptr); + report["lowering_supported"] = support + ? llvm::json::Value(*support) : llvm::json::Value(nullptr); + std::string failureReason = std::move(cost.reason); + cost.reason = "Spatial prediction accuracy has no formal bound"; + if (!cost.available()) cost.reason = "Spatial queue cost is unavailable"; + else if (cost.loweringSupport != StaticLoweringSupport::Supported) + cost.reason = "Spatial lowering support is unproved"; + else if (convergence != true) + cost.reason = "complete-state recurrence is unproved under one or both arbitration policies"; + if (!extraction.loweringReason.empty()) cost.reason = extraction.loweringReason; + if (!failureReason.empty()) cost.reason = std::move(failureReason); + report["lowering_support_scope"] = "static operation and transfer contract; final backend verification is required"; + if (!extraction.loweringReason.empty()) report["lowering_reason"] = extraction.loweringReason; + report["safe_for_selection"] = false; + report["diagnostic_only"] = true; + report["error_bound_available"] = false; + report["extraction_ms"] = extractionMs; + report["visited_operations"] = extraction.visited; + if (!options.workBreakdown) return estimate; + report["constant_load_sites"] = extraction.constantLoads; + uint64_t workNs = 0, records = 0, constantRequests = 0, constantBytes = 0; + constexpr const char *workClassNames[] = { + "matrix", "vector", "local_copy", "fill", "vector_copy", "scalar_issue"}; + std::array workByClass{}; + static_assert(Event::ScalarIssue + 1 == std::size(workClassNames)); + std::array counts{}; + std::map readSizes; + std::function summarize = + [&](const Sequence &sequence, uint64_t multiplier) { + for (const Event &event : sequence) { + ++records; + if (event.kind == Event::Repeat) summarize(event.body, mul(multiplier, event.amount)); + else { + counts[event.kind] = add(counts[event.kind], multiplier); + if (event.kind == Event::Work) { + workNs = add(workNs, mul(multiplier, event.amount)); + switch (event.workClass) { + case Event::Matrix: + case Event::Vector: + case Event::LocalCopy: + case Event::Fill: + case Event::VectorCopy: + case Event::ScalarIssue: + workByClass[event.workClass] = add(workByClass[event.workClass], mul(multiplier, event.amount)); + break; + default: throw pim_timing::Unrepresentable{"unsupported execution work class"}; + } + } else if (event.kind == Event::Load) { + readSizes[event.amount] = add(readSizes[event.amount], multiplier); + if (event.constant) { + constantRequests = add(constantRequests, multiplier); + constantBytes = add(constantBytes, mul(multiplier, event.amount)); + } + } + } + } + }; + for (const Sequence &sequence : extraction.program) summarize(sequence, 1); + report["compute_and_local_work_ns"] = workNs; + llvm::json::Array classes; + for (uint64_t value : workByClass) classes.push_back(value); + llvm::json::Array columns; + for (const char *name : workClassNames) columns.push_back(name); + report["work_class_columns"] = std::move(columns); + report["work_class_ns"] = std::move(classes); + llvm::json::Object eventCounts; + const char *names[] = {"work_chunks", "load", "store", "send", "receive", "sync", "wait"}; + for (size_t i = 0; i < counts.size(); ++i) eventCounts[names[i]] = counts[i]; + report["event_counts"] = std::move(eventCounts); + report["compact_records"] = records; + report["constant_read_requests"] = constantRequests; + report["constant_read_bytes"] = constantBytes; + llvm::json::Array sizes; + for (auto [bytes, count] : readSizes) + sizes.push_back(llvm::json::Object{{"bytes", bytes}, {"requests", count}}); + report["read_sizes"] = std::move(sizes); + llvm::json::Object gaps; + for (const auto &[name, count] : extraction.gaps) gaps[name] = count; + report["unresolved_decisions"] = std::move(gaps); + report["omitted_effects"] = llvm::json::Array{"scalar register setup", "indirect output collection", + "host slice splitting and folding", "copy forwarding and local reuse legality"}; + report["analysis_ms"] = std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + return estimate; +} catch (const pim_timing::Unrepresentable &failure) { + return unavailableCost(failure.reason); +} + +llvm::json::Object onnx_mlir::analyzePimSpatialSchedule(ModuleOp module, + const PimInstructionTiming &target, PimPeriodicOptions options) { + return evaluatePimSpatialSchedule(module, target, options).report; +} + +void onnx_mlir::configurePimScheduleCosts(spatial::SchedulingTarget &target, + PimInstructionTiming timing) { + auto contract = getPimPeriodicTimingContract(); + if (contract) timing.matrixCellBits = *contract; + auto hardware = std::make_shared(std::move(timing)); + target.reportPeriodicSchedule = [hardware](const spatial::ComputeGraph &graph, + const spatial::ScheduledComputeMaterializationResult &materialization, + const spatial::DeferredTransferPlan &transfers, + const spatial::DeferredBoundaryPlan &boundaries) { + dumpPeriodicScheduleEstimate(graph, materialization, transfers, boundaries, *hardware); + }; + if (!contract) target.wholeScheduleCostUnavailable = llvm::toString(contract.takeError()); + if (!hardware->hasPeriodicTransferTiming()) + target.wholeScheduleCostUnavailable = "target has no supported queue timing contract"; + if (pimDisableSynchronization) + target.wholeScheduleCostUnavailable = "synchronization ablation has no periodic queue timing contract"; + if (!target.wholeScheduleCostUnavailable.empty()) return; + target.estimateRealizedScheduleCost = [hardware](mlir::ModuleOp module, + llvm::json::Object &summary) { + PimPeriodicOptions options; + options.workBreakdown = pimPeriodicDiagnostics; + options.measureTime = true; + options.verifyPeriodicState = true; + auto estimate = evaluatePimSpatialSchedule(module, *hardware, options); + auto cost = std::move(estimate.cost); + summary = std::move(estimate.report); + if (estimate.queue) + summary["interval_source"] = cost.convergenceProved() == true + ? "complete-state recurrence under both arbitration policies" + : "recurrence is unproved under one or both arbitration policies"; + summary["cost_available"] = cost.available(); + return cost; + }; +} diff --git a/src/PIM/Compiler/PimSpatialCost.hpp b/src/PIM/Compiler/PimSpatialCost.hpp new file mode 100644 index 0000000..5f348c7 --- /dev/null +++ b/src/PIM/Compiler/PimSpatialCost.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include "PimPeriodicAnalysis.hpp" +#include "mlir/IR/BuiltinOps.h" + +namespace onnx_mlir { + +void configurePimScheduleCosts(spatial::SchedulingTarget &target, PimInstructionTiming timing); + +// Read-only, target-specific work extraction. No target dialect is constructed. +PimPeriodicEstimate evaluatePimSpatialSchedule(mlir::ModuleOp module, + const PimInstructionTiming &target, PimPeriodicOptions options = {}); +llvm::json::Object analyzePimSpatialSchedule(mlir::ModuleOp module, + const PimInstructionTiming &target, PimPeriodicOptions options = {}); + +} // namespace onnx_mlir diff --git a/src/PIM/Compiler/PimTimingArithmetic.hpp b/src/PIM/Compiler/PimTimingArithmetic.hpp new file mode 100644 index 0000000..2938d7b --- /dev/null +++ b/src/PIM/Compiler/PimTimingArithmetic.hpp @@ -0,0 +1,37 @@ +#pragma once + +#include +#include + +namespace onnx_mlir::pim_timing { + +struct Unrepresentable { const char *reason; }; + +// Optional timing queries must fail without wrapping or terminating compilation. +struct Arithmetic { + static constexpr uint64_t limit = std::numeric_limits::max(); + unsigned long long add(uint64_t a, uint64_t b) const { + if (a > limit || b > limit - a) throw Unrepresentable{"timing addition overflow"}; + return a + b; + } + unsigned long long multiply(uint64_t a, uint64_t b) const { + if (a > limit || b > limit || (a && b > limit / a)) + throw Unrepresentable{"timing multiplication overflow"}; + return a * b; + } + unsigned long long subtract(uint64_t a, uint64_t b) const { + if (a > limit || b > a) throw Unrepresentable{"timing subtraction underflow"}; + return a - b; + } + unsigned long long divideUp(uint64_t a, uint64_t b) const { + if (!b || a > limit || b > limit) throw Unrepresentable{"unrepresentable timing division"}; + return a / b + (a % b != 0); + } +}; +inline constexpr Arithmetic arithmetic{}; +inline uint64_t add(uint64_t a, uint64_t b) { return arithmetic.add(a, b); } +inline uint64_t mul(uint64_t a, uint64_t b) { return arithmetic.multiply(a, b); } +inline uint64_t sub(uint64_t a, uint64_t b) { return arithmetic.subtract(a, b); } +inline uint64_t ceil(uint64_t a, uint64_t b) { return arithmetic.divideUp(a, b); } + +} // namespace onnx_mlir::pim_timing diff --git a/src/PIM/Conversion/SpatialToPim/BatchCoreLoweringPatterns.cpp b/src/PIM/Conversion/SpatialToPim/BatchCoreLoweringPatterns.cpp index d1cac41..805d57d 100644 --- a/src/PIM/Conversion/SpatialToPim/BatchCoreLoweringPatterns.cpp +++ b/src/PIM/Conversion/SpatialToPim/BatchCoreLoweringPatterns.cpp @@ -15,6 +15,7 @@ #include "src/Accelerators/PIM/Common/PimCommon.hpp" #include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp" #include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp" +#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp" #include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp" using namespace mlir; @@ -34,15 +35,6 @@ static bool isUsedOnlyByExtractSlices(Value value) { && llvm::all_of(value.getUsers(), [](Operation* user) { return isa(user); }); } -static FailureOr getDirectReturnOperandIndex(OpResult result) { - if (!result.hasOneUse()) - return failure(); - - auto returnOp = dyn_cast(*result.getUsers().begin()); - if (!returnOp) - return failure(); - return result.getUses().begin()->getOperandNumber(); -} static FailureOr> collectFragmentAssemblyCopiesFromBlueprint(spatial::SpatBlueprintOp blueprint, @@ -85,7 +77,9 @@ collectFragmentAssemblyCopiesFromBlueprint(spatial::SpatBlueprintOp blueprint, flatStrides))) return failure(); - SmallVector hostStrides = computeRowMajorStrides(resultType.getShape()); + auto checkedStrides = computeRowMajorStridesChecked(resultType.getShape()); + if (failed(checkedStrides)) return failure(); + SmallVector hostStrides = std::move(*checkedStrides); for (int64_t fragmentIndex = 0; fragmentIndex < static_cast(operandIndices.size()); ++fragmentIndex) { Value source = fixedSource ? fixedSource : mapper.lookupOrDefault(fragmentOperands[operandIndices[fragmentIndex]]); auto sourceType = dyn_cast(source.getType()); @@ -183,7 +177,9 @@ collectTopLevelFragmentAssemblyCopies(OpResult result, RankedTensorType packedRe flatSizes, flatStrides))) return failure(); - SmallVector hostStrides = computeRowMajorStrides(hostResultType.getShape()); + auto checkedStrides = computeRowMajorStridesChecked(hostResultType.getShape()); + if (failed(checkedStrides)) return failure(); + SmallVector hostStrides = std::move(*checkedStrides); for (int64_t fragmentIndex = 0; fragmentIndex < static_cast(operandIndices.size()); ++fragmentIndex) { if (operandIndices[fragmentIndex] != static_cast(use.getOperandNumber())) continue; @@ -239,14 +235,16 @@ static Value createScaledIndexValue(IRRewriter& rewriter, Location loc, Value ba return arith::MulIOp::create(rewriter, loc, base, scaleValue).getResult(); } -static Value createHostTargetOffset(IRRewriter& rewriter, +static FailureOr createHostTargetOffset(IRRewriter& rewriter, Location loc, ShapedType destinationType, ArrayRef mixedOffsets, ArrayRef additionalOffsets, IRMapping& mapper) { int64_t elementBytes = static_cast(getElementTypeSizeInBytes(destinationType.getElementType())); - SmallVector strides = computeRowMajorStrides(destinationType.getShape()); + auto checkedStrides = computeRowMajorStridesChecked(destinationType.getShape()); + if (failed(checkedStrides)) return failure(); + SmallVector strides = std::move(*checkedStrides); Value totalOffset; for (auto [dim, offset] : llvm::enumerate(mixedOffsets)) { @@ -277,7 +275,7 @@ static Value createHostTargetOffset(IRRewriter& rewriter, return totalOffset; } -static Value createHostTargetOffset(IRRewriter& rewriter, +static FailureOr createHostTargetOffset(IRRewriter& rewriter, tensor::ParallelInsertSliceOp insertSlice, ShapedType destinationType, IRMapping& mapper) { @@ -380,6 +378,9 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul mapper.map(*oldWeightArg, coreBatchOp.getWeightArgument(weightIndex)); } for (unsigned inputIndex = 0; inputIndex < computeBatchOp.getInputs().size(); ++inputIndex) { + auto inputPlan = planComputeInputMaterialization(computeBatchOp, inputIndex); + if (failed(inputPlan) || *inputPlan != ComputeInputMaterialization::CopyAtEntry) + return computeBatchOp.emitOpError("expected body-entry compute_batch input materialization"); auto oldArg = computeBatchOp.getInputArgument(inputIndex); if (!oldArg) return computeBatchOp.emitOpError("expected compute_batch input block arguments before lowering"); @@ -491,19 +492,20 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul if (failed(fragmentAssemblyRuns)) return failure(); SmallVector zeroOffsets(hostTargetType.getRank(), 0); - Value baseHostOffset = createHostTargetOffset(rewriter, + auto baseHostOffset = createHostTargetOffset(rewriter, blueprint.getLoc(), hostTargetType, insertSlice.getMixedOffsets(), zeroOffsets, mapper); + if (failed(baseHostOffset)) return failure(); FailureOr updatedHostTarget = emitFragmentAssemblyCopyRuns(rewriter, blueprint.getLoc(), *fragmentAssemblyRuns, hostTarget, coreBatchOp.getOperation(), std::nullopt, - baseHostOffset); + *baseHostOffset); if (failed(updatedHostTarget)) return failure(); hostOutputTensors[resultIndex] = *updatedHostTarget; @@ -511,7 +513,8 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul } } - Value hostTargetOffset = createHostTargetOffset(rewriter, insertSlice, hostTargetType, mapper); + auto hostTargetOffset = createHostTargetOffset(rewriter, insertSlice, hostTargetType, mapper); + if (failed(hostTargetOffset)) return failure(); Value zeroOffset = getOrCreateIndexConstant(rewriter, coreBatchOp.getOperation(), 0); auto sizeAttr = getTensorSizeInBytesAttr(rewriter, coreBatchOp.getOperation(), mappedSource); if (failed(sizeAttr)) @@ -519,7 +522,7 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeBatchOp(spatial::SpatSchedul pim::PimMemCopyDevToHostOp::create(rewriter, insertSlice.getLoc(), hostTarget.getType(), - hostTargetOffset, + *hostTargetOffset, zeroOffset, hostTarget, mappedSource, diff --git a/src/PIM/Conversion/SpatialToPim/Common.cpp b/src/PIM/Conversion/SpatialToPim/Common.cpp index 6d5dfe5..0fb45ef 100644 --- a/src/PIM/Conversion/SpatialToPim/Common.cpp +++ b/src/PIM/Conversion/SpatialToPim/Common.cpp @@ -123,7 +123,7 @@ static bool isLoopCarriedOutput(mlir::Value operand, Operation* operation) { .getOperand(argument.getArgNumber() - 1) == operation->getResult(0); } -mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Operation* operation) { +mlir::Value getReusableOutputTensor(Operation* operation) { assert("Only support operations with a single result" && operation->getNumResults() == 1); mlir::Value result = operation->getResult(0); auto resultType = result.getType(); @@ -141,8 +141,13 @@ mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Op if (bestOperand != validOperands.end()) return *bestOperand; + return {}; +} - auto resultShapedType = cast(resultType); +mlir::Value getBestOutputTensorFromOperandsOrAllocate(RewriterBase& rewriter, Operation* operation) { + if (mlir::Value operand = getReusableOutputTensor(operation)) + return operand; + auto resultShapedType = cast(operation->getResult(0).getType()); rewriter.setInsertionPoint(operation); return tensor::EmptyOp::create( rewriter, operation->getLoc(), resultShapedType.getShape(), resultShapedType.getElementType()); diff --git a/src/PIM/Conversion/SpatialToPim/Common.hpp b/src/PIM/Conversion/SpatialToPim/Common.hpp index dfa73a4..c47a535 100644 --- a/src/PIM/Conversion/SpatialToPim/Common.hpp +++ b/src/PIM/Conversion/SpatialToPim/Common.hpp @@ -47,6 +47,10 @@ mlir::Operation* getEarliestUserWithinBlock(mlir::Value value); mlir::SmallVector getOpOperandsSortedByUses(mlir::Operation* operation); +// Destination selection only. Bufferization still proves physical aliasing. +// Valid for the current SSA graph only; recompute after use or order changes. +mlir::Value getReusableOutputTensor(mlir::Operation* operation); + mlir::Value getBestOutputTensorFromOperandsOrAllocate(mlir::RewriterBase& rewriter, mlir::Operation* operation); mlir::LogicalResult validateFragmentAssemblyMetadata(onnx_mlir::spatial::SpatBlueprintOp blueprint, diff --git a/src/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.cpp b/src/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.cpp index 9efc635..7a56b16 100644 --- a/src/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.cpp +++ b/src/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.cpp @@ -1,4 +1,6 @@ #include +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" #include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp" #include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" @@ -7,6 +9,41 @@ using namespace mlir; namespace onnx_mlir { +FailureOr getDirectReturnOperandIndex(OpResult result) { + if (!result.hasOneUse()) + return failure(); + + auto returnOp = dyn_cast(*result.getUsers().begin()); + if (!returnOp) + return failure(); + return result.getUses().begin()->getOperandNumber(); +} + + +FailureOr +planComputeInputMaterialization(Operation* owner, unsigned inputIndex) { + Value input; + bool batched = false; + if (auto compute = dyn_cast(owner)) { + if (inputIndex >= compute.getInputs().size() || !compute.getInputArgument(inputIndex)) + return failure(); + if (compute.getInputArgument(inputIndex)->use_empty()) + return ComputeInputMaterialization::Unused; + input = compute.getInputs()[inputIndex]; + } else if (auto batch = dyn_cast(owner)) { + if (inputIndex >= batch.getInputs().size() || !batch.getInputArgument(inputIndex)) + return failure(); + input = batch.getInputs()[inputIndex]; + batched = true; + } else return failure(); + if (auto argument = dyn_cast(input); argument + && isa(argument.getOwner()->getParentOp())) + return ComputeInputMaterialization::ForwardFunctionInput; + if (!batched && input.getDefiningOp()) + return ComputeInputMaterialization::ForwardConstant; + return ComputeInputMaterialization::CopyAtEntry; +} + std::optional getDirectComputeLikeInputIndex(Operation* owner, unsigned operandNumber) { auto getInputIndex = [operandNumber](Operation* op, unsigned inputCount) -> std::optional { if (inputCount == 0) diff --git a/src/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp b/src/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp index c0ac20f..0d269fe 100644 --- a/src/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp +++ b/src/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp @@ -7,6 +7,14 @@ namespace onnx_mlir { +// Input placement only; bufferization and folding determine physical requests. +enum class ComputeInputMaterialization { Unused, ForwardConstant, ForwardFunctionInput, CopyAtEntry }; + +mlir::FailureOr getDirectReturnOperandIndex(mlir::OpResult result); + +mlir::FailureOr +planComputeInputMaterialization(mlir::Operation* owner, unsigned inputIndex); + std::optional getDirectComputeLikeInputIndex(mlir::Operation* owner, unsigned operandNumber); void replaceAndEraseDirectComputeLikeInput(mlir::PatternRewriter& rewriter, diff --git a/src/PIM/Conversion/SpatialToPim/CoreLoweringPatterns.cpp b/src/PIM/Conversion/SpatialToPim/CoreLoweringPatterns.cpp index 199c62e..854cbb5 100644 --- a/src/PIM/Conversion/SpatialToPim/CoreLoweringPatterns.cpp +++ b/src/PIM/Conversion/SpatialToPim/CoreLoweringPatterns.cpp @@ -16,6 +16,7 @@ #include "src/Accelerators/PIM/Common/PimCommon.hpp" #include "src/Accelerators/PIM/Common/Support/CheckedArithmetic.hpp" #include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp" +#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp" #include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp" #include "src/Dialect/ONNX/ONNXOps.hpp" @@ -80,7 +81,9 @@ static FailureOr lowerFragmentAssemblyBlueprint(IRRewriter& rewriter, return blueprint.emitOpError("contiguous row-major fragment storage does not match its logical result"), failure(); return *reshaped; } - SmallVector hostStrides = computeRowMajorStrides(resultType.getShape()); + auto checkedStrides = computeRowMajorStridesChecked(resultType.getShape()); + if (failed(checkedStrides)) return failure(); + SmallVector hostStrides = std::move(*checkedStrides); SmallVector copies; for (int64_t fragmentIndex = 0; fragmentIndex < static_cast(operandIndices.size()); ++fragmentIndex) { int64_t operandIndex = operandIndices[fragmentIndex]; @@ -430,10 +433,16 @@ LogicalResult raptor::SpatialToPimPass::lowerComputeOp(spatial::SpatScheduledCom auto blockArg = computeOp.getInputArgument(inputIndex); if (!blockArg) return computeOp.emitOpError("expected compute input block arguments during input materialization"); - if (blockArg->use_empty()) + auto inputPlan = planComputeInputMaterialization(computeOp, inputIndex); + if (failed(inputPlan)) + return computeOp.emitOpError("failed to plan compute input materialization"); + if (*inputPlan == ComputeInputMaterialization::Unused) continue; + if (*inputPlan == ComputeInputMaterialization::ForwardFunctionInput) + return computeOp.emitOpError("function inputs must be forwarded before core input materialization"); - if (auto constantOp = input.getDefiningOp()) { + if (*inputPlan == ComputeInputMaterialization::ForwardConstant) { + auto constantOp = input.getDefiningOp(); blockArg->replaceAllUsesWith(getOrCreateConstantLike(constantFolder, constantOp)); continue; } diff --git a/src/PIM/Conversion/SpatialToPim/Patterns/ChannelLowering.cpp b/src/PIM/Conversion/SpatialToPim/Patterns/ChannelLowering.cpp index 340eb48..3a8dd73 100644 --- a/src/PIM/Conversion/SpatialToPim/Patterns/ChannelLowering.cpp +++ b/src/PIM/Conversion/SpatialToPim/Patterns/ChannelLowering.cpp @@ -19,10 +19,12 @@ static void copyRaptorDebugAttrs(Operation* source, Operation* target) { } } -static Value createDestinationByteOffset(PatternRewriter& rewriter, +static FailureOr createDestinationByteOffset(PatternRewriter& rewriter, tensor::InsertSliceOp insert) { auto destinationType = cast(insert.getDestType()); - SmallVector strides = computeRowMajorStrides(destinationType.getShape()); + auto checkedStrides = computeRowMajorStridesChecked(destinationType.getShape()); + if (failed(checkedStrides)) return failure(); + SmallVector strides = std::move(*checkedStrides); int64_t elementBytes = getElementTypeSizeInBytes(destinationType.getElementType()); Value total = arith::ConstantIndexOp::create(rewriter, insert.getLoc(), 0); for (auto [dimension, offset] : llvm::enumerate(insert.getMixedOffsets())) { @@ -114,9 +116,10 @@ static LogicalResult lowerReceive( } rewriter.setInsertionPoint(destinationInsert); - Value targetOffset = createDestinationByteOffset(rewriter, destinationInsert); + auto targetOffset = createDestinationByteOffset(rewriter, destinationInsert); + if (failed(targetOffset)) return failure(); auto copy = pim::PimMemCopyOp::create( - rewriter, op.getLoc(), destinationInsert.getDestType(), targetOffset, zero, + rewriter, op.getLoc(), destinationInsert.getDestType(), *targetOffset, zero, destinationInsert.getDest(), *received, *sizeAttr); rewriter.replaceOp(destinationInsert, copy.getOutput()); rewriter.eraseOp(op); diff --git a/src/PIM/Conversion/SpatialToPim/Patterns/GlobalTensorMaterialization.cpp b/src/PIM/Conversion/SpatialToPim/Patterns/GlobalTensorMaterialization.cpp index 1bd2179..5adcf17 100644 --- a/src/PIM/Conversion/SpatialToPim/Patterns/GlobalTensorMaterialization.cpp +++ b/src/PIM/Conversion/SpatialToPim/Patterns/GlobalTensorMaterialization.cpp @@ -179,29 +179,18 @@ struct FuncOpArgToGlobalMemoryPattern final : OpRewritePattern(argUser)) { - auto inputIndex = getDirectComputeLikeInputIndex(spatCompute, argUses.getOperandNumber()); - if (!inputIndex) + if (isa(argUser)) { + auto inputIndex = getDirectComputeLikeInputIndex(argUser, argUses.getOperandNumber()); + if (!inputIndex) return failure(); + auto plan = planComputeInputMaterialization(argUser, *inputIndex); + if (failed(plan) || (*plan != ComputeInputMaterialization::ForwardFunctionInput + && *plan != ComputeInputMaterialization::Unused)) return failure(); - auto BBArgIndex = *inputIndex; - rewriter.setInsertionPoint(&spatCompute.getBody().front().front()); + rewriter.setInsertionPoint(&argUser->getRegion(0).front().front()); auto getGlobalOp = memref::GetGlobalOp::create(rewriter, loc, memRefType, argName); auto toTensor = bufferization::ToTensorOp::create( rewriter, loc, argRankedTensorType, getGlobalOp, rewriter.getUnitAttr(), rewriter.getUnitAttr()); - - replaceAndEraseDirectComputeLikeInput(rewriter, spatCompute.getOperation(), BBArgIndex, toTensor); - } - else if (auto spatComputeBatch = dyn_cast(argUser)) { - auto inputIndex = getDirectComputeLikeInputIndex(spatComputeBatch, argUses.getOperandNumber()); - if (!inputIndex) - return failure(); - auto BBArgIndex = *inputIndex; - rewriter.setInsertionPoint(&spatComputeBatch.getBody().front().front()); - auto getGlobalOp = memref::GetGlobalOp::create(rewriter, loc, memRefType, argName); - auto toTensor = bufferization::ToTensorOp::create( - rewriter, loc, argRankedTensorType, getGlobalOp, rewriter.getUnitAttr(), rewriter.getUnitAttr()); - - replaceAndEraseDirectComputeLikeInput(rewriter, spatComputeBatch.getOperation(), BBArgIndex, toTensor); + replaceAndEraseDirectComputeLikeInput(rewriter, argUser, *inputIndex, toTensor); } else { rewriter.setInsertionPoint(argUser); diff --git a/src/PIM/Dialect/Pim/Passes/Transforms/Bufferization/ContiguityPatterns.cpp b/src/PIM/Dialect/Pim/Passes/Transforms/Bufferization/ContiguityPatterns.cpp index 48172a5..9b6e938 100644 --- a/src/PIM/Dialect/Pim/Passes/Transforms/Bufferization/ContiguityPatterns.cpp +++ b/src/PIM/Dialect/Pim/Passes/Transforms/Bufferization/ContiguityPatterns.cpp @@ -5,6 +5,7 @@ #include "llvm/Support/MathExtras.h" #include "ContiguityPatterns.hpp" +#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp" #include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp" #include "src/Accelerators/PIM/Common/IR/LoopUtils.hpp" #include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp" @@ -84,17 +85,6 @@ static FailureOr checkedPositiveMul(int64_t lhs, int64_t rhs) { return result; } -static FailureOr checkedPositiveProduct(ArrayRef values) { - int64_t result = 1; - for (int64_t value : values) { - auto product = checkedPositiveMul(result, value); - if (failed(product)) - return failure(); - result = *product; - } - return result; -} - static FailureOr> getStaticMemRefStrides(MemRefType type) { SmallVector strides; int64_t offset = 0; @@ -348,12 +338,9 @@ inferLogicalCopyShape(MemRefType targetType, MemRefType sourceType, int64_t size } static FailureOr getContiguousSuffixRank( - Value value, ArrayRef copyShape, Type elementType = {}) { + Value value, ArrayRef copyShape) { auto type = dyn_cast(value.getType()); - if (type && elementType && isPackedByteBuffer(type)) - return copyShape.size(); if (!type || !type.hasStaticShape() || !hasByteSizedElementType(type.getElementType()) - || (elementType && type.getElementType() != elementType) || type.getRank() != static_cast(copyShape.size())) return failure(); if (llvm::any_of(copyShape, [](int64_t dim) { return dim <= 0; })) @@ -363,42 +350,19 @@ static FailureOr getContiguousSuffixRank( if (failed(strides)) return failure(); - int64_t expectedStride = 1; - int64_t contiguousSuffixRank = 0; - for (int64_t dim = type.getRank() - 1; dim >= 0; --dim) { - if (copyShape[dim] != 1 && (*strides)[dim] != expectedStride) - break; - ++contiguousSuffixRank; - auto nextStride = checkedPositiveMul(expectedStride, copyShape[dim]); - if (failed(nextStride)) - return failure(); - expectedStride = *nextStride; - } - return contiguousSuffixRank; + return onnx_mlir::getContiguousSuffixRank(copyShape, *strides); } -static FailureOr> getOuterByteStrides( - Value value, const LogicalCopyShape ©Shape, size_t outerRank) { +static FailureOr> getCopyElementStrides( + Value value, const LogicalCopyShape ©Shape) { auto type = cast(value.getType()); - SmallVector strides; if (isPackedByteBuffer(type)) - strides = computeRowMajorStrides(copyShape.dimensions); - else { - auto proven = getProvenMemRefStrides(value); - if (failed(proven)) - return failure(); - strides = std::move(*proven); - } - int64_t elementByteWidth = static_cast( - getElementTypeSizeInBytes(copyShape.elementType)); - SmallVector result; - for (int64_t stride : ArrayRef(strides).take_front(outerRank)) { - auto byteStride = checkedPositiveMul(stride, elementByteWidth); - if (failed(byteStride)) - return failure(); - result.push_back(*byteStride); - } - return result; + return computeRowMajorStridesChecked(copyShape.dimensions); + if (!hasByteSizedElementType(type.getElementType()) + || type.getElementType() != copyShape.elementType + || type.getRank() != static_cast(copyShape.dimensions.size())) + return failure(); + return getProvenMemRefStrides(value); } static FailureOr analyzeCopyEndpoint(Value value, Value initialByteOffset, MemRefType logicalType) { @@ -498,20 +462,19 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO if (failed(logicalCopyShape)) return failure(); - auto targetSuffixRank = getContiguousSuffixRank( - target, logicalCopyShape->dimensions, logicalCopyShape->elementType); - auto sourceSuffixRank = getContiguousSuffixRank( - source, logicalCopyShape->dimensions, logicalCopyShape->elementType); - if (failed(targetSuffixRank) || failed(sourceSuffixRank)) + auto targetStrides = getCopyElementStrides(target, *logicalCopyShape); + auto sourceStrides = getCopyElementStrides(source, *logicalCopyShape); + if (failed(targetStrides) || failed(sourceStrides)) + return failure(); + auto access = CopyAccessPlan::build( + logicalCopyShape->dimensions, *sourceStrides, *targetStrides); + if (failed(access)) return failure(); CopyRewritePlan plan; plan.target = *targetPlan; plan.source = *sourcePlan; - - int64_t contiguousSuffixRank = std::min(*targetSuffixRank, *sourceSuffixRank); - if (contiguousSuffixRank - == static_cast(logicalCopyShape->dimensions.size())) { + if (access->getOuterShape().empty()) { plan.kind = CopyRewritePlan::Kind::Direct; plan.directBytes = size; return plan; @@ -522,30 +485,22 @@ analyzeCopyRewrite(Value target, Value source, Value targetOffset, Value sourceO plan.kind = CopyRewritePlan::Kind::Loop; plan.loop.targetBaseOffset = plan.target.offset; plan.loop.sourceBaseOffset = plan.source.offset; - plan.loop.outerShape.assign( - logicalCopyShape->dimensions.begin(), - logicalCopyShape->dimensions.end() - contiguousSuffixRank); - SmallVector chunkShape( - logicalCopyShape->dimensions.end() - contiguousSuffixRank, - logicalCopyShape->dimensions.end()); - auto outerElements = checkedPositiveProduct(plan.loop.outerShape); - auto chunkElements = checkedPositiveProduct(chunkShape); - auto chunkBytes = failed(chunkElements) - ? FailureOr(failure()) - : checkedPositiveMul(*chunkElements, elementByteWidth); - if (failed(outerElements) || failed(chunkBytes)) + plan.loop.outerShape.assign(access->getOuterShape().begin(), access->getOuterShape().end()); + auto chunkBytes = checkedPositiveMul(access->getRunElements(), elementByteWidth); + if (failed(chunkBytes)) return failure(); - plan.loop.outerElements = *outerElements; + plan.loop.outerElements = access->getRunCount(); plan.loop.chunkBytes = *chunkBytes; - auto targetStrides = getOuterByteStrides( - target, *logicalCopyShape, plan.loop.outerShape.size()); - auto sourceStrides = getOuterByteStrides( - source, *logicalCopyShape, plan.loop.outerShape.size()); - if (failed(targetStrides) || failed(sourceStrides)) - return failure(); - plan.loop.targetOuterByteStrides = std::move(*targetStrides); - plan.loop.sourceOuterByteStrides = std::move(*sourceStrides); - if (plan.loop.chunkBytes <= 0) + auto scaleStrides = [&](ArrayRef strides, SmallVectorImpl &result) { + for (int64_t stride : strides) { + auto byteStride = checkedPositiveMul(stride, elementByteWidth); + if (failed(byteStride)) return failure(); + result.push_back(*byteStride); + } + return success(); + }; + if (failed(scaleStrides(access->getTargetOuterStrides(), plan.loop.targetOuterByteStrides)) + || failed(scaleStrides(access->getSourceOuterStrides(), plan.loop.sourceOuterByteStrides))) return failure(); return plan; } diff --git a/src/PIM/Dialect/Pim/Passes/Transforms/HostConstantFolding/Common.cpp b/src/PIM/Dialect/Pim/Passes/Transforms/HostConstantFolding/Common.cpp index 8593dd5..5f790d0 100644 --- a/src/PIM/Dialect/Pim/Passes/Transforms/HostConstantFolding/Common.cpp +++ b/src/PIM/Dialect/Pim/Passes/Transforms/HostConstantFolding/Common.cpp @@ -94,8 +94,12 @@ foldDenseSubview(DenseElementsAttr denseAttr, ArrayRef staticOffsets, A if (numResultElements < 0) return failure(); - auto sourceStrides = computeRowMajorStrides(sourceType.getShape()); - auto resultStrides = computeRowMajorStrides(resultTensorType.getShape()); + auto checkedSourceStrides = computeRowMajorStridesChecked(sourceType.getShape()); + if (failed(checkedSourceStrides)) return failure(); + auto sourceStrides = std::move(*checkedSourceStrides); + auto checkedResultStrides = computeRowMajorStridesChecked(resultTensorType.getShape()); + if (failed(checkedResultStrides)) return failure(); + auto resultStrides = std::move(*checkedResultStrides); SmallVector sourceValues(denseAttr.getValues()); SmallVector resultValues(numResultElements); for (int64_t i = 0; i < numResultElements; ++i) { diff --git a/src/PIM/Dialect/Pim/Passes/Transforms/HostConstantFolding/Patterns/Constant.cpp b/src/PIM/Dialect/Pim/Passes/Transforms/HostConstantFolding/Patterns/Constant.cpp index a33f93e..6549be2 100644 --- a/src/PIM/Dialect/Pim/Passes/Transforms/HostConstantFolding/Patterns/Constant.cpp +++ b/src/PIM/Dialect/Pim/Passes/Transforms/HostConstantFolding/Patterns/Constant.cpp @@ -218,7 +218,9 @@ static FailureOr foldConstantAlloc(memref::AllocOp allocOp, M return failure(); SmallVector resultValues(numElements, fillValue); - auto resultStrides = computeRowMajorStrides(resultTensorType.getShape()); + auto checkedResultStrides = computeRowMajorStridesChecked(resultTensorType.getShape()); + if (failed(checkedResultStrides)) return failure(); + auto resultStrides = std::move(*checkedResultStrides); llvm::sort(copies, [](const ConstantSubviewCopy& lhs, const ConstantSubviewCopy& rhs) { return lhs.copyOp->isBeforeInBlock(rhs.copyOp); @@ -232,7 +234,11 @@ static FailureOr foldConstantAlloc(memref::AllocOp allocOp, M || sourceType.getRank() != static_cast(copy.strides.size())) return failure(); - auto sourceStrides = computeRowMajorStrides(sourceType.getShape()); + auto checkedSourceStrides = computeRowMajorStridesChecked(sourceType.getShape()); + + if (failed(checkedSourceStrides)) return failure(); + + auto sourceStrides = std::move(*checkedSourceStrides); SmallVector sourceValues(copy.source.getValues()); for (auto [linearIndex, value] : llvm::enumerate(sourceValues)) { SmallVector sourceIndices = diff --git a/src/PIM/Dialect/Pim/Passes/Transforms/LocalMemoryPlanning/LocalMemoryPlanning.cpp b/src/PIM/Dialect/Pim/Passes/Transforms/LocalMemoryPlanning/LocalMemoryPlanning.cpp index 1c238b9..6f94d83 100644 --- a/src/PIM/Dialect/Pim/Passes/Transforms/LocalMemoryPlanning/LocalMemoryPlanning.cpp +++ b/src/PIM/Dialect/Pim/Passes/Transforms/LocalMemoryPlanning/LocalMemoryPlanning.cpp @@ -7,6 +7,7 @@ #include #include "src/Accelerators/PIM/Common/PimCommon.hpp" +#include "src/Accelerators/PIM/Common/Support/Diagnostics.hpp" #include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp" #include "src/Accelerators/PIM/Dialect/Pim/Passes/Transforms/LocalMemoryPlanning/LocalMemoryPlanning.hpp" #include "src/Accelerators/PIM/Passes/PIMPasses.h" @@ -149,7 +150,7 @@ FailureOr buildCoreMemoryPlan(Operation* coreLikeOp) { plan.intervals = std::move(*intervals); auto placements = planLocalMemoryPlacements(plan.intervals, kPimLocalMemoryAddressLimit); if (failed(placements)) { - coreLikeOp->emitError("Pim local-memory plan exceeds the signed int32 address range"); + pim::emitCandidateResourceError(coreLikeOp, "Pim local-memory plan exceeds the signed int32 address range"); return failure(); } plan.placements = std::move(*placements); diff --git a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/ScheduledSpatialPasses.hpp b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/ScheduledSpatialPasses.hpp index 6626e2a..11fae77 100644 --- a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/ScheduledSpatialPasses.hpp +++ b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/ScheduledSpatialPasses.hpp @@ -11,6 +11,7 @@ struct PreparedSpatialSchedules { std::vector candidates; }; +// Success always includes at least one candidate, including an empty compute graph. mlir::FailureOr prepareSpatialSchedules( mlir::ModuleOp module, const SchedulingTarget &target, size_t pipelineStages, bool collectCandidates); diff --git a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ComputeGraph.cpp b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ComputeGraph.cpp index 5ba7f26..3d4eddb 100644 --- a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ComputeGraph.cpp +++ b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ComputeGraph.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/SetVector.h" #include "llvm/Support/Casting.h" +#include "llvm/Support/CheckedArithmetic.h" #include #include @@ -30,6 +31,7 @@ #include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/DeferredProjectionAnalysis.hpp" #include "src/Accelerators/PIM/Common/IR/AffineUtils.hpp" #include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp" +#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp" #include "src/Accelerators/PIM/Common/IR/ShapingUtils.hpp" #include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp" #include "src/Support/TypeUtilities.hpp" @@ -55,6 +57,7 @@ struct SchedulerCostModel { const SchedulingTarget& target; ComputeCostKind kind = ComputeCostKind::Execution; + mutable bool projectionFallbackReported = false; static Cost ceilDiv(Cost numerator, Cost denominator) { assert(denominator > 0 && "denominator must be positive"); @@ -534,18 +537,10 @@ static bool isSharedMemoryInput(Value input, const SchedulingTarget &target) { Attribute value; if (!matchPattern(input, m_Constant(&value)) || !isa(value)) return false; - auto dense = dyn_cast(value); - if (!target.zeroSplatUsesSharedMemory && dense && dense.isSplat()) { - auto element = dense.getSplatValue(); - if (auto floating = dyn_cast(element)) - return !floating.getValue().isZero() || floating.getValue().isNegative(); - if (auto integer = dyn_cast(element)) - return !integer.getValue().isZero(); - } - return true; + return target.zeroSplatUsesSharedMemory || !isZeroBitPatternSplat(value); } -static std::pair getProjectionMemoryRequests(Type type, Value projection, +static FailureOr> getProjectionMemoryRequests(Type type, Value projection, const SchedulerCostModel &costModel) { Cost bytes = costModel.getByteSize(type, costModel.getComputeBitwidth(type)); Cost requests = 1; @@ -553,40 +548,53 @@ static std::pair getProjectionMemoryRequests(Type type, Value projec extract && extract.getSourceType().hasStaticShape() && !ShapedType::isDynamicShape(extract.getStaticSizes()) && !ShapedType::isDynamicShape(extract.getStaticStrides())) { - auto strides = computeRowMajorStrides(extract.getSourceType().getShape()); - Cost run = 1; - for (size_t dim = strides.size(); dim-- > 0;) { - Cost size = extract.getStaticSizes()[dim]; - if (size == 1) continue; - if (checkedMultiply(static_cast(strides[dim]), - static_cast(extract.getStaticStrides()[dim])) != run) - break; - run = checkedMultiply(run, size); + auto unavailable = [&]() -> FailureOr> { + if (!costModel.projectionFallbackReported) { + extract.emitWarning("projection memory cost unavailable: using one request per element because static copy geometry exceeds supported offset or stride limits"); + costModel.projectionFallbackReported = true; + } + // A physical copy cannot need more than one request per element. Round + // each element separately so sub-byte elements cannot undercharge service. + return std::make_pair(SchedulerCostModel::ceilDiv(costModel.getComputeBitwidth(type), Cost{8}), + static_cast(cast(type).getNumElements())); + }; + auto strides = computeRowMajorStridesChecked(extract.getSourceType().getShape()); + auto targetStrides = computeRowMajorStridesChecked(extract.getStaticSizes()); + if (failed(strides) || failed(targetStrides)) return unavailable(); + for (size_t dim = 0; dim < strides->size(); ++dim) { + auto stride = llvm::checkedMul((*strides)[dim], extract.getStaticStrides()[dim]); + if (!stride) return unavailable(); + (*strides)[dim] = *stride; } Cost elements = cast(type).getNumElements(); if (elements) { + auto access = CopyAccessPlan::build(extract.getStaticSizes(), *strides, *targetStrides); + if (failed(access)) + return unavailable(); + Cost run = access->getRunElements(); requests = checkedMultiply(requests, elements / run); bytes = SchedulerCostModel::ceilDiv( checkedMultiply(run, costModel.getComputeBitwidth(type)), Cost{8}); } } - return {bytes, requests}; + return std::make_pair(bytes, requests); } -static SharedMemoryDemand getInputMemoryDemand(const InputFragment &fragment, +static FailureOr getInputMemoryDemand(const InputFragment &fragment, const SchedulerCostModel &costModel) { - auto [bytes, runs] = getProjectionMemoryRequests(fragment.type, fragment.projection, costModel); - return costModel.target.getSharedMemoryDemand(bytes, - checkedMultiply(runs, fragment.count), false); + auto access = getProjectionMemoryRequests(fragment.type, fragment.projection, costModel); + if (failed(access)) return failure(); + return costModel.target.getSharedMemoryDemand(access->first, + checkedMultiply(access->second, fragment.count), false); } -static SharedMemoryDemand getCapturedConstantMemoryDemand( +static FailureOr getCapturedConstantMemoryDemand( Operation *owner, const SchedulerCostModel &costModel) { // shortcut: later aliasing/load folding can change baseline requests; exact // accounting requires a placed, bufferized program. SharedMemoryDemand demand; - owner->walk([&](Operation *user) { - if (user == owner) return; + auto result = owner->walk([&](Operation *user) -> WalkResult { + if (user == owner) return WalkResult::advance(); for (Value value : user->getOperands()) { if (isa(value) || !isSharedMemoryInput(value, costModel.target)) continue; if (auto extract = dyn_cast(user)) { @@ -618,9 +626,13 @@ static SharedMemoryDemand getCapturedConstantMemoryDemand( } auto extract = dyn_cast(user); Type type = extract ? extract.getResult().getType() : value.getType(); - demand += getInputMemoryDemand({type, count, extract ? extract.getResult() : value}, costModel); + auto memory = getInputMemoryDemand({type, count, extract ? extract.getResult() : value}, costModel); + if (failed(memory)) return WalkResult::interrupt(); + demand += *memory; } + return WalkResult::advance(); }); + if (result.wasInterrupted()) return failure(); return demand; } @@ -840,10 +852,11 @@ static LogicalResult collectPublicationTransfer(Value input, size_t targetIndex, if (auto fragments = getProjectedInputFragments(input, consumer)) { for (const InputFragment &fragment : *fragments) { Cost bytes = costModel.getByteSize(fragment.type, costModel.getComputeBitwidth(fragment.type)); - auto [storeBytes, storeRuns] = getProjectionMemoryRequests( + auto access = getProjectionMemoryRequests( fragment.type, fragment.projection, costModel); + if (failed(access)) return failure(); graph.publicationTransfers.push_back({sourceIndex, targetIndex, value, - 0, fragment.projection, bytes, fragment.count, {}, storeBytes, storeRuns}); + 0, fragment.projection, bytes, fragment.count, {}, access->first, access->second}); } } else { Cost bytes = costModel.getByteSize(input.getType(), costModel.getComputeBitwidth(input.getType())); @@ -1292,8 +1305,11 @@ FailureOr buildComputeGraph(Operation* entryOp, DenseMap capturedMemory; for (const auto& [targetIndex, node] : llvm::enumerate(graph.nodes)) { auto [captured, inserted] = capturedMemory.try_emplace(node.instance.op); - if (inserted) - captured->second = getCapturedConstantMemoryDemand(node.instance.op, costModel); + if (inserted) { + auto memory = getCapturedConstantMemoryDemand(node.instance.op, costModel); + if (failed(memory)) return failure(); + captured->second = *memory; + } auto memory = captured->second; memory.readBytes = checkedMultiply(memory.readBytes, static_cast(node.instance.laneCount)); memory.readRequests = checkedMultiply(memory.readRequests, static_cast(node.instance.laneCount)); @@ -1304,8 +1320,11 @@ FailureOr buildComputeGraph(Operation* entryOp, size_t publicationsBefore = graph.publicationTransfers.size(); if (isSharedMemoryInput(input, target)) { if (auto fragments = getProjectedInputFragments(input, node.instance)) { - for (const InputFragment &fragment : *fragments) - graph.baselineMemory += getInputMemoryDemand(fragment, costModel); + for (const InputFragment &fragment : *fragments) { + auto memory = getInputMemoryDemand(fragment, costModel); + if (failed(memory)) return failure(); + graph.baselineMemory += *memory; + } } else { graph.baselineMemory += target.getSharedMemoryDemand( costModel.getByteSize(input.getType(), costModel.getComputeBitwidth(input.getType())), diff --git a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.cpp b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.cpp index 572b03f..1c187c3 100644 --- a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.cpp +++ b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.cpp @@ -6,11 +6,33 @@ namespace onnx_mlir::spatial { -bool WholeScheduleCost::eligible() const { +bool ScheduleRecurrenceEvidence::proved() const { + return status == Proved && periodGenerations && intervalNs + && std::isfinite(*intervalNs) && *intervalNs > 0; +} + +std::optional WholeScheduleCost::convergenceProved() const { + if (recurrence[0].status == ScheduleRecurrenceEvidence::NotChecked + && recurrence[1].status == ScheduleRecurrenceEvidence::NotChecked) + return std::nullopt; + return recurrence[0].proved() && recurrence[1].proved(); +} + +std::optional WholeScheduleCost::loweringSupported() const { + if (loweringSupport == StaticLoweringSupport::Unknown) return std::nullopt; + return loweringSupport == StaticLoweringSupport::Supported; +} + +bool WholeScheduleCost::available() const { return primary && alternative && std::isfinite(*primary) && std::isfinite(*alternative) - && *primary > 0 && *alternative > 0 && relativeUncertainty - && std::isfinite(*relativeUncertainty) && *relativeUncertainty >= 0 - && *relativeUncertainty < 1; + && *primary > 0 && *alternative > 0; +} + +bool WholeScheduleCost::eligible() const { + return available() && convergenceProved() == true + && loweringSupport == StaticLoweringSupport::Supported && selectionMargin + && std::isfinite(*selectionMargin) && *selectionMargin >= 0 + && *selectionMargin < 1; } size_t selectPipelineCandidate(llvm::ArrayRef scores, std::string &reason) { @@ -21,17 +43,17 @@ size_t selectPipelineCandidate(llvm::ArrayRef scores, std::st size_t best = 0; auto upper = [&](size_t i) { return std::max(*scores[i].primary, *scores[i].alternative) - / (1 - *scores[i].relativeUncertainty); + / (1 - *scores[i].selectionMargin); }; for (size_t i = 1; i < scores.size(); ++i) if (scores[i].eligible() && upper(i) < upper(best)) best = i; if (!best) { reason = "original has the lowest conservative estimate"; return 0; } double originalLower = std::min(*scores.front().primary, *scores.front().alternative); - if (upper(best) >= originalLower / (1 + *scores.front().relativeUncertainty)) { - reason = "candidate and original overlap static prediction uncertainty"; + if (upper(best) >= originalLower / (1 + *scores.front().selectionMargin)) { + reason = "candidate and original overlap the selection policy margin"; return 0; } - reason = "candidate improves on original under both arbitration tie policies and prediction uncertainty"; + reason = "candidate improves on original under both arbitration tie policies and the selection margin"; return best; } diff --git a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.hpp b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.hpp index 26b4251..f66c5f5 100644 --- a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.hpp +++ b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.hpp @@ -8,6 +8,7 @@ namespace onnx_mlir::spatial { struct MergeScheduleResult; +inline constexpr double kPipelineSelectionMargin = 0.021; size_t selectPipelineCandidate(llvm::ArrayRef scores, std::string &reason); diff --git a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp index 0a79184..d5e14e7 100644 --- a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp +++ b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -39,20 +40,36 @@ struct ScheduledComputeMaterializationResult; struct DeferredTransferPlan; struct DeferredBoundaryPlan; +// Unknown lacks a proof. Unsupported identifies a known static backend violation. +// Supported covers checked static forms, not final allocation or code generation. +enum class StaticLoweringSupport { Unknown, Supported, Unsupported }; + +struct ScheduleRecurrenceEvidence { + enum Status { NotChecked, Unproved, Proved }; + Status status = NotChecked; + size_t periodGenerations = 0; + std::optional intervalNs; + std::string reason; + bool proved() const; +}; + struct WholeScheduleCost { // Steady-state intervals in nanoseconds. The alternative tests sensitivity. std::optional primary, alternative; std::string reason; - std::string identity; - std::optional relativeUncertainty; + // Selection policy only. This is not a prediction error bound. + std::optional selectionMargin; + std::array recurrence; + StaticLoweringSupport loweringSupport = StaticLoweringSupport::Unknown; + struct AnalysisTiming { + double extractionSeconds = 0, expansionSeconds = 0, evaluationSeconds = 0; + } timing; + std::optional convergenceProved() const; + std::optional loweringSupported() const; + bool available() const; bool eligible() const; }; -struct ScheduleCostQuery { - std::string identity; - std::function evaluate; -}; - struct SchedulingTarget { size_t processorCount = 0; size_t residentWeightCapacity = 0; @@ -76,38 +93,38 @@ struct SchedulingTarget { Cost sharedMemoryWriteLatencyNs = 0; bool zeroSplatUsesSharedMemory = true; - // Input is a verified, target-lowered candidate. Preparation and evaluation - // are read-only. An absent query keeps the original schedule. - std::function(mlir::ModuleOp, - llvm::json::Object &)> prepareScheduleCost; + // Optional read-only estimate over realized Spatial. Finite costs do not + // establish convergence or lowering support. Callers must validate these + // properties and apply the Spatial selection policy. + std::function estimateRealizedScheduleCost; std::string wholeScheduleCostUnavailable; std::function reportPeriodicSchedule; - Cost getLocalMemoryServiceNs(Cost bytes, bool write) const { - Cost accesses = bytes / localMemoryWidthBytes - + (bytes % localMemoryWidthBytes != 0); - return checkedMultiply(accesses, checkedMultiply(localMemoryPeriodNs, - write ? localMemoryWriteLatencyCycles : localMemoryReadLatencyCycles)); + template + Cost getLocalMemoryServiceNs(Cost bytes, bool write, Arithmetic math = {}) const { + return math.multiply(math.divideUp(bytes, localMemoryWidthBytes), + math.multiply(localMemoryPeriodNs, + write ? localMemoryWriteLatencyCycles : localMemoryReadLatencyCycles)); } + template SharedMemoryDemand getSharedMemoryDemand( - Cost bytesPerRequest, Cost requests, bool write) const { + Cost bytesPerRequest, Cost requests, bool write, Arithmetic math = {}) const { SharedMemoryDemand demand; if (write) { - demand.writeBytes = checkedMultiply(bytesPerRequest, requests); + demand.writeBytes = math.multiply(bytesPerRequest, requests); demand.writeRequests = requests; } else { - demand.readBytes = checkedMultiply(bytesPerRequest, requests); + demand.readBytes = math.multiply(bytesPerRequest, requests); demand.readRequests = requests; } - if (sharedMemoryWidthBytes) { - Cost accesses = bytesPerRequest / sharedMemoryWidthBytes - + (bytesPerRequest % sharedMemoryWidthBytes != 0); - demand.serviceNs = checkedMultiply(requests, checkedMultiply(accesses, + if (sharedMemoryWidthBytes) + demand.serviceNs = math.multiply(requests, math.multiply( + math.divideUp(bytesPerRequest, sharedMemoryWidthBytes), write ? sharedMemoryWriteLatencyNs : sharedMemoryReadLatencyNs)); - } return demand; } @@ -127,33 +144,25 @@ struct SchedulingTarget { Cost matrixInputBufferLatencyCycles = 0; bool matrixPipeline = true; - Cost getMatrixComputeLatencyNs(Cost inputBitwidth) const { - auto divideRoundedUp = [](Cost value, Cost width) { - return value / width + (value % width != 0); - }; - Cost inputTimes = divideRoundedUp(inputBitwidth, matrixInputResolutionBits); - Cost inputPasses = divideRoundedUp(matrixRows, matrixInputParallelism); - Cost outputPasses = divideRoundedUp(matrixColumns, matrixOutputParallelism); - Cost readCycles = divideRoundedUp(matrixReadLatencyNs, matrixPeriodNs); - Cost frontStage = matrixInputBufferLatencyCycles + matrixInputLatencyCycles - + readCycles + matrixSampleLatencyCycles; + template + Cost getMatrixComputeLatencyNs(Cost inputBitwidth, Arithmetic math = {}) const { + Cost inputTimes = math.divideUp(inputBitwidth, matrixInputResolutionBits); + Cost inputPasses = math.divideUp(matrixRows, matrixInputParallelism); + Cost outputPasses = math.divideUp(matrixColumns, matrixOutputParallelism); + Cost readCycles = math.divideUp(matrixReadLatencyNs, matrixPeriodNs); + Cost frontStage = math.add(math.add(matrixInputBufferLatencyCycles, + matrixInputLatencyCycles), math.add(readCycles, matrixSampleLatencyCycles)); Cost backPipe = std::max(matrixOutputLatencyCycles, - checkedAdd(matrixShiftLatencyCycles, - matrixBufferLatencyCycles)); - Cost backStage = checkedAdd( - checkedAdd(matrixOutputLatencyCycles, matrixShiftLatencyCycles), - matrixBufferLatencyCycles); - backStage = checkedAdd(backStage, checkedMultiply(outputPasses - 1, backPipe)); - Cost totalTimes = checkedMultiply(inputTimes, inputPasses); + math.add(matrixShiftLatencyCycles, matrixBufferLatencyCycles)); + Cost backStage = math.add(math.add(matrixOutputLatencyCycles, + matrixShiftLatencyCycles), matrixBufferLatencyCycles); + backStage = math.add(backStage, math.multiply(math.subtract(outputPasses, 1), backPipe)); + Cost totalTimes = math.multiply(inputTimes, inputPasses); if (!matrixPipeline) - return checkedMultiply( - checkedMultiply(checkedAdd(frontStage, backStage), totalTimes), - matrixPeriodNs); + return math.multiply(math.multiply(math.add(frontStage, backStage), totalTimes), matrixPeriodNs); Cost stagePipe = std::max(frontStage, backStage); - return checkedMultiply( - checkedAdd(checkedAdd(frontStage, backStage), - checkedMultiply(totalTimes - 1, stagePipe)), - matrixPeriodNs); + return math.multiply(math.add(math.add(frontStage, backStage), + math.multiply(math.subtract(totalTimes, 1), stagePipe)), matrixPeriodNs); } Cost getInterProcessorLatencyNs(size_t source, diff --git a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/Utils.hpp b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/Utils.hpp index 2a615ef..8c1fe29 100644 --- a/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/Utils.hpp +++ b/src/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/Utils.hpp @@ -36,6 +36,13 @@ inline T checkedMultiply(T lhs, T rhs) { return lhs * rhs; } +struct SchedulingCostArithmetic { + Cost add(Cost a, Cost b) const { return checkedAdd(a, b); } + Cost multiply(Cost a, Cost b) const { return checkedMultiply(a, b); } + Cost subtract(Cost a, Cost b) const { assert(a >= b); return a - b; } + Cost divideUp(Cost a, Cost b) const { assert(b); return a / b + (a % b != 0); } +}; + template inline T addOrMax(T lhs, T rhs) { static_assert(std::is_unsigned_v, "addOrMax only supports unsigned types"); diff --git a/test/PIM/CMakeLists.txt b/test/PIM/CMakeLists.txt index 18d6e29..8b3b64c 100644 --- a/test/PIM/CMakeLists.txt +++ b/test/PIM/CMakeLists.txt @@ -30,6 +30,31 @@ add_pim_unittest(PimMemoryLivenessPlannerTest OMPimCompilerUtils ) +add_pim_unittest(CopyAccessPlanTest + CopyAccessPlanTest.cpp + LINK_LIBS PRIVATE OMPimCompilerUtils +) +add_test(NAME CopyAccessPlanChecksActive COMMAND CopyAccessPlanTest --check-failure) +set_tests_properties(CopyAccessPlanChecksActive PROPERTIES WILL_FAIL TRUE LABELS pim-unittest) +add_test(NAME RowMajorStrideOverflow + COMMAND "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_ShapeUtils.py" + --test "$") +set_tests_properties(RowMajorStrideOverflow PROPERTIES LABELS pim-unittest) + +add_pim_unittest(ComputeInputMaterializationTest + ComputeInputMaterializationTest.cpp + LINK_LIBS PRIVATE OMPimCompilerUtils +) + +add_pim_unittest(ProjectionMemoryCostTest + ProjectionMemoryCostTest.cpp + LINK_LIBS PRIVATE OMPimCompilerUtils +) +add_pim_unittest(PimAbstractExecutionTest + PimAbstractExecutionTest.cpp + LINK_LIBS PRIVATE OMPimCompilerUtils +) + add_pim_unittest(SpatialSchedulingTargetTest SpatialSchedulingTargetTest.cpp @@ -77,3 +102,8 @@ add_pim_unittest(PimPipelineSelectionTest PimPipelineSelectionTest.cpp LINK_LIBS PRIVATE OMPimCompilerUtils ) + +add_test(NAME PimSpatialSelectionTest + COMMAND "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_PimSpatialSelection.py" + --compiler "$") +set_tests_properties(PimSpatialSelectionTest PROPERTIES LABELS pim-unittest) diff --git a/test/PIM/ComputeInputMaterializationTest.cpp b/test/PIM/ComputeInputMaterializationTest.cpp new file mode 100644 index 0000000..02cec18 --- /dev/null +++ b/test/PIM/ComputeInputMaterializationTest.cpp @@ -0,0 +1,158 @@ +#include "TestCheck.hpp" +#include +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/IR/Verifier.h" +#include "src/Accelerators/PIM/Common/IR/ConstantUtils.hpp" +#include "src/Accelerators/PIM/Conversion/SpatialToPim/ComputeLikeRegionUtils.hpp" +#include "src/Accelerators/PIM/Conversion/SpatialToPim/Common.hpp" +#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" + +using namespace mlir; +using namespace onnx_mlir; +using namespace onnx_mlir::spatial; + +static void testZeroContent(OpBuilder &builder) { + auto type = RankedTensorType::get({4}, builder.getF32Type()); + for (double value : {0.0, -0.0, 1.0, -1.0}) { + auto attribute = DenseElementsAttr::get(type, builder.getF32FloatAttr(value)); + TEST_CHECK(isZeroBitPatternSplat(attribute) == (value == 0.0 && !std::signbit(value))); + } + auto integerType = RankedTensorType::get({4}, builder.getI32Type()); + TEST_CHECK(isZeroBitPatternSplat(DenseElementsAttr::get(integerType, builder.getI32IntegerAttr(0)))); + TEST_CHECK(!isZeroBitPatternSplat(DenseElementsAttr::get(integerType, builder.getI32IntegerAttr(1)))); + TEST_CHECK(!isZeroBitPatternSplat(DenseElementsAttr::get(type, ArrayRef{0, 1, 0, 0}))); + TEST_CHECK(!isZeroBitPatternSplat(builder.getF32FloatAttr(0))); + TEST_CHECK(!isZeroBitPatternSplat({})); +} + +static void testInputPlan(MLIRContext &context, bool batched) { + OwningOpRef module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module->getBodyRegion()); + auto loc = module->getLoc(); + auto type = RankedTensorType::get({1, 4}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({type}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + Value constant = arith::ConstantOp::create(builder, loc, + DenseElementsAttr::get(type, builder.getF32FloatAttr(2))); + Value view = tensor::CastOp::create(builder, loc, type, constant); + SmallVector inputs{constant, function.getArgument(0), constant, view, function.getArgument(0)}; + Operation *owner; + if (batched) { + auto batch = SpatScheduledComputeBatch::create(builder, loc, TypeRange{}, + builder.getI32IntegerAttr(2), ValueRange{}, inputs); + batch->setAttr("coreIds", builder.getDenseI32ArrayAttr({0, 1})); + owner = batch; + } else { + auto single = SpatScheduledCompute::create(builder, loc, TypeRange{}, ValueRange{}, inputs); + single->setAttr("coreId", builder.getI32IntegerAttr(0)); + owner = single; + } + auto *body = new Block; + owner->getRegion(0).push_back(body); + if (batched) body->addArgument(builder.getIndexType(), loc); + for (Value input : inputs) body->addArgument(input.getType(), loc); + builder.setInsertionPointToStart(body); + unsigned first = batched ? 1 : 0; + Value zero = arith::ConstantIndexOp::create(builder, loc, 0); + Value limit = arith::ConstantIndexOp::create(builder, loc, 1000000000000); + Value one = arith::ConstantIndexOp::create(builder, loc, 1); + auto loop = scf::ForOp::create(builder, loc, zero, limit, one); + builder.setInsertionPointToStart(loop.getBody()); + SpatVAddOp::create(builder, loc, type, body->getArgument(first), body->getArgument(first + 1)); + SpatVAddOp::create(builder, loc, type, body->getArgument(first + 3), body->getArgument(first)); + builder.setInsertionPointAfter(loop); + auto lastUse = SpatVAddOp::create(builder, loc, type, body->getArgument(first + 2), body->getArgument(first)); + auto lastFunctionUse = SpatVAddOp::create(builder, loc, type, + body->getArgument(first + 4), body->getArgument(first)); + SpatYieldOp::create(builder, loc, ValueRange{}); + builder.setInsertionPointAfter(owner); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(*module))); + // Body lowering can remove an input's final use before input materialization. + lastUse.erase(); + lastFunctionUse.erase(); + + std::string before; + llvm::raw_string_ostream stream(before); + module->print(stream); + for (unsigned repeat = 0; repeat < 2; ++repeat) { + auto constantPlan = planComputeInputMaterialization(owner, 0); + auto runtimePlan = planComputeInputMaterialization(owner, 1); + auto unusedPlan = planComputeInputMaterialization(owner, 2); + auto viewPlan = planComputeInputMaterialization(owner, 3); + TEST_CHECK(succeeded(constantPlan) && succeeded(runtimePlan) && succeeded(unusedPlan)); + TEST_CHECK(*constantPlan == (batched ? ComputeInputMaterialization::CopyAtEntry + : ComputeInputMaterialization::ForwardConstant)); + TEST_CHECK(*runtimePlan == ComputeInputMaterialization::ForwardFunctionInput); + auto unusedFunctionPlan = planComputeInputMaterialization(owner, 4); + TEST_CHECK(succeeded(unusedFunctionPlan) + && *unusedFunctionPlan == (batched ? ComputeInputMaterialization::ForwardFunctionInput + : ComputeInputMaterialization::Unused)); + TEST_CHECK(*unusedPlan == (batched ? ComputeInputMaterialization::CopyAtEntry + : ComputeInputMaterialization::Unused)); + TEST_CHECK(succeeded(viewPlan) && *viewPlan == ComputeInputMaterialization::CopyAtEntry); + } + TEST_CHECK(failed(planComputeInputMaterialization(owner, 5))); + TEST_CHECK(failed(planComputeInputMaterialization(function, 0))); + std::string after; + llvm::raw_string_ostream afterStream(after); + module->print(afterStream); + TEST_CHECK(before == after); +} + +static void testOutputReuse(MLIRContext &context) { + OwningOpRef module = ModuleOp::create(UnknownLoc::get(&context)); + IRRewriter builder(&context); + builder.setInsertionPointToStart(module->getBody()); + auto loc = module->getLoc(); + auto type = RankedTensorType::get({1, 4}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "reuse", builder.getFunctionType({}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + Value input = tensor::EmptyOp::create(builder, loc, type.getShape(), type.getElementType()); + Value constant = arith::ConstantOp::create(builder, loc, + DenseElementsAttr::get(type, builder.getF32FloatAttr(2))); + auto first = SpatVAddOp::create(builder, loc, type, input, constant); + TEST_CHECK(getReusableOutputTensor(first) == input); + auto second = SpatVAddOp::create(builder, loc, type, input, constant); + TEST_CHECK(!getReusableOutputTensor(first)); + TEST_CHECK(getReusableOutputTensor(second) == input); + second->moveBefore(first); + TEST_CHECK(getReusableOutputTensor(first) == input); + TEST_CHECK(!getReusableOutputTensor(second)); + second->moveAfter(first); + TEST_CHECK(!getReusableOutputTensor(first)); + TEST_CHECK(getReusableOutputTensor(second) == input); + Value view = tensor::CastOp::create(builder, loc, type, input); + auto fromView = SpatVAddOp::create(builder, loc, type, view, constant); + TEST_CHECK(!getReusableOutputTensor(fromView)); + TEST_CHECK(getBestOutputTensorFromOperandsOrAllocate(builder, fromView) + .getDefiningOp()); + builder.setInsertionPointAfter(fromView); + Value zero = arith::ConstantIndexOp::create(builder, loc, 0); + Value limit = arith::ConstantIndexOp::create(builder, loc, 10); + Value one = arith::ConstantIndexOp::create(builder, loc, 1); + auto loop = scf::ForOp::create(builder, loc, zero, limit, one, ValueRange{input}); + builder.setInsertionPointToStart(loop.getBody()); + auto carried = SpatVAddOp::create(builder, loc, type, loop.getRegionIterArg(0), constant); + scf::YieldOp::create(builder, loc, ValueRange{carried}); + TEST_CHECK(getReusableOutputTensor(carried) == loop.getRegionIterArg(0)); + TEST_CHECK(getBestOutputTensorFromOperandsOrAllocate(builder, carried) == loop.getRegionIterArg(0)); + builder.setInsertionPointAfter(loop); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(*module))); +} + +int main() { + MLIRContext context(MLIRContext::Threading::DISABLED); + context.loadDialect(); + OpBuilder builder(&context); + testZeroContent(builder); + testInputPlan(context, false); + testInputPlan(context, true); + testOutputReuse(context); +} diff --git a/test/PIM/CopyAccessPlanTest.cpp b/test/PIM/CopyAccessPlanTest.cpp new file mode 100644 index 0000000..8232f00 --- /dev/null +++ b/test/PIM/CopyAccessPlanTest.cpp @@ -0,0 +1,96 @@ +#include "src/Accelerators/PIM/Common/IR/CopyAccessPlan.hpp" +#include "src/Accelerators/PIM/Common/IR/ShapeUtils.hpp" +#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/SchedulingTarget.hpp" +#include "TestCheck.hpp" +#include +#include + +using namespace onnx_mlir; + +int main(int argc, char **argv) { + if (argc > 1 && std::string(argv[1]) == "--stride-overflow") { + computeRowMajorStrides({2, std::numeric_limits::max(), 2}); + return 0; + } + if (argc > 1) TEST_CHECK(false); + auto dense = CopyAccessPlan::build({3, 4}, {4, 1}, {4, 1}); + TEST_CHECK(mlir::succeeded(dense) && dense->getRunCount() == 1 && dense->getRunElements() == 12); + auto strided = CopyAccessPlan::build({3, 4}, {8, 1}, {4, 1}); + TEST_CHECK(mlir::succeeded(strided) && strided->getRunCount() == 3 && strided->getRunElements() == 4); + TEST_CHECK(*strided->getRunOffsets(2) == std::make_pair(int64_t{16}, int64_t{8})); + TEST_CHECK(mlir::failed(strided->getRunOffsets(-1)) && mlir::failed(strided->getRunOffsets(3))); + auto large = CopyAccessPlan::build({1000000000, 16}, {32, 1}, {16, 1}); + TEST_CHECK(mlir::succeeded(large) && large->getOuterShape().size() == 1); + TEST_CHECK(large->getRunCount() == 1000000000 && large->getRunElements() == 16); + TEST_CHECK(*large->getRunOffsets(999999999) == std::make_pair(int64_t{31999999968}, int64_t{15999999984})); + auto scalar = CopyAccessPlan::build({}, {}, {}); + TEST_CHECK(mlir::succeeded(scalar) && scalar->getRunCount() == 1 && scalar->getRunElements() == 1); + // Observed input planes and channel-wise publication slices, without PIM IR. + auto input = CopyAccessPlan::build({3, 1, 224}, {50176, 224, 1}, {224, 224, 1}); + TEST_CHECK(mlir::succeeded(input) && input->getRunCount() == 3 && input->getRunElements() == 224); + TEST_CHECK(*input->getRunOffsets(1) == std::make_pair(int64_t{50176}, int64_t{224})); + for (int64_t channels : {32, 96}) { + auto publication = CopyAccessPlan::build({1, channels, 1, 80}, + {channels * 6400, 6400, 80, 1}, {channels * 80, 80, 80, 1}); + TEST_CHECK(mlir::succeeded(publication) && publication->getRunCount() == channels); + TEST_CHECK(publication->getRunElements() == 80); + TEST_CHECK(*publication->getRunOffsets(1) == std::make_pair(int64_t{6400}, int64_t{80})); + } + TEST_CHECK(mlir::failed(CopyAccessPlan::build({2}, {}, {1}))); + TEST_CHECK(mlir::failed(CopyAccessPlan::build({0}, {1}, {1}))); + TEST_CHECK(mlir::failed(CopyAccessPlan::build({-1}, {1}, {1}))); + TEST_CHECK(mlir::failed(CopyAccessPlan::build({2}, {-1}, {1}))); + int64_t limit = std::numeric_limits::max(); + TEST_CHECK(mlir::failed(computeRowMajorStridesChecked({2, limit, 2}))); + auto limitStrides = computeRowMajorStridesChecked({2, limit}); + TEST_CHECK(mlir::succeeded(limitStrides) && (*limitStrides)[0] == limit); + TEST_CHECK(mlir::failed(CopyAccessPlan::build({limit, 2}, {2, 1}, {2, 1}))); + TEST_CHECK(mlir::failed(CopyAccessPlan::build({limit, 2}, {3, 1}, {3, 1}))); + TEST_CHECK(mlir::failed(CopyAccessPlan::build({3}, {limit}, {1}))); + TEST_CHECK(mlir::failed(CopyAccessPlan::build({3}, {1}, {limit}))); + // A run start can fit even when its final element does not. + TEST_CHECK(mlir::failed(CopyAccessPlan::build({2, 2}, {limit, 1}, {2, 1}))); + TEST_CHECK(mlir::failed(CopyAccessPlan::build({2, 2}, {2, 1}, {limit, 1}))); + auto boundary = CopyAccessPlan::build({2}, {limit}, {1}); + TEST_CHECK(mlir::succeeded(boundary) && boundary->getRunOffsets(1)->first == limit); + auto unit = CopyAccessPlan::build({1, 2}, {limit, 1}, {-1, 1}); + TEST_CHECK(mlir::succeeded(unit) && unit->getRunElements() == 2); + + // Independent element-by-element reference checks ordered offsets, not totals. + std::mt19937 random(123); + for (unsigned trial = 0; trial < 2000; ++trial) { + size_t rank = random() % 4 + 1; + llvm::SmallVector shape(rank), source(rank), target(rank); + for (auto &size : shape) size = random() % 4 + 1; + auto rowMajor = computeRowMajorStrides(shape); + for (size_t dim = 0; dim < rank; ++dim) { + source[dim] = random() % 2 ? rowMajor[dim] : random() % 20; + target[dim] = random() % 2 ? rowMajor[dim] : random() % 20; + } + auto plan = CopyAccessPlan::build(shape, source, target); + TEST_CHECK(mlir::succeeded(plan)); + int64_t linear = 0; + for (int64_t run = 0; run < plan->getRunCount(); ++run) { + auto offsets = plan->getRunOffsets(run); + TEST_CHECK(mlir::succeeded(offsets)); + for (int64_t element = 0; element < plan->getRunElements(); ++element, ++linear) { + auto indices = delinearizeIndex(linear, shape, rowMajor); + TEST_CHECK(offsets->first + element == linearizeIndex(indices, source)); + TEST_CHECK(offsets->second + element == linearizeIndex(indices, target)); + } + } + TEST_CHECK(linear == getNumElements(shape)); + } + + // The cost caller and lowering use the same geometry; target timing stays separate. + for (int64_t width : {8, 16}) { + spatial::SchedulingTarget target; + target.sharedMemoryWidthBytes = width; + target.sharedMemoryReadLatencyNs = width == 8 ? 3 : 8; + auto access = CopyAccessPlan::build({3, 5}, {8, 1}, {5, 1}); + TEST_CHECK(mlir::succeeded(access)); + auto demand = target.getSharedMemoryDemand(access->getRunElements() * 4, access->getRunCount(), false); + TEST_CHECK(demand.readBytes == 60 && demand.readRequests == 3); + TEST_CHECK(demand.serviceNs == 3 * ((20 + width - 1) / width) * target.sharedMemoryReadLatencyNs); + } +} diff --git a/test/PIM/PimAbstractExecutionTest.cpp b/test/PIM/PimAbstractExecutionTest.cpp new file mode 100644 index 0000000..b21abd6 --- /dev/null +++ b/test/PIM/PimAbstractExecutionTest.cpp @@ -0,0 +1,864 @@ +#include "TestCheck.hpp" +#include +#include "src/Accelerators/PIM/Compiler/PimPeriodicAnalysis.hpp" +#include "src/Accelerators/PIM/Compiler/PimInstructionTiming.hpp" +#include "src/Accelerators/PIM/Compiler/PimSpatialCost.hpp" +#include "src/Accelerators/PIM/Compiler/PimBinaryFormat.hpp" +#include "src/Accelerators/PIM/Compiler/PimDependencyTokens.hpp" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/IR/Verifier.h" +#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" + +using namespace onnx_mlir; +using namespace mlir; +using E = PimExecutionEvent; +using O = pim_binary::Opcode; + +static PimInstructionTiming hardware(int64_t period) { + PimInstructionTiming target; + target.processorCount = 2; + target.processorPeriodNs = period; + target.localMemoryPeriodNs = 3; + target.peerPermissionThenSourceRead = true; + target.signalIssueNs = period; + target.controlMessageBytes = 1; + target.sharedMemoryWidthBytes = 64; + target.sharedMemoryReadLatencyNs = 80; + target.sharedMemoryWriteLatencyNs = 120; + target.sharedMemoryRequestLatencyNs = {2, 3}; + target.sharedMemoryResponseLatencyNs = {2, 3}; + target.interProcessorLatencyNs = {0, 2, 2, 0}; + target.synchronizationRegisterCount = 4; + return target; +} + +static std::string binary(const std::vector &records) { + std::string result(12, '\0'); + result.replace(0, 4, "PIMB"); + llvm::support::endian::write32le(result.data() + 4, 1); + llvm::support::endian::write32le(result.data() + 8, records.size()); + for (const auto &record : records) { + auto encoded = pim_binary::encodeInstructionRecord(record); + result.append(encoded.data(), encoded.size()); + } + return result; +} + +static void testTypedResult(PimInstructionTiming target) { + PimPeriodicOptions options; + options.verifyPeriodicState = true; + PimExecutionProgram program{{{E::Work, 29}, {E::Load, 65}}}; + auto estimate = evaluatePimExecution(program, target, options); + TEST_CHECK(estimate.cost.available() && estimate.cost.convergenceProved() == true); + TEST_CHECK(estimate.report.getString("status") == "estimated"); + TEST_CHECK(estimate.cost.loweringSupport == spatial::StaticLoweringSupport::Unknown); + TEST_CHECK(!estimate.cost.eligible() && !estimate.cost.selectionMargin); + auto *queue = estimate.report.getObject("memory_controller"); + for (size_t policy = 0; policy < 2; ++policy) { + auto *state = queue->getObject(policy ? "alternative_periodic_state" : "periodic_state"); + const auto &proof = estimate.cost.recurrence[policy]; + TEST_CHECK(proof.proved() && state->getString("status") == "proved"); + TEST_CHECK(proof.intervalNs == state->getNumber("interval_ns")); + TEST_CHECK(proof.periodGenerations == state->getInteger("period_generations")); + } + auto primary = estimate.cost.primary, alternative = estimate.cost.alternative; + estimate.report = llvm::json::Object{{"status", "unavailable"}, {"convergence_proved", false}}; + TEST_CHECK(estimate.cost.primary == primary && estimate.cost.alternative == alternative); + TEST_CHECK(estimate.cost.convergenceProved() == true); + options.verifyPeriodicState = false; + estimate = evaluatePimExecution(program, target, options); + TEST_CHECK(estimate.cost.available() && !estimate.cost.convergenceProved().has_value()); + TEST_CHECK(estimate.report.getString("status") == "estimated"); + TEST_CHECK(!estimate.report.getObject("memory_controller")->getObject("periodic_state")); + options.intervalWindowEventBudget = 0; + for (bool detailed : {false, true}) { + options.detailed = detailed; + estimate = evaluatePimExecution(program, target, options); + TEST_CHECK(estimate.queue && !estimate.cost.available()); + TEST_CHECK(!estimate.cost.convergenceProved().has_value()); + TEST_CHECK(estimate.report.getString("status") == "unavailable"); + TEST_CHECK(estimate.report.getString("reason") == estimate.cost.reason); + TEST_CHECK(estimate.report.getObject("memory_controller")->getString("status") == "unavailable"); + TEST_CHECK(!estimate.report.getObject("memory_controller")->getObject("periodic_state")); + } + options.intervalWindowEventBudget.reset(); + options.detailed = false; + options.verifyPeriodicState = true; + estimate = evaluatePimExecution({{{E::Wait, 0, 0, 0, 1}, {E::Load, 65}}}, target, options); + TEST_CHECK(!estimate.cost.available() && estimate.cost.convergenceProved() == false); + TEST_CHECK(!estimate.cost.reason.empty() && !estimate.cost.eligible()); + target.sharedMemoryWidthBytes = 0; + estimate = evaluatePimExecution(program, target, options); + TEST_CHECK(!estimate.cost.available() && !estimate.cost.reason.empty()); + TEST_CHECK(estimate.report.getString("status") == "unavailable"); + TEST_CHECK(estimate.report.getString("reason") == estimate.cost.reason); +} + +static void testLateRecurrenceAuthority(PimInstructionTiming target) { + PimPeriodicOptions options; + options.verifyPeriodicState = true; + options.intervalWindowEventBudget = 1; + PimExecutionProgram program{{{E::Work, 29}, {E::Load, 65}}}; + auto late = evaluatePimExecution(program, target, options); + TEST_CHECK(late.queue && late.cost.available() && late.cost.convergenceProved() == true); + for (const auto &policy : late.queue->policies) { + TEST_CHECK(policy.legacy.termination == PimQueueTermination::EventBudget); + TEST_CHECK(!policy.legacy.intervalNs && policy.recurrence.proved()); + TEST_CHECK(policy.proofProcessedEvents > policy.legacy.progress.processedEvents); + TEST_CHECK(policy.intervalNs() == policy.recurrence.intervalNs); + TEST_CHECK(policy.termination == PimQueueTermination::RecurrenceProved); + } + auto first = serializePimQueueEvaluation(*late.queue); + auto second = serializePimQueueEvaluation(*late.queue); + TEST_CHECK(first.getString("status") == "estimated" && !first.getString("reason")); + TEST_CHECK(first.getString("alternative_tie_status") == "estimated"); + TEST_CHECK(first.getNumber("interval_ns") == late.cost.primary); + TEST_CHECK(first.getNumber("alternative_tie_interval_ns") == late.cost.alternative); + TEST_CHECK(first.getObject("legacy_interval_window")->getString("status") == "unavailable"); + TEST_CHECK(first.getObject("legacy_interval_window")->getString("termination") == "event-budget"); + TEST_CHECK(llvm::json::Value(std::move(first)) == llvm::json::Value(std::move(second))); + + auto blocked = evaluatePimExecution({{{E::Wait, 0, 0, 0, 1}, {E::Signal, 0, 1}}, + {{E::Wait, 0, 0, 0, 1}, {E::Signal, 0, 0}}}, target, options); + TEST_CHECK(blocked.queue && blocked.cost.convergenceProved() == false); + TEST_CHECK(blocked.queue->policies[1].termination == PimQueueTermination::NoReadyEvents); + for (size_t failedPolicy = 0; failedPolicy < 2; ++failedPolicy) { + auto asymmetric = *late.queue; + asymmetric.policies[failedPolicy] = blocked.queue->policies[failedPolicy]; + auto cost = asymmetric.cost(); + cost.loweringSupport = spatial::StaticLoweringSupport::Supported; + cost.selectionMargin = 0.021; + TEST_CHECK(cost.primary.has_value() == (failedPolicy != 0)); + TEST_CHECK(cost.alternative.has_value() == (failedPolicy != 1)); + TEST_CHECK(cost.convergenceProved() == false && !cost.eligible()); + TEST_CHECK(cost.reason == blocked.queue->policies[failedPolicy].recurrence.reason); + auto report = serializePimQueueEvaluation(asymmetric); + TEST_CHECK(report.getString("status") == (failedPolicy == 0 ? "unavailable" : "estimated")); + TEST_CHECK(report.getString("alternative_tie_status") == (failedPolicy == 1 ? "unavailable" : "estimated")); + TEST_CHECK(report.getObject(failedPolicy == 0 ? "periodic_state" : "alternative_periodic_state") + ->getString("status") == "unproved"); + } + + options.intervalWindowEventBudget.reset(); + auto normal = evaluatePimExecution(program, target, options); + options.workBreakdown = true; + auto detailed = evaluatePimExecution(program, target, options); + TEST_CHECK(late.cost.primary == normal.cost.primary && normal.cost.primary == detailed.cost.primary); + TEST_CHECK(late.cost.alternative == normal.cost.alternative && normal.cost.alternative == detailed.cost.alternative); + TEST_CHECK(normal.queue->policies[0].progress.processedEvents + <= detailed.queue->policies[0].progress.processedEvents); +} + +static void testSharedEngine(PimInstructionTiming target) { + auto record = [](O op, int32_t peer = 0, int32_t reg = 0, int32_t count = 0, int32_t bytes = 0) { + pim_binary::InstructionRecord r; + r.opcode = op; r.r2OrImm = peer; r.generic1 = reg; r.generic2 = count; r.generic3 = bytes; + return r; + }; + auto vectorWork = getPimVectorWorkNs(target, 65, 1, 1, true); + PimInstructionProgram instructions; + instructions.instructions = { + binary({record(O::setbw, 0, 8, 8), record(O::ld, 0, 0, 0, 65), + record(O::vvadd, 0, 0, 0, 65), record(O::send, 1, 0, 0, 65), record(O::sync, 1, 0)}), + binary({record(O::recv, 0, 0, 0, 65), record(O::wait, 0, 0, 1), record(O::st, 0, 0, 0, 33)})}; + instructions.arrayGroups.resize(2); + PimExecutionProgram abstract(2); + abstract[0] = {{E::Work, target.processorPeriodNs}, {E::Load, 65}, {E::Work, vectorWork}, + {E::Send, 65, 1}, {E::Signal, 0, 1, 0}}; + abstract[1] = {{E::Receive, 65, 0}, {E::Wait, 0, 0, 0, 1}, {E::Store, 33}}; + PimPeriodicOptions details; + details.workBreakdown = true; + PimExecutionProgram trace; + details.executionTrace = &trace; + auto reference = analyzePimInstructions(instructions, target, 2, details); + TEST_CHECK(reference.getString("status") == "estimated"); + TEST_CHECK(!reference.getObject("memory_controller")->getObject("periodic_state")); + details.executionTrace = nullptr; + auto roundTrip = analyzePimExecution(trace, target, details); + for (const char *field : {"interval_ns", "alternative_tie_interval_ns"}) + TEST_CHECK(roundTrip.getObject("memory_controller")->getNumber(field) + == reference.getObject("memory_controller")->getNumber(field)); + for (size_t resource = 0; resource < 7; ++resource) + TEST_CHECK((*roundTrip.getArray("resource_service_ns"))[resource].getAsInteger() + == (*reference.getArray("resource_service_ns"))[resource].getAsInteger()); + details.verifyPeriodicState = true; + auto proof = analyzePimExecution(trace, target, details); + TEST_CHECK(proof.getObject("memory_controller")->getString("status") + == reference.getObject("memory_controller")->getString("status")); + TEST_CHECK(proof.getObject("memory_controller")->getObject("legacy_interval_window")->getNumber("interval_ns") + == reference.getObject("memory_controller")->getNumber("interval_ns")); + TEST_CHECK(proof.getObject("memory_controller")->getNumber("interval_ns") + == proof.getObject("memory_controller")->getObject("periodic_state")->getNumber("interval_ns")); + TEST_CHECK(proof.getObject("memory_controller")->getObject("periodic_state")); + details.verifyPeriodicState = false; + auto actual = analyzePimExecution(abstract, target, details); + for (size_t core = 0; core < 2; ++core) + TEST_CHECK((*actual.getArray("request_order"))[core].getAsObject()->getString("identity") + == (*reference.getArray("request_order"))[core].getAsObject()->getString("identity")); + for (size_t core = 0; core < 2; ++core) + TEST_CHECK((*actual.getArray("request_order"))[core].getAsObject()->getString("blocking_identity") + == (*reference.getArray("request_order"))[core].getAsObject()->getString("blocking_identity")); + for (const char *field : {"global_read_bytes", "global_write_bytes", "global_read_requests", + "global_write_requests", "global_memory_service_ns"}) + TEST_CHECK(actual.getInteger(field) == reference.getInteger(field)); + auto *a = actual.getObject("memory_controller"), *b = reference.getObject("memory_controller"); + TEST_CHECK(a && b && a->getString("status") == "estimated"); + TEST_CHECK(a->getNumber("interval_ns") == b->getNumber("interval_ns")); + TEST_CHECK(a->getNumber("alternative_tie_interval_ns") == b->getNumber("alternative_tie_interval_ns")); + abstract[1].clear(); + abstract[0] = {{E::Load, 1}, {E::Load, 65}}; + auto ordered = analyzePimExecution(abstract, target, details); + abstract[0] = {{E::Load, 65}, {E::Load, 1}}; + auto reversed = analyzePimExecution(abstract, target, details); + TEST_CHECK(ordered.getInteger("global_read_bytes") == reversed.getInteger("global_read_bytes")); + TEST_CHECK((*ordered.getArray("request_order"))[0].getAsObject()->getString("identity") + != (*reversed.getArray("request_order"))[0].getAsObject()->getString("identity")); + // Preserve repeated request order, not a single larger request. + abstract[0] = {{E::Repeat, 2, 0, 0, 0, {{E::Load, 1}, {E::Work, 4}}}}; + abstract[1].clear(); + actual = analyzePimExecution(abstract, target); + TEST_CHECK(actual.getInteger("global_read_requests") == 2); + TEST_CHECK(actual.getInteger("global_memory_service_ns") == 160); + abstract[0] = {{E::Repeat, 1000000000000ull, 0, 0, 0, {{E::Load, 1}}}}; + actual = analyzePimExecution(abstract, target); + TEST_CHECK(actual.getString("status") == "unavailable"); + abstract[0] = {{E::Repeat, 1000000000000ull, 0, 0, 0, {{E::Work, 4}}}}; + actual = analyzePimExecution(abstract, target); + TEST_CHECK(actual.getObject("memory_controller")->getNumber("interval_ns") == 4000000000000.0); + abstract[0] = {{E::Signal, 0, int64_t(1) << 32}}; + TEST_CHECK(analyzePimExecution(abstract, target).getString("status") == "unavailable"); + TEST_CHECK(getPimMatrixOutputBytes(target, 2, pim_binary::kMatrixWeightPrecisionBits, 3, 1) + == target.matrixColumns * 2 / pim_binary::kMatrixWeightPrecisionBits * 3); + E copy{E::Work, 4000}; + copy.workClass = E::LocalCopy; + abstract[0] = {copy, {E::Load, 1}}; + auto charged = analyzePimExecution(abstract, target); + PimPeriodicOptions sensitivity; + sensitivity.ignoreLocalCopies = true; + auto omitted = analyzePimExecution(abstract, target, sensitivity); + TEST_CHECK(*charged.getObject("memory_controller")->getNumber("interval_ns") + - *omitted.getObject("memory_controller")->getNumber("interval_ns") == 4000); +} + +static void testAccountingAndOverflow(PimInstructionTiming target) { + auto record = [](O op, int64_t bytes) { + pim_binary::InstructionRecord r; + r.opcode = op; r.generic3 = bytes; + return r; + }; + PimPeriodicOptions options; + options.scalarIssue = false; + options.workBreakdown = true; + auto setbw = record(O::setbw, 0); + setbw.generic1 = setbw.generic2 = 8; + auto matrixInstruction = record(O::mvmul, 0); + matrixInstruction.r2OrImm = pim_binary::kMatrixWeightPrecisionBits; + PimInstructionProgram reference; + reference.instructions = {binary({setbw, matrixInstruction, record(O::vvadd, 65), record(O::vmv, 3), record(O::lmv, 65), record(O::lldi, 65)})}; + reference.arrayGroups = {{3}}; + E matrix{E::Work, getPimMatrixWorkNs(target, 8, + getPimMatrixOutputBytes(target, 2, pim_binary::kMatrixWeightPrecisionBits, 3, 1))}; + matrix.workClass = E::Matrix; + E vector{E::Work, getPimVectorWorkNs(target, 65, 1, 1, true)}; + vector.workClass = E::Vector; + E vectorCopy{E::Work, getPimVectorWorkNs(target, 3, 1, 1, false)}; + vectorCopy.workClass = E::VectorCopy; + E copy{E::Work, target.getLocalMemoryServiceNs(65, false) + + target.getLocalMemoryServiceNs(65, true) + target.processorPeriodNs}; + copy.workClass = E::LocalCopy; + E fill{E::Work, target.getLocalMemoryServiceNs(65, true) + target.processorPeriodNs}; + fill.workClass = E::Fill; + auto exact = analyzePimInstructions(reference, target, 2, options); + auto abstract = analyzePimExecution({{matrix, vector, vectorCopy, copy, fill}}, target, options); + TEST_CHECK(*abstract.getArray("resource_service_ns") == *exact.getArray("resource_service_ns")); + TEST_CHECK(abstract.getObject("memory_controller")->getNumber("interval_ns") + == exact.getObject("memory_controller")->getNumber("interval_ns")); + auto repeated = analyzePimExecution({{{E::Repeat, 7, 0, 0, 0, {matrix, vector, vectorCopy, copy, fill}}}}, target, options); + for (size_t i = 0; i < 7; ++i) + TEST_CHECK(*(*repeated.getArray("resource_service_ns"))[i].getAsInteger() + == 7 * *(*abstract.getArray("resource_service_ns"))[i].getAsInteger()); + auto nested = analyzePimExecution({{{E::Repeat, 1000000, 0, 0, 0, + {{E::Repeat, 1000000, 0, 0, 0, {fill}}}}}}, target, options); + TEST_CHECK(*(*nested.getArray("resource_service_ns"))[2].getAsInteger() + == static_cast(1000000000000ull * fill.amount)); + TEST_CHECK(nested.getInteger("expanded_graph_events").value() < 10); + PimExecutionProgram pattern(2), expanded(2); + pattern[0] = {{E::Repeat, 3, 0, 0, 0, {{E::Load, 65}, + {E::Repeat, 5, 0, 0, 0, {matrix, vector, copy, fill}}}}}; + for (size_t repeat = 0; repeat < 3; ++repeat) { + expanded[0].push_back({E::Load, 65}); + for (size_t work = 0; work < 5; ++work) + expanded[0].insert(expanded[0].end(), {matrix, vector, copy, fill}); + } + pattern[1] = expanded[1] = {{E::Work, 311}, {E::Load, 257}}; + auto compactCost = analyzePimExecution(pattern, target, options); + auto expandedCost = analyzePimExecution(expanded, target, options); + for (const char *policy : {"interval_ns", "alternative_tie_interval_ns"}) + TEST_CHECK(compactCost.getObject("memory_controller")->getNumber(policy) + == expandedCost.getObject("memory_controller")->getNumber(policy)); + TEST_CHECK(*compactCost.getArray("resource_service_ns") == *expandedCost.getArray("resource_service_ns")); + std::vector body{{E::Load, 1}}; + for (size_t i = 0; i < 10000; ++i) { + E event{E::Work, 1}; event.workClass = i % 2 ? E::Vector : E::Fill; + body.push_back(event); + } + auto cachedWork = analyzePimExecution({{{E::Repeat, 10000, 0, 0, 0, body}}}, target, options); + TEST_CHECK(cachedWork.getInteger("global_read_requests") == 10000); + TEST_CHECK(cachedWork.getInteger("expanded_graph_events").value() < 100000); + TEST_CHECK((*cachedWork.getArray("resource_service_ns"))[1].getAsInteger() == 50000000); + TEST_CHECK((*cachedWork.getArray("resource_service_ns"))[2].getAsInteger() == 50000000); + uint64_t maximum = std::numeric_limits::max(); + for (auto program : std::vector{ + {{{E::Work, maximum}}}, {{{E::Load, maximum}}}, + {{{E::Repeat, maximum, 0, 0, 0, {fill}}}}, + {{{E::Work, uint64_t(1) << 62}, {E::Work, uint64_t(1) << 62}}}}) + TEST_CHECK(analyzePimExecution(program, target).getString("status") == "unavailable"); + target.sharedMemoryReadLatencyNs = maximum; + TEST_CHECK(analyzePimExecution({{{E::Load, 65}}}, target).getString("status") == "unavailable"); +} + +static PimPeriodicOptions spatialDetails() { + PimPeriodicOptions options; + options.workBreakdown = true; + return options; +} + +static void testCoreAssignments() { + MLIRContext context; + context.loadDialect(); + OwningOpRef module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module->getBodyRegion()); + auto loc = module->getLoc(); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + auto empty = [&](std::optional id) { + auto compute = spatial::SpatScheduledCompute::create(builder, loc, TypeRange{}, ValueRange{}, ValueRange{}); + if (id) compute->setAttr("coreId", builder.getI32IntegerAttr(*id)); + compute.getBody().push_back(new Block); + builder.setInsertionPointToStart(&compute.getBody().front()); + spatial::SpatYieldOp::create(builder, loc, ValueRange{}); + builder.setInsertionPointAfter(compute); + return compute; + }; + auto first = empty(0); + auto second = empty(1); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(*module))); + auto target = hardware(1); + auto reason = [&]() { + auto report = analyzePimSpatialSchedule(*module, target); + return report.getString("reason").value_or("").str(); + }; + TEST_CHECK(reason().find("physical core") == std::string::npos); + second->setAttr("coreId", builder.getI32IntegerAttr(0)); + TEST_CHECK(reason() == "duplicate physical core assignment: 0"); + TEST_CHECK(evaluatePimSpatialSchedule(*module, target).cost.loweringSupport + == spatial::StaticLoweringSupport::Unsupported); + second->removeAttr("coreId"); + TEST_CHECK(reason() == "missing physical core"); + TEST_CHECK(evaluatePimSpatialSchedule(*module, target).cost.loweringSupport + == spatial::StaticLoweringSupport::Unknown); + for (int32_t id : {-1, 2}) { + second->setAttr("coreId", builder.getI32IntegerAttr(id)); + TEST_CHECK(reason() == "physical core ID is outside the target: " + std::to_string(id)); + } + second.erase(); + builder.setInsertionPoint(function.getBody().front().getTerminator()); + auto batch = spatial::SpatScheduledComputeBatch::create(builder, loc, TypeRange{}, + builder.getI32IntegerAttr(2), ValueRange{}, ValueRange{}); + batch.getBody().push_back(new Block); + batch.getBody().front().addArgument(builder.getIndexType(), loc); + builder.setInsertionPointToStart(&batch.getBody().front()); + spatial::SpatYieldOp::create(builder, loc, ValueRange{}); + TEST_CHECK(reason() == "missing physical batch cores"); + batch->setAttr("coreIds", builder.getDenseI32ArrayAttr({1, 1})); + TEST_CHECK(reason() == "duplicate physical core assignment: 1"); + batch->setAttr("coreIds", builder.getDenseI32ArrayAttr({1, 0})); + TEST_CHECK(reason() == "duplicate physical core assignment: 0"); + TEST_CHECK(evaluatePimSpatialSchedule(*module, target).cost.loweringSupport + == spatial::StaticLoweringSupport::Unsupported); + first.erase(); + batch->setAttr("coreIds", builder.getDenseI32ArrayAttr({1, 2})); + TEST_CHECK(reason() == "physical core ID is outside the target: 2"); +} + +static void testDependencyTokens() { + PimDependencyTokens fresh, drained; + TEST_CHECK(fresh.empty() && fresh.remaining().empty()); + drained.initialize(1); + TEST_CHECK(drained.consume() == 0); + TEST_CHECK(drained.empty() && drained.remaining().empty()); + // Empty queues have identical future behavior, regardless of their capacity. + for (uint64_t time : {0, 7, 7, 123}) { + fresh.push(time); drained.push(time); + TEST_CHECK(fresh.remaining() == drained.remaining()); + TEST_CHECK(fresh.consume() == drained.consume()); + TEST_CHECK(fresh.empty() && drained.empty()); + } + // Consumed prefixes and compaction cannot change FIFO contents or readiness. + drained.initialize(3); + TEST_CHECK(drained.consume() == 0); + drained.push(11); + TEST_CHECK(drained.consume() == 0); + TEST_CHECK(drained.consume() == 0); + drained.push(11); + drained.push(12); + for (uint64_t time : {11, 11, 12}) TEST_CHECK(drained.consume() == time); + TEST_CHECK(drained.empty()); + drained.initialize(0); + TEST_CHECK(drained.empty() && drained.remaining().empty()); +} + +static void testReadOnlySpatial(PimInstructionTiming target, bool batched) { + MLIRContext context; + context.loadDialect(); + auto module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module.getBodyRegion()); + auto loc = module.getLoc(); + auto type = RankedTensorType::get({1, 64}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + Value constant = arith::ConstantOp::create(builder, loc, + DenseElementsAttr::get(type, builder.getF32FloatAttr(2))); + Operation *compute; + if (batched) { + compute = spatial::SpatScheduledComputeBatch::create(builder, loc, TypeRange{}, + builder.getI32IntegerAttr(2), ValueRange{}, ValueRange{constant}); + compute->setAttr("coreIds", builder.getDenseI32ArrayAttr({0, 1})); + } else { + compute = spatial::SpatScheduledCompute::create(builder, loc, TypeRange{}, ValueRange{}, ValueRange{constant}); + compute->setAttr("coreId", builder.getI32IntegerAttr(0)); + } + Block *body = new Block; + compute->getRegion(0).push_back(body); + if (batched) body->addArgument(builder.getIndexType(), loc); + Value input = body->addArgument(type, loc); + builder.setInsertionPointToStart(body); + Value zero = arith::ConstantIndexOp::create(builder, loc, 0); + Value upper = arith::ConstantIndexOp::create(builder, loc, 28); + Value one = arith::ConstantIndexOp::create(builder, loc, 1); + Value local = tensor::EmptyOp::create(builder, loc, type.getShape(), type.getElementType()); + auto loop = scf::ForOp::create(builder, loc, zero, upper, one); + builder.setInsertionPointToStart(loop.getBody()); + auto add = spatial::SpatVAddOp::create(builder, loc, type, local, input); + builder.setInsertionPointAfter(loop); + spatial::SpatWaitOp::create(builder, loc, zero, zero); + spatial::SpatYieldOp::create(builder, loc, ValueRange{}); + builder.setInsertionPointAfter(compute); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(module))); + std::string before, after; + llvm::raw_string_ostream os(before); module.print(os); + auto report = analyzePimSpatialSchedule(module, target, spatialDetails()); + auto fast = analyzePimSpatialSchedule(module, target); + PimPeriodicOptions proofOptions; + proofOptions.verifyPeriodicState = true; + auto typed = evaluatePimSpatialSchedule(module, target, proofOptions); + TEST_CHECK(typed.cost.available() && typed.cost.convergenceProved() == true); + TEST_CHECK(typed.cost.loweringSupport == spatial::StaticLoweringSupport::Supported); + TEST_CHECK(typed.report.getBoolean("lowering_supported") == typed.cost.loweringSupported()); + TEST_CHECK(typed.report.getBoolean("convergence_proved") == typed.cost.convergenceProved()); + TEST_CHECK(!fast.getArray("request_order") && !fast.getArray("resource_service_ns")); + TEST_CHECK(!fast.getObject("without_local_copy_work") && !fast.getArray("work_class_ns")); + for (const char *field : {"interval_ns", "alternative_tie_interval_ns"}) + TEST_CHECK(fast.getObject("memory_controller")->getNumber(field) + == report.getObject("memory_controller")->getNumber(field)); + TEST_CHECK(report.getBoolean("safe_for_selection") == false); + TEST_CHECK(report.getInteger("global_read_requests") == (batched ? 2 : 28)); + TEST_CHECK(report.getInteger("global_read_bytes") == (batched ? 2 : 28) * 64); + TEST_CHECK(report.getInteger("constant_read_requests") == (batched ? 2 : 28)); + llvm::raw_string_ostream out(after); module.print(out); + TEST_CHECK(before == after); + add->setAttr("test.annotation", builder.getUnitAttr()); + report = analyzePimSpatialSchedule(module, target, spatialDetails()); + TEST_CHECK(report.getInteger("global_read_requests") == (batched ? 2 : 28)); + builder.setInsertionPoint(body->getTerminator()); + Value source = tensor::EmptyOp::create(builder, loc, ArrayRef{1, 3, 1, 224}, builder.getF32Type()); + Value destination = tensor::EmptyOp::create(builder, loc, ArrayRef{1, 1, 224, 3}, builder.getF32Type()); + linalg::TransposeOp::create(builder, loc, source, destination, ArrayRef{0, 2, 3, 1}); + TEST_CHECK(succeeded(verify(module))); + report = analyzePimSpatialSchedule(module, target, spatialDetails()); + auto expectedCopyWork = 224 * getPimVectorWorkNs(target, 3, 1, 1, false) * (batched ? 2 : 1); + auto *classes = report.getArray("work_class_ns"); + auto *columns = report.getArray("work_class_columns"); + TEST_CHECK(classes && columns && classes->size() == 6 && columns->size() == 6); + const char *names[] = {"matrix", "vector", "local_copy", "fill", "vector_copy", "scalar_issue"}; + uint64_t totalWork = 0; + for (size_t i = 0; i < classes->size(); ++i) { + TEST_CHECK((*columns)[i].getAsString() == names[i]); + totalWork += *(*classes)[i].getAsInteger(); + } + TEST_CHECK((*classes)[E::VectorCopy].getAsInteger() == expectedCopyWork); + TEST_CHECK((*classes)[E::LocalCopy].getAsInteger() == 0); + TEST_CHECK((*classes)[E::ScalarIssue].getAsInteger() == 0); + TEST_CHECK(totalWork == report.getInteger("compute_and_local_work_ns")); + auto softmax = spatial::SpatSoftmaxOp::create(builder, loc, type, input); + TEST_CHECK(succeeded(verify(module))); + TEST_CHECK(analyzePimSpatialSchedule(module, target, spatialDetails()).getString("status") == "unavailable"); + softmax.erase(); + source = tensor::EmptyOp::create(builder, loc, ArrayRef{1, 0, 1, 224}, builder.getF32Type()); + destination = tensor::EmptyOp::create(builder, loc, ArrayRef{1, 1, 224, 0}, builder.getF32Type()); + linalg::TransposeOp::create(builder, loc, source, destination, ArrayRef{0, 2, 3, 1}); + TEST_CHECK(succeeded(verify(module))); + report = analyzePimSpatialSchedule(module, target, spatialDetails()); + TEST_CHECK(report.getString("reason") == "zero-element transpose has no abstract timing contract"); + module.erase(); +} + +static void testHostSlice(PimInstructionTiming target) { + MLIRContext context; + context.loadDialect(); + auto module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module.getBodyRegion()); + auto loc = module.getLoc(); + auto type = RankedTensorType::get({1, 3, 4, 4}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({type}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + auto batch = spatial::SpatScheduledComputeBatch::create(builder, loc, TypeRange{}, + builder.getI32IntegerAttr(2), ValueRange{}, ValueRange{function.getArgument(0)}); + batch->setAttr("coreIds", builder.getDenseI32ArrayAttr({0, 1})); + Block *body = new Block; + batch.getBody().push_back(body); + body->addArgument(builder.getIndexType(), loc); + Value input = body->addArgument(type, loc); + builder.setInsertionPointToStart(body); + auto offsets = getAsOpFoldResult(builder.getI64ArrayAttr({0, 0, 1, 0})); + auto sizes = getAsOpFoldResult(builder.getI64ArrayAttr({1, 3, 1, 4})); + auto strides = getAsOpFoldResult(builder.getI64ArrayAttr({1, 1, 1, 1})); + auto slice = tensor::ExtractSliceOp::create(builder, loc, input, offsets, sizes, strides); + spatial::SpatReluOp::create(builder, loc, slice.getType(), slice.getResult()); + spatial::SpatYieldOp::create(builder, loc, ValueRange{}); + builder.setInsertionPointAfter(batch); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(module))); + auto report = analyzePimSpatialSchedule(module, target, spatialDetails()); + TEST_CHECK(report.getInteger("global_read_requests") == 6); + TEST_CHECK(report.getInteger("global_read_bytes") == 24); + module.erase(); +} + +static void testCapturedVectorWeights(PimInstructionTiming target) { + MLIRContext context; + context.loadDialect(); + auto module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module.getBodyRegion()); + auto loc = module.getLoc(); + auto type = RankedTensorType::get({1, 64}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + Value zero = arith::ConstantOp::create(builder, loc, + DenseElementsAttr::get(type, builder.getF32FloatAttr(0))); + auto compute = spatial::SpatScheduledCompute::create(builder, loc, TypeRange{}, + ValueRange{zero}, ValueRange{}); + compute->setAttr("coreId", builder.getI32IntegerAttr(0)); + Block *body = new Block; + compute.getBody().push_back(body); + Value weight = body->addArgument(type, loc); + builder.setInsertionPointToStart(body); + Value local = tensor::EmptyOp::create(builder, loc, type.getShape(), type.getElementType()); + spatial::SpatVAddOp::create(builder, loc, type, local, weight); + spatial::SpatYieldOp::create(builder, loc, ValueRange{}); + builder.setInsertionPointAfter(compute); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(module))); + auto report = analyzePimSpatialSchedule(module, target, spatialDetails()); + // Captured weight operands stay block arguments; codegen cannot fold this load to LLDI. + TEST_CHECK(report.getInteger("global_read_requests") == 1); + TEST_CHECK(report.getInteger("constant_read_bytes") == 64); + module.erase(); +} + +static void testPublicationAndOutput(PimInstructionTiming target) { + MLIRContext context; + context.loadDialect(); + auto module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module.getBodyRegion()); + auto loc = module.getLoc(); + auto inputType = RankedTensorType::get({1, 2, 3, 5}, builder.getF32Type()); + auto outputType = RankedTensorType::get({1, 4, 3, 5}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({}, {outputType})); + builder.setInsertionPointToStart(function.addEntryBlock()); + auto compute = spatial::SpatScheduledCompute::create(builder, loc, TypeRange{outputType}, + ValueRange{}, ValueRange{}); + compute->setAttr("coreId", builder.getI32IntegerAttr(0)); + compute.getBody().push_back(new Block); + builder.setInsertionPointToStart(&compute.getBody().front()); + Value zero = arith::ConstantIndexOp::create(builder, loc, 0); + Value one = arith::ConstantIndexOp::create(builder, loc, 1); + Value input = tensor::EmptyOp::create(builder, loc, inputType.getShape(), inputType.getElementType()); + auto concat = spatial::SpatConcatOp::create(builder, loc, outputType, + builder.getI64IntegerAttr(1), ValueRange{input, input}); + auto slice = tensor::ExtractSliceOp::create(builder, loc, concat.getOutput(), + getAsOpFoldResult(builder.getI64ArrayAttr({0, 0, 1, 0})), + getAsOpFoldResult(builder.getI64ArrayAttr({1, 4, 1, 5})), + getAsOpFoldResult(builder.getI64ArrayAttr({1, 1, 1, 1}))); + spatial::SpatHostStoreSyncOp::create(builder, loc, zero, one, zero, zero, slice.getResult()); + spatial::SpatWaitOp::create(builder, loc, one, one); + spatial::SpatYieldOp::create(builder, loc, ValueRange{concat.getOutput()}); + builder.setInsertionPointAfter(compute); + auto reader = spatial::SpatScheduledCompute::create(builder, loc, TypeRange{}, ValueRange{}, ValueRange{}); + reader->setAttr("coreId", builder.getI32IntegerAttr(1)); + reader.getBody().push_back(new Block); + builder.setInsertionPointToStart(&reader.getBody().front()); + zero = arith::ConstantIndexOp::create(builder, loc, 0); + one = arith::ConstantIndexOp::create(builder, loc, 1); + spatial::SpatWaitOp::create(builder, loc, zero, one); + spatial::SpatSyncOp::create(builder, loc, zero, one); + spatial::SpatYieldOp::create(builder, loc, ValueRange{}); + builder.setInsertionPointAfter(reader); + func::ReturnOp::create(builder, loc, compute.getResults()); + TEST_CHECK(succeeded(verify(module))); + auto report = analyzePimSpatialSchedule(module, target, spatialDetails()); + TEST_CHECK(report.getInteger("global_write_requests") == 5); + TEST_CHECK(report.getInteger("global_write_bytes") == 80); + auto *runs = (*report.getArray("request_order"))[0].getAsObject()->getArray("runs"); + TEST_CHECK(runs->size() == 2); + TEST_CHECK((*(*runs)[0].getAsArray())[1].getAsInteger() == 60); + TEST_CHECK((*(*runs)[1].getAsArray())[1].getAsInteger() == 5); + TEST_CHECK((*(*runs)[1].getAsArray())[2].getAsInteger() == 4); + module.erase(); +} + +static void testUnrepresentableSpatial(PimInstructionTiming target) { + MLIRContext context; + context.loadDialect(); + for (bool loopOverflow : {false, true}) { + auto module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module.getBodyRegion()); + auto loc = module.getLoc(); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + auto compute = spatial::SpatScheduledCompute::create(builder, loc, TypeRange{}, ValueRange{}, ValueRange{}); + compute->setAttr("coreId", builder.getI32IntegerAttr(0)); + compute.getBody().push_back(new Block); + builder.setInsertionPointToStart(&compute.getBody().front()); + if (loopOverflow) { + Value lower = arith::ConstantIndexOp::create(builder, loc, std::numeric_limits::min()); + Value upper = arith::ConstantIndexOp::create(builder, loc, std::numeric_limits::max()); + Value step = arith::ConstantIndexOp::create(builder, loc, 1); + scf::ForOp::create(builder, loc, lower, upper, step); + } else { + auto type = RankedTensorType::get({std::numeric_limits::max(), 2}, builder.getF32Type()); + Value input = tensor::EmptyOp::create(builder, loc, type.getShape(), type.getElementType()); + spatial::SpatReluOp::create(builder, loc, type, input); + } + spatial::SpatYieldOp::create(builder, loc, ValueRange{}); + builder.setInsertionPointAfter(compute); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(module))); + auto report = analyzePimSpatialSchedule(module, target, spatialDetails()); + TEST_CHECK(report.getString("status") == "unavailable"); + TEST_CHECK(!report.getBoolean("safe_for_selection").value()); + module.erase(); + } +} + + +static void testMatrixGeometry(PimInstructionTiming target) { + MLIRContext context; + context.loadDialect(); + target.matrixCellBits = 2; + for (bool encoded : {false, true}) + for (int64_t rows : {64, 128, 192}) + for (int64_t columns : {64, 128, 192, 256}) { + auto module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module.getBodyRegion()); + auto loc = module.getLoc(); + auto inputType = RankedTensorType::get({1, rows}, builder.getF32Type()); + auto weightType = RankedTensorType::get({rows, columns}, builder.getF32Type(), + encoded ? builder.getStringAttr("unsupported-layout") : Attribute{}); + auto resultType = RankedTensorType::get({1, columns}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + Value weight = arith::ConstantOp::create(builder, loc, + DenseElementsAttr::get(weightType, builder.getF32FloatAttr(1))); + auto compute = spatial::SpatScheduledCompute::create(builder, loc, TypeRange{}, + ValueRange{weight}, ValueRange{}); + compute->setAttr("coreId", builder.getI32IntegerAttr(0)); + compute.getBody().push_back(new Block); + Value captured = compute.getBody().front().addArgument(weightType, loc); + builder.setInsertionPointToStart(&compute.getBody().front()); + Value input = tensor::EmptyOp::create(builder, loc, inputType.getShape(), inputType.getElementType()); + spatial::SpatVMMOp::create(builder, loc, resultType, captured, input); + spatial::SpatYieldOp::create(builder, loc, ValueRange{}); + builder.setInsertionPointAfter(compute); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(module))); + auto report = analyzePimSpatialSchedule(module, target); + if (encoded || columns % target.matrixColumns || static_cast(rows) > target.matrixRows) + TEST_CHECK(report.getString("reason") == "unavailable physical matrix weight geometry"); + else TEST_CHECK(report.getString("status") != "unavailable"); + module.erase(); + } +} + +static void testOutputMultiplicity(PimInstructionTiming target) { + MLIRContext context; + context.loadDialect(); + for (bool repeatedResult : {false, true}) { + auto module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module.getBodyRegion()); + auto loc = module.getLoc(); + auto type = RankedTensorType::get({1, 64}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "test", builder.getFunctionType({}, {type, type})); + builder.setInsertionPointToStart(function.addEntryBlock()); + SmallVector results(repeatedResult ? 1 : 2, type); + auto compute = spatial::SpatScheduledCompute::create(builder, loc, results, ValueRange{}, ValueRange{}); + compute->setAttr("coreId", builder.getI32IntegerAttr(0)); + compute.getBody().push_back(new Block); + builder.setInsertionPointToStart(&compute.getBody().front()); + Value input = tensor::EmptyOp::create(builder, loc, type.getShape(), type.getElementType()); + Value output = spatial::SpatReluOp::create(builder, loc, type, input); + SmallVector yielded(results.size(), output); + spatial::SpatYieldOp::create(builder, loc, yielded); + builder.setInsertionPointAfter(compute); + func::ReturnOp::create(builder, loc, ValueRange{compute.getResult(0), compute.getResult(repeatedResult ? 0 : 1)}); + TEST_CHECK(succeeded(verify(module))); + auto report = analyzePimSpatialSchedule(module, target, spatialDetails()); + if (repeatedResult) + TEST_CHECK(report.getString("reason") == "unavailable indirect compute output collection"); + else { + TEST_CHECK(report.getInteger("global_write_requests") == 2); + TEST_CHECK(report.getInteger("global_write_bytes") == 128); + auto *runs = (*report.getArray("request_order"))[0].getAsObject()->getArray("runs"); + TEST_CHECK((*(*runs)[0].getAsArray())[2].getAsInteger() == 2); + } + module.erase(); + } +} + +static void testPeriodicStateProof() { + auto target = hardware(1); + target.processorCount = 3; + target.sharedMemoryReadReturnsPayload = false; + target.localMemoryPeriodNs = 1; + target.localMemoryReadLatencyCycles = 20; + target.localMemoryWriteLatencyCycles = 30; + target.transferWidthBytes = target.sharedMemoryWidthBytes = 32; + target.sharedMemoryReadLatencyNs = 40; + target.sharedMemoryWriteLatencyNs = 50; + target.sharedMemoryRequestLatencyNs = target.sharedMemoryResponseLatencyNs = {6, 6, 6}; + target.interProcessorLatencyNs = {0, 3, 4, 3, 0, 3, 4, 3, 0}; + PimExecutionProgram program{ + {{E::Work, 377}, {E::Store, 875}, {E::Work, 545}, {E::Load, 367}, + {E::Work, 333}, {E::Store, 80}, {E::Signal, 0, 1}, {E::Wait, 0, 0, 0, 1}}, + {{E::Work, 924}, {E::Load, 312}, {E::Work, 70}, {E::Store, 434}, + {E::Work, 337}, {E::Load, 275}, {E::Signal, 0, 2}, {E::Wait, 0, 0, 0, 1}}, + {{E::Work, 158}, {E::Store, 565}, {E::Work, 126}, {E::Load, 245}, + {E::Work, 1124}, {E::Store, 367}, {E::Signal, 0, 0}, {E::Wait, 0, 0, 0, 1}}}; + auto baseline = analyzePimExecution(program, target); + PimPeriodicOptions options; + options.verifyPeriodicState = true; + options.workBreakdown = true; + auto verified = analyzePimExecution(program, target, options); + auto *queue = verified.getObject("memory_controller"); + auto *legacy = queue->getObject("legacy_interval_window"); + TEST_CHECK(legacy); + for (const char *field : {"interval_ns", "alternative_tie_interval_ns"}) + TEST_CHECK(legacy->getNumber(field) == baseline.getObject("memory_controller")->getNumber(field)); + auto *state = queue->getObject("periodic_state"); + TEST_CHECK(state && state->getString("status") == "proved"); + TEST_CHECK(state->getInteger("period_generations") == 3); + TEST_CHECK(state->getNumber("interval_ns") == 17485.0 / 3); + TEST_CHECK(queue->getNumber("interval_ns") == state->getNumber("interval_ns")); + TEST_CHECK(legacy->getNumber("interval_ns") != state->getNumber("interval_ns")); + // A fast writer builds signal tokens. Repeated local intervals are insufficient. + program = {{{E::Signal, 0, 1}, {E::Work, 10}}, + {{E::Wait, 0, 0, 0, 1}, {E::Work, 100}}}; + verified = analyzePimExecution(program, target, options); + state = verified.getObject("memory_controller")->getObject("periodic_state"); + TEST_CHECK(state && state->getString("status") == "unproved"); +} + +static void testCompleteSchedulerState() { + auto target = hardware(1); + PimPeriodicOptions options; + options.verifyPeriodicState = true; + auto verify = [&](const PimExecutionProgram &program, bool expected) { + auto baseline = analyzePimExecution(program, target); + auto typed = evaluatePimExecution(program, target, options); + TEST_CHECK(typed.cost.convergenceProved() == expected); + auto detailed = std::move(typed.report); + auto *queue = detailed.getObject("memory_controller"); + TEST_CHECK(queue); + TEST_CHECK(detailed.getString("status") == (typed.cost.available() ? "estimated" : "unavailable")); + if (!typed.cost.available()) TEST_CHECK(detailed.getString("reason") == typed.cost.reason); + if (expected) { + TEST_CHECK(queue->getNumber("interval_ns") == typed.cost.primary); + TEST_CHECK(queue->getNumber("alternative_tie_interval_ns") == typed.cost.alternative); + } else { + for (const char *field : {"interval_ns", "alternative_tie_interval_ns"}) + TEST_CHECK(queue->getNumber(field) == baseline.getObject("memory_controller")->getNumber(field)); + } + size_t policy = 0; + for (const char *field : {"periodic_state", "alternative_periodic_state"}) { + auto *state = queue->getObject(field); + TEST_CHECK(state && (state->getString("status") == "proved") == expected); + const auto &evidence = typed.cost.recurrence[policy++]; + TEST_CHECK(evidence.proved() == expected); + TEST_CHECK(evidence.intervalNs == state->getNumber("interval_ns")); + if (expected) TEST_CHECK(state->getBoolean("complete_scheduler_state") == true); + } + return detailed; + }; + verify({{{E::Work, 0}}}, false); + auto steady = verify({{{E::Work, 100}}}, true); + auto *state = steady.getObject("memory_controller")->getObject("periodic_state"); + TEST_CHECK(state->getInteger("period_generations") == 1); + TEST_CHECK(state->getNumber("interval_ns") == 100); + verify({{{E::Work, uint64_t{1} << 60}}}, true); + // Simultaneous signals, drained token queues, and controller arrivals must + // recur under each deterministic arbitration policy. + verify({{{E::Work, 10}, {E::Signal, 0, 1}, {E::Wait, 0, 0, 0, 1}}, + {{E::Work, 10}, {E::Signal, 0, 0}, {E::Wait, 0, 0, 0, 1}}}, true); + verify({{{E::Load, 65}, {E::Signal, 0, 1}, {E::Wait, 0, 0, 0, 1}}, + {{E::Load, 65}, {E::Signal, 0, 0}, {E::Wait, 0, 0, 0, 1}}}, true); + // Repeated output intervals do not prove uniform progress on every core. + auto unequal = verify({{{E::Work, 7}}, {{E::Work, 11}}}, false); + TEST_CHECK(unequal.getObject("memory_controller")->getString("status") == "estimated"); + // The complete state retains growing dependency-token queues. + auto backlog = verify({{{E::Signal, 0, 1}, {E::Work, 10}}, + {{E::Wait, 0, 0, 0, 1}, {E::Work, 100}}}, false); + TEST_CHECK(backlog.getObject("memory_controller")->getString("status") == "estimated"); + // Both endpoints exist, but neither WAIT can receive its signal. + auto deadlock = verify({{{E::Wait, 0, 0, 0, 1}, {E::Signal, 0, 1}}, + {{E::Wait, 0, 0, 0, 1}, {E::Signal, 0, 0}}}, false); + TEST_CHECK(deadlock.getObject("memory_controller")->getString("status") == "unavailable"); + verify({{{E::Send, 65, 1}, {E::Receive, 65, 1}}, + {{E::Send, 65, 0}, {E::Receive, 65, 0}}}, false); + // An independent active core must not hide an inactive dependency cycle. + verify({{{E::Work, 10}}, {{E::Wait, 0, 0, 0, 1}, {E::Signal, 0, 1}}}, false); + auto overflow = analyzePimExecution({{{E::Work, uint64_t{1} << 62}}}, target, options); + TEST_CHECK(overflow.getString("status") == "unavailable"); +} + +int main() { + testDependencyTokens(); + testCoreAssignments(); + testPeriodicStateProof(); + testCompleteSchedulerState(); + for (int64_t period : {1, 5}) { + testLateRecurrenceAuthority(hardware(period)); + testTypedResult(hardware(period)); + testSharedEngine(hardware(period)); + testAccountingAndOverflow(hardware(period)); + testMatrixGeometry(hardware(period)); + testOutputMultiplicity(hardware(period)); + testHostSlice(hardware(period)); + testCapturedVectorWeights(hardware(period)); + testPublicationAndOutput(hardware(period)); + testUnrepresentableSpatial(hardware(period)); + testReadOnlySpatial(hardware(period), false); + testReadOnlySpatial(hardware(period), true); + } +} diff --git a/test/PIM/PimPipelineSelectionTest.cpp b/test/PIM/PimPipelineSelectionTest.cpp index 6efb227..eef873f 100644 --- a/test/PIM/PimPipelineSelectionTest.cpp +++ b/test/PIM/PimPipelineSelectionTest.cpp @@ -1,11 +1,259 @@ #include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ScheduleEvaluation.hpp" #include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/ScheduledSpatialPasses.hpp" #include "llvm/Support/JSON.h" -#include +#include "TestCheck.hpp" #include +#include "src/Accelerators/PIM/Common/Support/Diagnostics.hpp" +#include "src/Accelerators/PIM/Passes/PIMPasses.h" +#include "src/Accelerators/PIM/Compiler/PimPipelineSelection.hpp" +#include "src/Accelerators/PIM/Compiler/PimCodeGen.hpp" +#include "src/Accelerators/PIM/Compiler/PimCompilerUtils.hpp" +#include "src/Accelerators/PIM/Compiler/PimCompilerOptions.hpp" +#include "src/Accelerators/PIM/Common/Support/FileSystemUtils.hpp" +#include "src/Accelerators/PIM/Dialect/Spatial/SpatialOps.hpp" +#include "src/Accelerators/PIM/Dialect/Spatial/SpatialTargetResources.hpp" +#include "src/Accelerators/PIM/Dialect/Pim/PimOps.hpp" +#include "src/Accelerators/PIM/Dialect/Pim/Passes/Transforms/Bufferization/OpBufferizationInterfaces.hpp" +#include "mlir/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.h" +#include "mlir/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.h" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/MemRef/IR/MemRef.h" +#include "mlir/Dialect/SCF/IR/SCF.h" +#include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/Affine/IR/AffineOps.h" +#include "mlir/Dialect/Bufferization/IR/Bufferization.h" +#include "mlir/Pass/PassManager.h" +#include "mlir/IR/Verifier.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" using namespace onnx_mlir; using namespace onnx_mlir::spatial; +static WholeScheduleCost scheduleCost(double interval) { + WholeScheduleCost cost; + cost.primary = cost.alternative = interval; + cost.selectionMargin = 0.021; + cost.loweringSupport = StaticLoweringSupport::Supported; + for (auto &recurrence : cost.recurrence) + recurrence = {ScheduleRecurrenceEvidence::Proved, 1, interval, {}}; + return cost; +} + +static void testEmptyGraphPreparation() { + using namespace mlir; + MLIRContext context; + context.loadDialect(); + OwningOpRef module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module->getBodyRegion()); + auto function = func::FuncOp::create(builder, module->getLoc(), "main_graph", + builder.getFunctionType({}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + func::ReturnOp::create(builder, module->getLoc()); + SchedulingTarget target; + target.processorCount = target.residentWeightCapacity = 1; + target.interProcessorLatencyNs = {0}; + for (bool collectCandidates : {false, true}) { + auto prepared = prepareSpatialSchedules(*module, target, 1, collectCandidates); + TEST_CHECK(succeeded(prepared) && prepared->candidates.size() == 1); + TEST_CHECK(prepared->candidates.front().dominanceOrderCompute.empty()); + } +} + +static void testSelectedLoweringFailure() { + using namespace mlir; + MLIRContext context(MLIRContext::Threading::DISABLED); + context.loadDialect(); + OwningOpRef module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module->getBodyRegion()); + auto loc = module->getLoc(); + auto type = RankedTensorType::get({1, 536870912}, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "main_graph", builder.getFunctionType({type}, {type})); + builder.setInsertionPointToStart(function.addEntryBlock()); + auto compute = SpatGraphCompute::create(builder, loc, TypeRange{type}, ValueRange{}, + ValueRange{function.getArgument(0)}); + compute.getBody().push_back(new Block); + Value input = compute.getBody().front().addArgument(type, loc); + builder.setInsertionPointToStart(&compute.getBody().front()); + Value result = SpatReluOp::create(builder, loc, type, input); + SpatYieldOp::create(builder, loc, ValueRange{result}); + builder.setInsertionPointAfter(compute); + func::ReturnOp::create(builder, loc, compute.getResults()); + TEST_CHECK(succeeded(verify(*module))); + SchedulingTarget target; + target.processorCount = 2; + target.residentWeightCapacity = 1; + target.synchronizationRegisterCount = 64; + target.interProcessorLatencyNs = {0, 2, 2, 0}; + target.averageInterProcessorLatencyNs = 2; + target.estimateRealizedScheduleCost = [](ModuleOp, llvm::json::Object &) { + TEST_CHECK(false && "single-candidate compilation must skip costs"); + return WholeScheduleCost{}; + }; + SpatialTargetResources resources{{128, 128}, 1, 2, 16}; + std::string diagnostics; + ScopedDiagnosticHandler handler(&context, [&](Diagnostic &diagnostic) { + diagnostics += diagnostic.str(); return success(); + }); + ScopedCompilerOutputSuppression silence; + // Match the compiler's preloaded backend registry before nested pipelines run. + PassManager backend(&context); + addPimLoweringPasses(backend, resources, EmitPimCodegen, false); + DialectRegistry registry; + backend.getDependentDialects(registry); + pim::registerOpBufferizationInterfaces(registry); + arith::registerBufferizableOpInterfaceExternalModels(registry); + bufferization::func_ext::registerBufferizableOpInterfaceExternalModels(registry); + context.appendDialectRegistry(registry); + context.loadAllAvailableDialects(); + for (bool collectCandidates : {false, true}) { + OwningOpRef checkpoint = cast((*module)->clone()); + auto prepared = prepareSpatialSchedules(*checkpoint, target, 2, collectCandidates); + TEST_CHECK(succeeded(prepared) && !prepared->candidates.empty()); + } + PassManager manager(&context); + manager.addPass(createPimPipelineSelectionPass(target, resources, 2, SpatialDataflowExportStage::None)); + TEST_CHECK(failed(manager.run(*module))); + TEST_CHECK(diagnostics.find("backend selection failed") != std::string::npos); + TEST_CHECK(diagnostics.find("outside representable range") != std::string::npos); +} + +static void testResourceRetries() { + using namespace mlir; + auto previousCores = coresCount.getValue(); + auto previousSize = crossbarSize.getValue(); + auto previousCrossbars = crossbarCountInCore.getValue(); + coresCount = 2; crossbarSize = 128; crossbarCountInCore = 1; + MLIRContext context; + context.loadDialect(); + OwningOpRef checkpoint = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(checkpoint->getBodyRegion()); + auto loc = checkpoint->getLoc(); + auto function = func::FuncOp::create(builder, loc, "main_graph", builder.getFunctionType({}, {})); + builder.setInsertionPointToStart(function.addEntryBlock()); + auto core = pim::PimCoreOp::create(builder, loc, ValueRange{}, uint32_t{0}); + core.getBody().push_back(new Block); + builder.setInsertionPointToStart(&core.getBody().front()); + auto allocation = memref::AllocOp::create(builder, loc, + MemRefType::get({int64_t{1} << 31}, builder.getI8Type())); + auto smallType = MemRefType::get({1}, builder.getI8Type()); + auto destination = memref::AllocOp::create(builder, loc, smallType); + Value zero = arith::ConstantIndexOp::create(builder, loc, 0); + pim::PimMemCopyOp::create(builder, loc, smallType, zero, zero, destination, + allocation, builder.getI32IntegerAttr(1)); + pim::PimHaltOp::create(builder, loc); + builder.setInsertionPointAfter(core); + func::ReturnOp::create(builder, loc); + TEST_CHECK(succeeded(verify(*checkpoint))); + std::string before; + llvm::raw_string_ostream stream(before); checkpoint->print(stream, OpPrintingFlags().printGenericOpForm()); + auto original = scheduleCost(100); + auto faster = scheduleCost(90); + std::vector costs{original, faster}; + std::string reason; + SpatialTargetResources resources{{128, 128}, 1, 2, 16}; + PassManager backend(&context); + addPimLoweringPasses(backend, resources, EmitPimCodegen, false); + DialectRegistry registry; + backend.getDependentDialects(registry); + pim::registerOpBufferizationInterfaces(registry); + arith::registerBufferizableOpInterfaceExternalModels(registry); + bufferization::func_ext::registerBufferizableOpInterfaceExternalModels(registry); + context.appendDialectRegistry(registry); + context.loadAllAvailableDialects(); + auto emit = [&](ModuleOp module) { + llvm::SmallString<128> temporary; + TEST_CHECK(!llvm::sys::fs::createUniqueDirectory("raptor-retry-program", temporary)); + std::string output = temporary.str().str(); + ScopedCompilerOutputSuppression silence; + TEST_CHECK(compileToPimCode(module, output) == CompilerSuccess); + auto binary = llvm::MemoryBuffer::getFile(output + "/core_0.pim"); + TEST_CHECK(binary && (*binary)->getBufferSize() > 12); + std::string program = (*binary)->getBuffer().str(); + TEST_CHECK(!llvm::sys::fs::remove_directories(temporary)); + return program; + }; + std::string emitted; + for (bool allFail : {false, true}) { + std::vector attempted; + OwningOpRef winner; + std::vector summaries; + auto selected = lowerSelectedPimCandidate(costs, [&](size_t index) { + attempted.push_back(index); + IRMapping mapping; + OwningOpRef candidate = cast((*checkpoint)->clone(mapping)); + if (!index && !allFail) + cast(mapping.lookup(allocation.getResult()).getDefiningOp()) + .getResult().setType(MemRefType::get({16}, builder.getI8Type())); + llvm::json::Object summary; + auto result = lowerPimPipelineCandidate(*candidate, resources, summary); + if (result == PimCandidateResult::Success) winner = std::move(candidate); + summaries.push_back(std::move(summary)); + return result; + }, reason); + TEST_CHECK(attempted == std::vector({1, 0})); + TEST_CHECK(allFail ? !selected : selected == 0); + TEST_CHECK(summaries.front().getString("status") == "resource-limit"); + TEST_CHECK(summaries.front().getString("reason")->contains("signed int32 address range")); + if (selected) emitted = emit(*winner); + if (allFail) TEST_CHECK(reason.find("all candidate schedules failed") != std::string::npos); + std::string after; + llvm::raw_string_ostream out(after); checkpoint->print(out, OpPrintingFlags().printGenericOpForm()); + TEST_CHECK(before == after); + } + TEST_CHECK(!emitted.empty()); + IRMapping mapping; + OwningOpRef alternative = cast((*checkpoint)->clone(mapping)); + cast(mapping.lookup(allocation.getResult()).getDefiningOp()) + .getResult().setType(MemRefType::get({16}, builder.getI8Type())); + llvm::json::Object summary; + TEST_CHECK(lowerPimPipelineCandidate(*alternative, resources, summary) == PimCandidateResult::Success); + TEST_CHECK(emit(*alternative) == emitted); + size_t attempts = 0; + auto empty = lowerSelectedPimCandidate({}, [&](size_t) { + ++attempts; + return PimCandidateResult::Success; + }, reason); + TEST_CHECK(!empty && attempts == 0 && reason.find("at least one candidate") != std::string::npos); + auto selected = lowerSelectedPimCandidate(costs, [&](size_t) { + ++attempts; + OwningOpRef invalid = cast((*alternative)->clone()); + invalid->walk([&](pim::PimMemCopyOp copy) { + copy->setAttr("size", builder.getI64IntegerAttr(1)); + }); + llvm::json::Object failure; + auto result = lowerPimPipelineCandidate(*invalid, resources, failure); + TEST_CHECK(result == PimCandidateResult::Failure); + TEST_CHECK(failure.getString("status") == "systemic-or-unclassified-error"); + return result; + }, reason); + TEST_CHECK(!selected && attempts == 1); + TEST_CHECK(reason.find("recovery is not justified") != std::string::npos); + for (unsigned unknown = 0; unknown < 4; ++unknown) { + auto invalid = faster; + if (unknown == 0) invalid.primary.reset(); + if (unknown == 1) invalid.recurrence[0].status = ScheduleRecurrenceEvidence::Unproved; + if (unknown == 2) invalid.loweringSupport = StaticLoweringSupport::Unknown; + if (unknown == 3) invalid.loweringSupport = StaticLoweringSupport::Unsupported; + attempts = 0; + std::vector incomplete{original, invalid}; + selected = lowerSelectedPimCandidate(incomplete, [&](size_t) { + ++attempts; + return PimCandidateResult::Success; + }, reason); + TEST_CHECK(!selected && attempts == 0); + } + coresCount = previousCores; crossbarSize = previousSize; crossbarCountInCore = previousCrossbars; +} + int main() { + testEmptyGraphPreparation(); + testSelectedLoweringFailure(); + testResourceRetries(); mlir::MLIRContext context; context.allowUnregisteredDialects(); mlir::OwningOpRef checkpoint = mlir::ModuleOp::create(mlir::UnknownLoc::get(&context)); @@ -25,56 +273,52 @@ int main() { mlir::OwningOpRef trial = mlir::cast((*checkpoint)->clone(mapping)); auto remapped = spatial::remapSpatialSchedule(schedule, mapping); auto copy = remapped.dominanceOrderCompute.front(); - assert(copy.op != op && copy.laneStart == 3 && copy.laneCount == 2); - assert(remapped.computeToCpuMap.lookup(copy) == 7 && remapped.computeToCpuSlotMap.lookup(copy) == 4); - assert(remapped.computeToAestMap.lookup(copy) == 42 && remapped.isLastComputeOfCpu.contains(copy)); - assert(remapped.cpuToLastComputeMap.lookup(7) == copy); + TEST_CHECK(copy.op != op && copy.laneStart == 3 && copy.laneCount == 2); + TEST_CHECK(remapped.computeToCpuMap.lookup(copy) == 7 && remapped.computeToCpuSlotMap.lookup(copy) == 4); + TEST_CHECK(remapped.computeToAestMap.lookup(copy) == 42 && remapped.isLastComputeOfCpu.contains(copy)); + TEST_CHECK(remapped.cpuToLastComputeMap.lookup(7) == copy); copy.op->erase(); - assert(schedule.dominanceOrderCompute.front().op == op && op->getBlock() == checkpoint->getBody()); + TEST_CHECK(schedule.dominanceOrderCompute.front().op == op && op->getBlock() == checkpoint->getBody()); } - auto score = [](double value, std::string id) { return WholeScheduleCost{value, value, {}, id, 0.021}; }; + auto score = [](double value, std::string) { + return scheduleCost(value); + }; std::string reason; auto original = score(100, "0"), faster = score(90, "1"); + for (auto support : {StaticLoweringSupport::Unknown, + StaticLoweringSupport::Supported, StaticLoweringSupport::Unsupported}) { + auto cost = faster; + cost.loweringSupport = support; + TEST_CHECK(cost.available()); + TEST_CHECK(cost.eligible() == (support == StaticLoweringSupport::Supported)); + TEST_CHECK(cost.loweringSupported().has_value() == (support != StaticLoweringSupport::Unknown)); + } + auto incompleteProof = faster; + incompleteProof.recurrence[1].periodGenerations = 0; + TEST_CHECK(incompleteProof.available() && !incompleteProof.eligible()); + incompleteProof.recurrence[1] = {}; + TEST_CHECK(incompleteProof.convergenceProved() == false && !incompleteProof.eligible()); + incompleteProof.recurrence[0] = {}; + TEST_CHECK(!incompleteProof.convergenceProved().has_value()); auto select = [&](std::initializer_list values) { return selectPipelineCandidate(llvm::ArrayRef(values), reason); }; - assert(select({original, faster}) == 1); - assert(select({original, score(97, "2")}) == 0); - assert(select({original, score(105, "2")}) == 0); + TEST_CHECK(select({original, faster}) == 1); + TEST_CHECK(select({original, score(97, "2")}) == 0); + TEST_CHECK(select({original, score(105, "2")}) == 0); auto sensitive = faster; sensitive.alternative = 101; - assert(select({original, sensitive}) == 0); - assert(select({original, faster, score(91, "2")}) == 1); - assert(select({original, faster, score(90, "1")}) == 1); + TEST_CHECK(select({original, sensitive}) == 0); + TEST_CHECK(select({original, faster, score(91, "2")}) == 1); + TEST_CHECK(select({original, faster, score(90, "1")}) == 1); auto unavailable = WholeScheduleCost{}; - assert(select({unavailable, faster}) == 0); - assert(select({original, unavailable}) == 0); + TEST_CHECK(select({unavailable, faster}) == 0); + TEST_CHECK(select({original, unavailable}) == 0); auto unknownUncertainty = faster; - unknownUncertainty.relativeUncertainty.reset(); - assert(select({original, unknownUncertainty}) == 0); + unknownUncertainty.selectionMargin.reset(); + TEST_CHECK(select({original, unknownUncertainty}) == 0); for (double value : {0., -1., std::numeric_limits::infinity(), std::numeric_limits::quiet_NaN()}) - assert(select({original, score(value, "2")}) == 0); - for (int i = 0; i < 8; ++i) assert(select({original, faster}) == 1); + TEST_CHECK(select({original, score(value, "2")}) == 0); + for (int i = 0; i < 8; ++i) TEST_CHECK(select({original, faster}) == 1); - // The generic query accepts target-lowered IR without a target dialect API. - for (double interval : {90., 97.}) { - SchedulingTarget target; - assert(!target.prepareScheduleCost); - target.prepareScheduleCost = [interval](mlir::ModuleOp module, - llvm::json::Object &) -> mlir::FailureOr { - assert(module.getBody()->getOperations().size() == 1); - ScheduleCostQuery query; - query.identity = "static-program"; - query.evaluate = [interval](llvm::json::Object &) { - return WholeScheduleCost{interval, interval, {}, "static-program", 0.021}; - }; - return query; - }; - llvm::json::Object summary; - auto query = target.prepareScheduleCost(*checkpoint, summary); - assert(mlir::succeeded(query) && query->identity == "static-program"); - auto cost = query->evaluate(summary); - assert(cost.eligible() && select({original, cost}) == (interval == 90. ? 1 : 0)); - assert(checkpoint->getBody()->front().getName().getStringRef() == "test.compute"); - } } diff --git a/test/PIM/ProjectionMemoryCostTest.cpp b/test/PIM/ProjectionMemoryCostTest.cpp new file mode 100644 index 0000000..d4fe0fd --- /dev/null +++ b/test/PIM/ProjectionMemoryCostTest.cpp @@ -0,0 +1,99 @@ +#include "TestCheck.hpp" +#include "mlir/Dialect/Func/IR/FuncOps.h" +#include "mlir/IR/Diagnostics.h" +#include "mlir/IR/Verifier.h" +#include "src/Accelerators/PIM/Conversion/ONNXToSpatial/Common/ComputeRegionBuilder.hpp" +#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/Scheduling/ComputeGraph.hpp" +#include "src/Accelerators/PIM/Dialect/Spatial/Passes/Transforms/MergeComputeNodes/ScheduledSpatialPasses.hpp" +#include + +using namespace mlir; +using namespace onnx_mlir; +using namespace onnx_mlir::spatial; + +static void checkProjection(ArrayRef shape, ArrayRef sizes, + ArrayRef strides, bool available, bool valid = true) { + MLIRContext context(MLIRContext::Threading::DISABLED); + context.loadDialect(); + OwningOpRef module = ModuleOp::create(UnknownLoc::get(&context)); + OpBuilder builder(module->getBodyRegion()); + auto loc = module->getLoc(); + auto sourceType = RankedTensorType::get(shape, builder.getF32Type()); + auto resultType = RankedTensorType::get(sizes, builder.getF32Type()); + auto function = func::FuncOp::create(builder, loc, "projection", + builder.getFunctionType({sourceType}, {resultType})); + builder.setInsertionPointToStart(function.addEntryBlock()); + auto compute = createEmptySpatGraphCompute(builder, loc, TypeRange{resultType}, + ValueRange{}, ValueRange{function.getArgument(0)}); + auto attrs = [&](ArrayRef values) { + SmallVector result; + for (int64_t value : values) result.push_back(builder.getIndexAttr(value)); + return result; + }; + auto slice = tensor::ExtractSliceOp::create(builder, loc, resultType, + *compute.getInputArgument(0), attrs(SmallVector(shape.size(), 0)), + attrs(sizes), attrs(strides)); + Value result = SpatReluOp::create(builder, loc, resultType, slice); + SpatYieldOp::create(builder, loc, ValueRange{result}); + builder.setInsertionPointAfter(compute); + func::ReturnOp::create(builder, loc, compute.getResults()); + std::string diagnostics; + ScopedDiagnosticHandler handler(&context, [&](Diagnostic &diagnostic) { + diagnostics += diagnostic.str(); + return success(); + }); + bool verified = succeeded(verify(*module)); + if (verified != valid) std::fprintf(stderr, "%s\n", diagnostics.c_str()); + TEST_CHECK(verified == valid); + if (!valid) return; + diagnostics.clear(); + for (Cost width : {1, 16}) { + diagnostics.clear(); + SchedulingTarget target; + target.processorCount = 1; + target.sharedMemoryWidthBytes = width; + target.sharedMemoryReadLatencyNs = width == 1 ? 7 : 3; + auto graph = buildComputeGraph(function, target); + TEST_CHECK(succeeded(graph)); + if (available) { + TEST_CHECK(graph->baselineMemory.readRequests == 2); + TEST_CHECK(graph->baselineMemory.readBytes == 4); + TEST_CHECK(graph->baselineMemory.serviceNs == (width == 1 ? 28 : 6)); + TEST_CHECK(diagnostics.empty()); + } else { + TEST_CHECK(graph->baselineMemory.readRequests == static_cast(resultType.getNumElements())); + TEST_CHECK(graph->baselineMemory.readBytes == static_cast(resultType.getNumElements())); + TEST_CHECK(graph->baselineMemory.serviceNs == resultType.getNumElements() + * target.sharedMemoryReadLatencyNs); + auto message = diagnostics.find("projection memory cost unavailable"); + TEST_CHECK(message != std::string::npos); + TEST_CHECK(diagnostics.find("projection memory cost unavailable", message + 1) == std::string::npos); + } + // The optional estimate must not make otherwise verified Spatial IR fail + // schedule preparation. Target lowering retains its own legality checks. + target.residentWeightCapacity = 1; + target.interProcessorLatencyNs = {0}; + function.setName("main_graph"); + auto schedules = prepareSpatialSchedules(*module, target, 1, false); + TEST_CHECK(succeeded(schedules) && schedules->candidates.size() == 1); + IRMapping mapping; + OwningOpRef scheduled = cast(module->getOperation()->clone(mapping)); + auto schedule = remapSpatialSchedule(schedules->candidates.front(), mapping); + auto realized = realizeSpatialSchedule(*scheduled, schedule, target, 1, + SpatialDataflowExportStage::None); + if (failed(realized)) std::fprintf(stderr, "%s\n", diagnostics.c_str()); + TEST_CHECK(succeeded(realized)); + TEST_CHECK(succeeded(verify(*scheduled))); + } +} + +int main() { + int64_t limit = std::numeric_limits::max(); + checkProjection({3, 5}, {2, 2}, {1, 1}, true); + // These tensor types and slices are valid Spatial IR. The cost query cannot + // represent their flattened offsets or strides with signed 64-bit values. + checkProjection({2, limit}, {2, 2}, {1, 1}, false); + checkProjection({1, limit, 2}, {1, 1, 1}, {1, 1, 1}, false); + checkProjection({2, limit}, {1, 1}, {2, 1}, false); + checkProjection({3, 5}, {4, 2}, {1, 1}, false, false); +} diff --git a/test/PIM/SpatialSchedulingTargetTest.cpp b/test/PIM/SpatialSchedulingTargetTest.cpp index 9b24d10..39c5973 100644 --- a/test/PIM/SpatialSchedulingTargetTest.cpp +++ b/test/PIM/SpatialSchedulingTargetTest.cpp @@ -1048,6 +1048,7 @@ static void checkPublicationMemoryTraffic(mlir::func::FuncOp function, auto report = getPeriodicScheduleReport(graph, *materialized, *shared, *boundaries, timed); assert(report.getString("status") == "estimated"); assert(report.getBoolean("diagnostic_only") == true); + assert(report.getBoolean("selection_eligible") == false); assert(report.getInteger("boundary_write_bytes") == expected.writeBytes); assert(report.getInteger("boundary_read_bytes") == expected.readBytes); timed.sharedMemoryReadReturnsPayload = false; diff --git a/test/PIM/TestCheck.hpp b/test/PIM/TestCheck.hpp new file mode 100644 index 0000000..60716d0 --- /dev/null +++ b/test/PIM/TestCheck.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include +#include + +#define TEST_CHECK(condition) do { \ + if (!(condition)) { \ + std::fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \ + std::exit(EXIT_FAILURE); \ + } \ +} while (false) diff --git a/test/PIM/test_PimEmission.py b/test/PIM/test_PimEmission.py index 461b65d..41e93fd 100644 --- a/test/PIM/test_PimEmission.py +++ b/test/PIM/test_PimEmission.py @@ -53,9 +53,7 @@ def main(): report = json.loads((out / "reports/pipeline_selection.json").read_text()) assert report["generated_candidates"] == 1 assert report["single_candidate_analysis_skipped"] - assert report["analyzed_candidates"] == report["lowered_candidates"] == 0 - assert report["unavailable_estimates"] == 0 - assert not report["reused_lowered_candidate"] + assert report["lowered_candidates"] == 1 assert "no cost comparison required" in report["reason"] else: assert "intermediate emission retains the original schedule" in output @@ -83,6 +81,10 @@ def main(): capture_output=True, text=True, timeout=30) assert result.returncode == 0, result.stdout + result.stderr assert (out / "reports/pipeline_periodic.json").exists() + report = json.loads((out / "reports/pipeline_periodic.json").read_text()) + assert report["diagnostic_only"] + assert not report["selection_eligible"] + assert not (out / "pim").exists() if __name__ == "__main__": diff --git a/test/PIM/test_PimPeriodicDiagnostic.py b/test/PIM/test_PimPeriodicDiagnostic.py index 5d259a4..963acd6 100644 --- a/test/PIM/test_PimPeriodicDiagnostic.py +++ b/test/PIM/test_PimPeriodicDiagnostic.py @@ -69,8 +69,11 @@ def check_controller(run): controller=True, issue=True) reader = next(core for core in zero['memory_controller']['per_core'] if core['core'] == 0) assert reader['queue_wait_ns'] > 0 # Zero service still queues behind an active request. - bounded = run([[scalar, scalar], [scalar, load(65536), scalar]], controller=True, issue=True) + bounded = json.loads(run([[scalar, scalar], [scalar, load(65536), scalar]], + controller=True, issue=True, valid=False).stdout) + assert bounded['status'] == 'unavailable' assert bounded['memory_controller']['status'] == 'unavailable' + assert 'periodic_state' not in bounded['memory_controller'] shared = [[record(31, a=0, b=2), store(128), record(32, arg=1, a=1), record(32, arg=2, a=1)], [record(32, arg=0, a=0), record(31, a=1, b=1), load(128), record(29, arg=2, length=64)], [record(32, arg=0, a=0), record(31, a=1, b=1), load(128), record(30, arg=1, length=64)]] diff --git a/test/PIM/test_PimSpatialSelection.py b/test/PIM/test_PimSpatialSelection.py new file mode 100644 index 0000000..f2c8065 --- /dev/null +++ b/test/PIM/test_PimSpatialSelection.py @@ -0,0 +1,112 @@ +"""Spatial selection lowers one winner and rejects incomplete comparisons.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile + +import onnx +from onnx import TensorProto, helper + +REPO = Path(__file__).resolve().parents[2] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--compiler", required=True) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="raptor-spatial-selection-") as temporary: + root = Path(temporary) + guard = root / "static-audit.so" + subprocess.run([os.environ.get("CC", "cc"), "-shared", "-fPIC", + str(REPO / "test/PIM/StaticSchedulingAudit.c"), "-o", str(guard)], check=True) + env = dict(os.environ, LD_PRELOAD=str(guard), OMP_NUM_THREADS="1") + iteration = 0 + + def compile(source, pipeline=2, emit="EmitPimCodegen", extra=(), succeeds=True): + nonlocal iteration + output = root / str(iteration) + iteration += 1 + output.mkdir() + command = [args.compiler, str(source), "-o", str(output / "model"), "--maccel=PIM", + f"--{emit}", f"--pipeline={pipeline}", "--core-count=138", "--crossbar-count=128", + "--crossbar-size=128", "--mlir-disable-threading", "--pim-detect-communication-deadlock", + f"--pim-target-config={REPO}/validation/pimsim_configs/pimcomp/arch-b/throughput_config_1000ms.json"] + command.extend(extra) + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=120) + assert (result.returncode == 0) == succeeds, result.stdout + result.stderr + path = output / "reports/pipeline_selection.json" + report = json.loads(path.read_text()) if path.exists() else None + program = {str(p.relative_to(output / "pim")): hashlib.sha256(p.read_bytes()).hexdigest() + for p in (output / "pim").rglob("*") if p.is_file()} + if "--pim-export-spatial-dataflow=all" in extra: + for stage in ("spatial3_scheduled_no_comm", "spatial4_scheduled"): + for suffix in ("nodes.csv", "edges.csv"): + assert (output / f"reports/{stage}.{suffix}").stat().st_size > 0 + assert (output / "dialects/spatial4_scheduled.mlir").exists() + assert (output / "dialects/pim1_buff.mlir").exists() + shutil.rmtree(output) + return report, program, result.stdout + result.stderr + + vgg = REPO / "validation/networks/pimcomp_models/vgg8/vgg8-mnist-reconstructed.onnx" + selected, program, _ = compile(vgg) + count = selected["generated_candidates"] + assert count > 1 and selected["strategy"] == "spatial-queue-aware" + assert selected["lowered_candidates"] == 1 + assert len(selected["backend_attempts"]) == 1 + assert selected["backend_attempts"][0]["candidate"] == selected["selection"] + assert not selected["prediction_error_bound_available"] + assert selected["selection_margin"] == 0.021 + for candidate in selected["candidates"]: + assert candidate["cost_available"] and candidate["convergence_proved"] + assert candidate["lowering_supported"] and candidate["eligible"] + assert "request_order" not in candidate["spatial_cost"] + assert "without_local_copy_work" not in candidate["spatial_cost"] + diagnostic, diagnostic_program, _ = compile(vgg, extra=( + "--pim-export-spatial-dataflow=all", "--pim-dump-ir", "--pim-periodic-diagnostics")) + assert diagnostic_program == program and diagnostic["selection"] == selected["selection"] + + # A single schedule needs no optional timing contract. + source = root / "softmax.onnx" + tensor = helper.make_tensor_value_info("x", TensorProto.FLOAT, [1, 8]) + output = helper.make_tensor_value_info("y", TensorProto.FLOAT, [1, 8]) + model = helper.make_model(helper.make_graph([helper.make_node("Softmax", ["x"], ["y"], axis=1)], + "softmax", [tensor], [output]), opset_imports=[helper.make_opsetid("", 18)]) + model.ir_version = 10 + onnx.save(model, source) + selected, program, _ = compile(source) + assert selected["single_candidate_analysis_skipped"] and selected["lowered_candidates"] == 1 + assert "spatial_cost" not in selected["candidates"][0] + + # An unknown estimate cannot silently exclude any alternative. + model = onnx.load(vgg) + output = model.graph.output[0].name + internal = output + "_before_softmax" + for node in model.graph.node: + for index, value in enumerate(node.output): + if value == output: + node.output[index] = internal + for index, value in enumerate(node.input): + if value == output: + node.input[index] = internal + model.graph.node.append(helper.make_node("Softmax", [internal], [output], axis=-1)) + source = root / "vgg-softmax.onnx" + onnx.save(model, source) + report, program, diagnostics = compile(source, succeeds=False) + assert report["generated_candidates"] > 1 and report["status"] == "incomplete-costs" + assert "cannot compare the complete candidate set" in diagnostics + assert "unsupported abstract operation" in diagnostics + assert not program and "backend_attempts" not in report + # Latency PEFT and earlier emission do not invoke candidate costing. + report, program, _ = compile(vgg, pipeline=1) + assert report is None and program + report, program, stderr = compile(vgg, emit="EmitSpatial") + assert report is None and not program + assert "intermediate emission retains the original schedule" in stderr + + +if __name__ == "__main__": + main() diff --git a/test/PIM/test_QueueSelection.py b/test/PIM/test_QueueSelection.py index 14d6cd5..5f6161c 100644 --- a/test/PIM/test_QueueSelection.py +++ b/test/PIM/test_QueueSelection.py @@ -13,16 +13,7 @@ REPO = Path(__file__).resolve().parents[2] def check_selected_program(out): report = json.loads((out / "reports/pipeline_selection.json").read_text()) - program = json.loads((out / "pim/config.json").read_text()) - digest = hashlib.sha256() - for core in range(program["core_cnt"]): - digest.update((out / f"pim/core_{core}.pim").read_bytes()) - for count in program["array_group_map"][f"core{core}"]: - digest.update(f"{count},".encode()) - digest.update(b";") - assert digest.hexdigest().upper() == report["candidates"][report["selection"]]["identity"], \ - "emitted program differs from the analyzed winner" - assert report["reused_lowered_candidate"] + assert report["strategy"] == "spatial-queue-aware" and report["lowered_candidates"] == 1 for name in ("spatial3_scheduled_no_comm.txt", "memory_report.txt"): assert (out / "reports" / name).stat().st_size > 0, name return report @@ -71,12 +62,13 @@ def main(): assert result.returncode == 0, result.stdout + result.stderr report = json.loads((out / "reports/pipeline_selection.json").read_text()) assert report["static_analysis"] and report["temporary_disk_bytes"] == 0 - assert report["analyzed_candidates"] > 1, report + assert report["generated_candidates"] > 1, report + assert all("identity" not in c and "request_order" not in c["spatial_cost"] + for c in report["candidates"]) check_selected_program(out) - assert not report["reconstructed_spatial_artifacts"] assert not (out / "dialects").exists() assert not list(out.glob("candidate*")) - decisions.append((report["selection"], [(c["identity"], c.get("interval_ns"), c.get("alternative_tie_interval_ns")) + decisions.append((report["selection"], [(c.get("interval_ns"), c.get("alternative_tie_interval_ns")) for c in report["candidates"]])) # Keep the test's disk peak to one selected program. shutil.rmtree(out) @@ -95,8 +87,9 @@ def main(): result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, timeout=90) assert result.returncode == 0, result.stdout + result.stderr report = check_selected_program(out) - assert report["selection"] != 0 and report["reconstructed_spatial_artifacts"] == diagnostics - identity = report["candidates"][report["selection"]]["identity"] + assert report["selection"] != 0 and bool((out / "dialects").exists()) == diagnostics + identity = {str(p.relative_to(out / "pim")): hashlib.sha256(p.read_bytes()).hexdigest() + for p in (out / "pim").rglob("*") if p.is_file()} schedule = (out / "reports/spatial3_scheduled_no_comm.txt").read_bytes() if diagnostics: assert identity == previous_identity and schedule == previous_schedule @@ -123,9 +116,9 @@ def main(): result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, timeout=60) assert result.returncode == 0, result.stdout + result.stderr report = json.loads((out / "reports/pipeline_selection.json").read_text()) - assert report["selection"] == 0 and report["analyzed_candidates"] == 0 - assert report["generated_candidates"] == 1 and report["lowered_candidates"] == 0 - assert "ROB capacity one" in report["reason"] and report["unavailable_estimates"] > 0 + assert report["selection"] == 0 and report["single_candidate_analysis_skipped"] + assert report["generated_candidates"] == 1 and report["lowered_candidates"] == 1 + assert "no cost comparison required" in report["reason"] assert report["temporary_disk_bytes"] == 0 and not list(root.glob("candidate*")) shutil.rmtree(out) out.mkdir() @@ -134,9 +127,9 @@ def main(): result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, timeout=60) assert result.returncode == 0, result.stdout + result.stderr report = json.loads((out / "reports/pipeline_selection.json").read_text()) - assert report["selection"] == 0 and report["analyzed_candidates"] == 0 - assert report["generated_candidates"] == 1 and report["lowered_candidates"] == 0 - assert "synchronization ablation" in report["reason"] + assert report["selection"] == 0 and report["single_candidate_analysis_skipped"] + assert report["generated_candidates"] == 1 and report["lowered_candidates"] == 1 + assert "no cost comparison required" in report["reason"] assert not (root / "executed").exists(), "production compiler executed a diagnostic or simulator" diff --git a/test/PIM/test_ShapeUtils.py b/test/PIM/test_ShapeUtils.py new file mode 100644 index 0000000..5ad566b --- /dev/null +++ b/test/PIM/test_ShapeUtils.py @@ -0,0 +1,11 @@ +"""Overflow must fail explicitly, including builds with disabled assertions.""" +import argparse +import subprocess + +parser = argparse.ArgumentParser() +parser.add_argument("--test", required=True) +args = parser.parse_args() +result = subprocess.run([args.test, "--stride-overflow"], capture_output=True, text=True) +assert result.returncode != 0 +assert "row-major strides are not representable in int64_t" in result.stderr, result.stderr +assert "Assertion" not in result.stderr