fix failing pipeline runs
Validate Operations / validate-operations (push) Failing after 2h17m50s

This commit is contained in:
NiccoloN
2026-09-07 11:05:47 +02:00
parent 62a38bdb76
commit a6ebd047b5
2 changed files with 32 additions and 10 deletions
@@ -638,7 +638,8 @@ static FailureOr<PipelineWorkloadPreparation> preparePipelineWorkloadImpl(
if (tasks.empty()) if (tasks.empty())
continue; continue;
Cost maximumCost = findMaximumPackCost( Cost maximumCost = findMaximumPackCost(
graph, taskCosts, tasks, physicalTarget.residentWeightCapacity, pipelineStages); graph, taskCosts, tasks, physicalTarget.residentWeightCapacity,
pipelineStages);
if (fits(graph, taskCosts, tasks, maximumCost, if (fits(graph, taskCosts, tasks, maximumCost,
physicalTarget.residentWeightCapacity, pipelineStages)) physicalTarget.residentWeightCapacity, pipelineStages))
continue; continue;
@@ -653,7 +654,7 @@ static FailureOr<PipelineWorkloadPreparation> preparePipelineWorkloadImpl(
}); });
std::string candidateError; std::string candidateError;
for (size_t task : candidates) { for (size_t task : candidates) {
auto batch = dyn_cast<SpatGraphComputeBatch>( auto batch = dyn_cast_or_null<SpatGraphComputeBatch>(
graph.nodes[task].instance.op); graph.nodes[task].instance.op);
if (!batch || batch->hasAttr("pipeline.split")) if (!batch || batch->hasAttr("pipeline.split"))
continue; continue;
@@ -664,10 +665,16 @@ static FailureOr<PipelineWorkloadPreparation> preparePipelineWorkloadImpl(
if (!currentError.empty()) if (!currentError.empty())
candidateError = currentError; candidateError = currentError;
} }
error = candidateError.empty() if (llvm::any_of(tasks, [&](size_t task) {
? "pipeline scheduling could not find a splittable graph compute" return graph.nodes[task].residentWeights.size()
: candidateError; > physicalTarget.residentWeightCapacity;
return failure(); })) {
error = candidateError.empty()
? "pipeline scheduling cannot split a compute instance to fit one "
"physical core's crossbars"
: candidateError;
return failure();
}
} }
return PipelineWorkloadPreparation::Ready; return PipelineWorkloadPreparation::Ready;
@@ -1139,7 +1146,7 @@ static bool packPipelineStage(
std::vector<TaskList> &tasksByCpu, const PipelineCoreLayout &layout, std::vector<TaskList> &tasksByCpu, const PipelineCoreLayout &layout,
ArrayRef<size_t> topologicalPosition, size_t stage, ArrayRef<size_t> topologicalPosition, size_t stage,
size_t residentWeightCapacity, const SchedulingTarget &target, size_t residentWeightCapacity, const SchedulingTarget &target,
std::vector<size_t> &taskCpus) { std::vector<size_t> &taskCpus, bool prioritizeWeightReuse = false) {
PipelineStageRange range = layout.getStageRange(stage); PipelineStageRange range = layout.getStageRange(stage);
TaskList tasks; TaskList tasks;
for (size_t cpu = range.begin; cpu < range.begin + range.size; ++cpu) for (size_t cpu = range.begin; cpu < range.begin + range.size; ++cpu)
@@ -1174,8 +1181,11 @@ static bool packPipelineStage(
transferTime, getPeftTransferTime( transferTime, getPeftTransferTime(
transferCost, taskCpus[predecessor], transferCost, taskCpus[predecessor],
candidateCpu, target)); candidateCpu, target));
PackScore score { PackScore score = prioritizeWeightReuse
assemblyLoad, transferTime, schedulingLoad, addedWeights, core}; ? PackScore {
addedWeights, assemblyLoad, transferTime, schedulingLoad, core}
: PackScore {
assemblyLoad, transferTime, schedulingLoad, addedWeights, core};
if (!bestScore || score < *bestScore) { if (!bestScore || score < *bestScore) {
bestCore = core; bestCore = core;
bestScore = score; bestScore = score;
@@ -1238,7 +1248,11 @@ static LogicalResult packPipelineStages(
if (!packPipelineStage( if (!packPipelineStage(
graph, model.schedulingCosts, balanceCosts, tasksByCpu, graph, model.schedulingCosts, balanceCosts, tasksByCpu,
layout, topologicalPosition, stage, residentWeightCapacity, layout, topologicalPosition, stage, residentWeightCapacity,
target, taskCpus)) { target, taskCpus)
&& !packPipelineStage(
graph, model.schedulingCosts, balanceCosts, tasksByCpu,
layout, topologicalPosition, stage, residentWeightCapacity,
target, taskCpus, /*prioritizeWeightReuse=*/true)) {
failedStage = stage; failedStage = stage;
error = "pipeline scheduling cannot pack dependency-monotone stage " error = "pipeline scheduling cannot pack dependency-monotone stage "
+ std::to_string(stage) + std::to_string(stage)
+8
View File
@@ -130,6 +130,14 @@ int main() {
3, 3, 3, 0, 3, 3, 3, 0,
}; };
std::string pipelineError; std::string pipelineError;
ComputeGraph preparationGraph = graph;
ResidentWeight extraWeight;
extraWeight.opaqueLane = graph.nodes.size();
preparationGraph.nodes[2].residentWeights.push_back(extraWeight);
auto preparation = preparePipelineWorkload(
preparationGraph, logicalSchedule, 2, physical, pipelineError);
assert(mlir::succeeded(preparation));
assert(*preparation == PipelineWorkloadPreparation::Ready);
ComputeGraph emptyGraph; ComputeGraph emptyGraph;
MergeScheduleResult emptySchedule; MergeScheduleResult emptySchedule;
emptySchedule.processorCount = 2; emptySchedule.processorCount = 2;