[Buffers] fix non-CFDFC paths having non-zero occupancy

include/dynamatic/Transforms/BufferPlacement/Utils/BufferPlacementMILP.h

@@ -437,6 +437,15 @@ class BufferPlacementMILP : public MILP<BufferPlacement> {
   void addBackedgeConstraints(ArrayRef<CFDFC *> cfdfcs,
                               llvm::MapVector<Value, CPVar> &channelOccupancy);
 
+  /// [FPGA24] Adds path-level occupancy equality constraints for reconvergent
+  /// paths that are entirely within a CFDFC (Paper: Section 5, Equations
+  /// 10-11). We skip paths with forks outside all CFDFC's.
+  void addPathOccupancyEqualityConstraints(
+      ArrayRef<fpga24::ReconvergentPathWithGraph> reconvergentPaths,
+      ArrayRef<CFDFC *> cfdfcs,
+      const llvm::MapVector<CFDFC *, double> &cfdfcIIs,
+      llvm::MapVector<Value, CPVar> &channelOccupancy);
+
   /// [FPGA24] Adds imbalance constraints for reconvergent paths in LP1.
   void addReconvergentPathConstraints(
       ArrayRef<fpga24::ReconvergentPathWithGraph> reconvergentPaths);

lib/Transforms/BufferPlacement/FPGA24Buffers.cpp

@@ -26,6 +26,7 @@
 #include "mlir/IR/Value.h"
 #include "mlir/Support/LLVM.h"
 #include "mlir/Support/LogicalResult.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Debug.h"
@@ -150,51 +151,58 @@ void OccupancyBalancingLP::setup() {
     return;
   }
 
-  /// Target II = 1 for maximum throughput
   double targetII = latencyResult.targetII;
   if (targetII <= 0.0) {
     targetII = 1.0;
   }
-  /// Create variables for each channel
-  /// N_c: Maximal token occupancy on channel c.
+
   /// (Paper: Section 5, Table 2)
+  /// N_c: Maximal token occupancy on channel c.
   this->addOccupancyVars(allChannels, channelOccupancy, MAX_OCCUPANCY);
 
-  /// (Paper: Section 5, Equation 8): N_c >= L_c / II
-  /// We enforce this for the global II, but also for each CFDFC's specific II
-  /// to ensure sufficient buffering in faster loops.
-  /// (Paper: Section 5, Equation 15): Making the required occupancy the
-  /// maximum of all CFDFCs' II.
+  /// Determine which channels belong to at least one CFDFC.
+  llvm::DenseSet<Value> cfdfcChannels;
+  for (CFDFC *cfdfc : cfdfcs)
+    for (Value channel : cfdfc->channels)
+      cfdfcChannels.insert(channel);
 
+  /// (Paper: Section 5, Equation 8): N_c >= L_c / II
+  /// Per-channel lower bound ensuring the pipeline has enough tokens in flight.
+  /// For non-CFDFC channels (executed once), N_c >= 1 suffices.
   llvm::MapVector<Value, double> requiredOccupancy;
-
-  // Initialize with global II constraint
   for (Value channel : allChannels) {
     unsigned latency = latencyResult.channelExtraLatency.lookup(channel);
-    requiredOccupancy[channel] = static_cast<double>(latency) / targetII;
+    if (cfdfcChannels.contains(channel))
+      requiredOccupancy[channel] = static_cast<double>(latency) / targetII;
+    else
+      requiredOccupancy[channel] = (latency > 0) ? 1.0 : 0.0;
   }
 
-  // Update by taking the maximum of the per-CFDFC constraints
+  /// (Paper: Section 5, Equation 15): Take the maximum across per-CFDFC IIs.
   for (CFDFC *cfdfc : cfdfcs) {
     double cfdfcII = latencyResult.cfdfcTargetIIs.lookup(cfdfc);
     if (cfdfcII < 1.0)
       continue;
-
     for (Value channel : cfdfc->channels) {
-      if (requiredOccupancy.count(channel)) {
-        unsigned latency = latencyResult.channelExtraLatency.lookup(channel);
-        double specificOcc = static_cast<double>(latency) / cfdfcII;
-        if (specificOcc > requiredOccupancy[channel]) {
-          requiredOccupancy[channel] = specificOcc;
-        }
-      }
+      if (!requiredOccupancy.count(channel))
+        continue;
+      unsigned latency = latencyResult.channelExtraLatency.lookup(channel);
+      double specificOcc = static_cast<double>(latency) / cfdfcII;
+      if (specificOcc > requiredOccupancy[channel])
+        requiredOccupancy[channel] = specificOcc;
     }
   }
 
   addMinOccupancyConstraints(requiredOccupancy, channelOccupancy);
   addBackedgeConstraints(cfdfcs, channelOccupancy);
   addChannelPropertyOccupancyConstraints(channelOccupancy);
 
+  /// (Paper: Section 5, Equations 10-11): Path occupancy equality.
+  addPathOccupancyEqualityConstraints(reconvergentPaths, cfdfcs,
+                                      latencyResult.cfdfcTargetIIs,
+                                      channelOccupancy);
+
+  /// (Paper: Section 5, Equation 14): Minimize sum(B_c * N_c).
   this->setOccupancyBalancingObjective(channelOccupancy);
 
   markReadyToOptimize();

lib/Transforms/BufferPlacement/Utils/BufferPlacementMILP.cpp

@@ -1350,6 +1350,78 @@ void BufferPlacementMILP::addBackedgeConstraints(
   }
 }
 
+void BufferPlacementMILP::addPathOccupancyEqualityConstraints(
+    ArrayRef<fpga24::ReconvergentPathWithGraph> reconvergentPaths,
+    ArrayRef<CFDFC *> cfdfcs, const llvm::MapVector<CFDFC *, double> &cfdfcIIs,
+    llvm::MapVector<Value, CPVar> &channelOccupancy) {
+
+  for (auto [pathIdx, pathWithGraph] : llvm::enumerate(reconvergentPaths)) {
+    const ReconvergentPath &rp = pathWithGraph.path;
+    const CFGTransitionSequenceSubgraph *graph = pathWithGraph.graph;
+    const DataflowGraphNode &forkNode = graph->nodes[rp.forkNodeId];
+
+    // We skip paths whose fork is outside all CFDFCs.
+    if (forkNode.type == DataflowGraphNode::REGULAR &&
+        llvm::none_of(cfdfcs, [&](CFDFC *cfdfc) {
+          return cfdfc->units.contains(forkNode.op);
+        }))
+      continue;
+
+    std::vector<SimplePath> simplePaths =
+        enumerateSimplePaths(*graph, rp.forkNodeId, rp.joinNodeId, rp.nodeIds);
+    if (simplePaths.size() < 2)
+      continue;
+
+    // For each simple path, the sum of channel occupancies (N_c) and the total
+    // latency of the units.
+    std::vector<std::pair<LinExpr, double>> pathTerms;
+    for (const auto &path : simplePaths) {
+      LinExpr occupancySum;
+      double latencySum = 0.0;
+
+      for (NodeIdType nodeId : path.nodes) {
+        if (nodeId == rp.forkNodeId || nodeId == rp.joinNodeId)
+          continue;
+        const DataflowGraphNode &node = graph->nodes[nodeId];
+        if (node.type != DataflowGraphNode::REGULAR)
+          continue;
+        auto latOrFail =
+            timingDB.getLatency(node.op, SignalType::DATA, targetPeriod);
+        if (succeeded(latOrFail) && *latOrFail > 0.0)
+          latencySum += *latOrFail;
+      }
+
+      for (EdgeIdType edgeId : path.edges) {
+        Value channel = graph->edges[edgeId].channel;
+        if (channelOccupancy.count(channel))
+          occupancySum += channelOccupancy[channel];
+      }
+
+      pathTerms.emplace_back(std::move(occupancySum), latencySum);
+    }
+
+    // (Paper: Section 5, Equation 11): Occupancy(p0) = Occupancy(pi)
+    for (auto [cfdfcIdx, cfdfc] : llvm::enumerate(cfdfcs)) {
+      if (forkNode.type == DataflowGraphNode::REGULAR &&
+          !cfdfc->units.contains(forkNode.op))
+        continue;
+
+      double cfdfcII = std::max(1.0, cfdfcIIs.lookup(cfdfc));
+
+      for (size_t i = 1; i < pathTerms.size(); ++i) {
+        auto &[vars0, lat0] = pathTerms[0];
+        auto &[varsI, latI] = pathTerms[i];
+
+        std::string name =
+            llvm::formatv("pathOccEq_{0}_cfdfc{1}_{2}", pathIdx, cfdfcIdx, i)
+                .str();
+        model->addConstr(vars0 + lat0 / cfdfcII == varsI + latI / cfdfcII,
+                         name);
+      }
+    }
+  }
+}
+
 void BufferPlacementMILP::addReconvergentPathConstraints(
     ArrayRef<fpga24::ReconvergentPathWithGraph> reconvergentPaths) {
   for (size_t pathIdx = 0; pathIdx < reconvergentPaths.size(); ++pathIdx) {