Memory Clustering in the Enumerator

example/optsched-cfg/sched.ini

@@ -8,12 +8,38 @@ USE_OPT_SCHED YES
 # Same options as use optimal scheduling.
 PRINT_SPILL_COUNTS YES
 
+# Print clustering information
+# YES
+# NO
+PRINT_CLUSTER YES
+
 # Use two pass scheduling approach. 
 # First pass minimizes RP and second pass tries to balances RP and ILP.
 # YES
 # NO
 USE_TWO_PASS NO
 
+# Sets a limit for occupancy in the second ILP pass. We will not go below this
+# occupancy when attempting rescheduling.
+# Valid values: 1-10 (whole integers)
+MIN_OCCUPANCY_FOR_RESCHEDULE 3
+
+# Sets the required schedule length improvement percentage for the second ILP
+# pass. If we do not meet this minimum improvement then we do not keep the
+# lower occupancy schedules.
+# Valid values: 1-100 (whole integers)
+MIN_ILP_IMPROVEMENT 10
+
+# Allow enumerator to try to cluster memory operations together in the second
+# pass.
+# YES
+# NO
+CLUSTER_MEMORY_OPS NO
+
+# The weight for clustering. This factor determines the importance of
+# trying to find clusters when enumerating.
+CLUSTER_WEIGHT 1000
+
 # These 3 flags control which schedulers will be used.
 # Each one can be individually toggled. The heuristic
 # list scheduler or ACO must be run before the 
@@ -85,7 +111,8 @@ HEURISTIC LUC_CP_NID
 ENUM_HEURISTIC LUC_CP_NID
 
 # The heuuristic used for the enumerator in the second pass in the two-pass scheduling approach.
-# Same valid values as HEURISTIC.
+# Same valid values as HEURISTIC with an additional heuristic:
+# Cluster: Favor instructions that are part of an active memory clustering group.
 SECOND_PASS_ENUM_HEURISTIC LUC_CP_NID
 
 # The spill cost function to be used. Valid values are:

include/opt-sched/Scheduler/OptSchedDDGWrapperBase.h

@@ -14,9 +14,12 @@ class OptSchedDDGWrapperBase {
 public:
   virtual ~OptSchedDDGWrapperBase() = default;
 
-  virtual void convertSUnits() = 0;
+  virtual void convertSUnits(bool IgnoreRealEdges,
+                             bool IgnoreArtificialEdges) = 0;
 
   virtual void convertRegFiles() = 0;
+
+  virtual int findPossibleClusters(bool IsLoad) = 0;
 };
 
 } // namespace opt_sched

include/opt-sched/Scheduler/bb_spill.h

@@ -12,8 +12,11 @@ Last Update:  Apr. 2011
 #include "opt-sched/Scheduler/OptSchedTarget.h"
 #include "opt-sched/Scheduler/defines.h"
 #include "opt-sched/Scheduler/sched_region.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
 #include <map>
+#include <memory>
 #include <set>
 #include <vector>
 
@@ -32,6 +35,91 @@ class BBWithSpill : public SchedRegion {
 
   InstCount crntSpillCost_;
   InstCount optmlSpillCost_;
+  int CurrentClusterCost;
+
+  /// Used to calculate the dynamic lower bound for clustering.
+  llvm::SmallVector<int, 32> ClusterCount;
+  llvm::SmallVector<int, 32> ClusterInstrRemainderCount;
+  int ClusterGroupCount;
+
+  void computeAndPrintClustering(InstSchedule *Sched) override;
+
+  /// Print the current clusters found so far in the schedule.
+  void printCurrentClustering() override;
+
+  void initForClustering();
+
+  /// Calculate the lower bound cost for memory operations clustering and
+  /// return the lower bound cost. Does not take into account the clustering
+  /// weight.
+  int calculateClusterStaticLB();
+
+  /// Helper function for clustering to save the state of the current cluster.
+  void saveCluster(SchedInstruction *inst);
+
+  /// Helper function for clustering to start a new clustering.
+  void initCluster(SchedInstruction *inst);
+
+  /// Reset the active cluster to 0 (none).
+  void resetActiveCluster(SchedInstruction *inst);
+
+  /// Helper function to restore the previous cluster.
+  void restorePreviousCluster(SchedInstruction *inst);
+
+  bool isClusterFinished();
+
+  int calculateClusterDLB();
+
+  /// Current cluster size
+  unsigned int CurrentClusterSize;
+
+  /// The minimum amount of cluster blocks possible.
+  int MinClusterBlocks;
+
+  /// The minimum amount of cluster blocks + the optimistic expected cluster
+  /// blocks remaining.
+  int DynamicClusterLowerBound;
+
+  /// Current active cluster group.
+  int ClusterActiveGroup;
+
+  int StartCycle;
+
+  /// Data struct to contain information about the previous clusters
+  struct PastClusters {
+    /// The cluster group
+    int ClusterGroup;
+    /// Size of the cluster when it was ended by an instruction not in the
+    /// cluster
+    int ClusterSize;
+
+    /// Instruction number that ended this cluster. Used to check if we should
+    /// restore the cluster state when backtracking.
+    int InstNum;
+
+    int Start;
+
+    /// Contains the actual names of the instructions in the cluster. Only used
+    /// for printing and debugging purposes.
+    std::unique_ptr<llvm::SmallVector<llvm::StringRef, 4>> InstrList;
+
+    /// Constructor for this struct
+    PastClusters(int Cluster, int Size, int Instructions, int CycleStart)
+        : ClusterGroup(Cluster), ClusterSize(Size), InstNum(Instructions),
+          Start(CycleStart) {}
+  };
+
+  /// Vector containing the (n-1) past clusters
+  llvm::SmallVector<std::unique_ptr<PastClusters>, 4> PastClustersList;
+
+  /// Contains the actual names of the instructions in the current cluster.
+  /// Only used for printing and debugging purposes.
+  std::unique_ptr<llvm::SmallVector<llvm::StringRef, 4>> InstrList;
+
+  /// Pointer to the last cluster. This is kept out of the vector to avoid
+  /// having to fetch it every time we compare the current instruction
+  /// number to the one that ended the cluster.
+  std::unique_ptr<PastClusters> LastCluster;
 
   // The target machine
   const OptSchedTarget *OST;
@@ -103,7 +191,8 @@ class BBWithSpill : public SchedRegion {
   void InitForCostCmputtn_();
   InstCount CmputDynmcCost_();
 
-  void UpdateSpillInfoForSchdul_(SchedInstruction *inst, bool trackCnflcts);
+  void UpdateSpillInfoForSchdul_(SchedInstruction *inst, bool trackCnflcts,
+                                 int Start);
   void UpdateSpillInfoForUnSchdul_(SchedInstruction *inst);
   void SetupPhysRegs_();
   void CmputCrntSpillCost_();

include/opt-sched/Scheduler/data_dep.h

@@ -13,6 +13,7 @@ Last Update:  Mar. 2011
 #include "opt-sched/Scheduler/buffers.h"
 #include "opt-sched/Scheduler/defines.h"
 #include "opt-sched/Scheduler/sched_basic_data.h"
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include <memory>
 
@@ -291,7 +292,24 @@ class DataDepGraph : public llvm::opt_sched::OptSchedDDGWrapperBase,
 
   RegisterFile *getRegFiles() { return RegFiles.get(); }
 
+  // Memory clustering helper functions
+  int getMinClusterCount() { return MinClusterCount; }
+  void setMinClusterCount(int Max) { MinClusterCount = Max; }
+  int getTotalInstructionsInAllClusters() {
+    return TotalInstructionsInAllClusters;
+  }
+  void setTotalInstructionsInAllClusters(int Max) {
+    TotalInstructionsInAllClusters = Max;
+  }
+  int getTotalInstructionsInCluster(int Cluster);
+
 protected:
+  int MinClusterCount;
+  int TotalInstructionsInAllClusters;
+  /// Map the cluster block to the total number of instructions found in the
+  /// block
+  MapVector<int, int> MaxInstructionsInEachClusters;
+
   // TODO(max): Get rid of this.
   // Number of basic blocks
   int32_t bscBlkCnt_;
@@ -391,7 +409,7 @@ class DataDepGraph : public llvm::opt_sched::OptSchedDDGWrapperBase,
                                 InstCount fileUB, int blkNum);
   FUNC_RESULT FinishNode_(InstCount nodeNum, InstCount edgeCnt = -1);
   void CreateEdge_(InstCount frmInstNum, InstCount toInstNum, int ltncy,
-                   DependenceType depType);
+                   DependenceType depType, bool IsArtificial = false);
 
   FUNC_RESULT Finish_();
 
@@ -629,6 +647,9 @@ class InstSchedule {
   // The schedule's spill cost according to the cost function used
   InstCount spillCost_;
 
+  // The number of clusters
+  int ClusterSize;
+
   // An array of peak reg pressures for all reg types in the schedule
   InstCount *peakRegPressures_;
 
@@ -676,6 +697,8 @@ class InstSchedule {
   InstCount GetExecCost() const;
   void SetSpillCost(InstCount cost);
   InstCount GetSpillCost() const;
+  void setClusterSize(int size);
+  int getClusterSize() const;
 
   void ResetInstIter();
   InstCount GetFrstInst(InstCount &cycleNum, InstCount &slotNum);
@@ -699,6 +722,7 @@ class InstSchedule {
   void Print(std::ostream &out, char const *const title);
   void PrintInstList(FILE *file, DataDepGraph *dataDepGraph,
                      const char *title) const;
+  void Print(std::ostream &out, char const *const title, DataDepGraph *ddg);
   void PrintRegPressures() const;
   bool Verify(MachineModel *machMdl, DataDepGraph *dataDepGraph);
   void PrintClassData();

include/opt-sched/Scheduler/enumerator.h

@@ -153,6 +153,12 @@ class EnumTreeNode {
   InstCount peakSpillCost_;
   InstCount spillCostSum_;
   InstCount totalCost_ = -1;
+  int ClusterCost;
+  int ClusterActiveGroup;
+  int ClusterAbsorbCount;
+  int ClusterDLB;
+  int ClusterTotalCost = -1;
+  int ClusterBestCost;
   bool totalCostIsActualCost_ = false;
   ReserveSlot *rsrvSlots_;
 
@@ -276,6 +282,18 @@ class EnumTreeNode {
   inline void SetSpillCostSum(InstCount cost);
   inline InstCount GetSpillCostSum();
 
+  inline void setClusteringCost(int Cost);
+  inline int getClusteringCost();
+  inline void setCurClusteringGroup(int Group);
+  inline int getCurClusteringGroup();
+  inline void setClusterAbsorbCount(int Absorb);
+  inline int getClusterAbsorbCount();
+  inline void setClusterLwrBound(int ClusterDynamicLowerBound);
+  inline int getClusterLwrBound();
+  inline void setTotalClusterCost(int Cost);
+  inline int getTotalClusterCost();
+  inline bool isClustering();
+
   bool ChkInstRdndncy(SchedInstruction *inst, int brnchNum);
   bool IsNxtSlotStall();
 
@@ -317,6 +335,9 @@ class Enumerator : public ConstrainedScheduler {
   friend class HistEnumTreeNode;
   friend class CostHistEnumTreeNode;
 
+  // Should we cluster memory operations
+  bool Clustering;
+
   // TODO(max): Document.
   bool isCnstrctd_;
 
@@ -508,7 +529,7 @@ class Enumerator : public ConstrainedScheduler {
              InstCount schedUprBound, int16_t sigHashSize,
              SchedPriorities prirts, Pruning PruningStrategy,
              bool SchedForRPOnly, bool enblStallEnum, Milliseconds timeout,
-             InstCount preFxdInstCnt = 0,
+             bool ClusteringEnabled, InstCount preFxdInstCnt = 0,
              SchedInstruction *preFxdInsts[] = NULL);
   virtual ~Enumerator();
   virtual void Reset();
@@ -525,6 +546,8 @@ class Enumerator : public ConstrainedScheduler {
   // (Chris)
   inline bool IsSchedForRPOnly() const { return SchedForRPOnly_; }
 
+  inline bool isClustering() const { return Clustering; }
+
   // Calculates the schedule and returns it in the passed argument.
   FUNC_RESULT FindSchedule(InstSchedule *sched, SchedRegion *rgn) {
     return RES_ERROR;
@@ -586,6 +609,7 @@ class LengthCostEnumerator : public Enumerator {
   bool WasObjctvMet_();
   bool BackTrack_();
   InstCount GetBestCost_();
+  int GetBestClusterCost_();
   void CreateRootNode_();
 
   // Check if branching from the current node by scheduling this instruction
@@ -603,7 +627,7 @@ class LengthCostEnumerator : public Enumerator {
                        SchedPriorities prirts, Pruning PruningStrategy,
                        bool SchedForRPOnly, bool enblStallEnum,
                        Milliseconds timeout, SPILL_COST_FUNCTION spillCostFunc,
-                       InstCount preFxdInstCnt = 0,
+                       bool ClusteringEnabled, InstCount preFxdInstCnt = 0,
                        SchedInstruction *preFxdInsts[] = NULL);
   virtual ~LengthCostEnumerator();
   void Reset();
@@ -616,6 +640,7 @@ class LengthCostEnumerator : public Enumerator {
   bool IsCostEnum();
   SPILL_COST_FUNCTION GetSpillCostFunc() { return spillCostFunc_; }
   inline InstCount GetBestCost() { return GetBestCost_(); }
+  int getBestClusterCost() { return GetBestClusterCost_(); }
 };
 /*****************************************************************************/
 
@@ -851,6 +876,44 @@ void EnumTreeNode::SetSpillCostSum(InstCount cost) {
 InstCount EnumTreeNode::GetSpillCostSum() { return spillCostSum_; }
 /*****************************************************************************/
 
+void EnumTreeNode::setClusteringCost(int Cost) {
+  assert(Cost >= 0);
+  ClusterCost = Cost;
+}
+
+int EnumTreeNode::getClusteringCost() { return ClusterCost; }
+
+void EnumTreeNode::setCurClusteringGroup(int Group) {
+  assert(Group >= 0);
+  ClusterActiveGroup = Group;
+}
+
+int EnumTreeNode::getCurClusteringGroup() { return ClusterActiveGroup; }
+
+void EnumTreeNode::setClusterAbsorbCount(int Absorb) {
+  assert(Absorb >= 0);
+  ClusterAbsorbCount = Absorb;
+}
+
+int EnumTreeNode::getClusterAbsorbCount() { return ClusterAbsorbCount; }
+
+void EnumTreeNode::setClusterLwrBound(int ClusterDynamicLowerBound) {
+  assert(ClusterDynamicLowerBound >= 0);
+  ClusterDLB = ClusterDynamicLowerBound;
+}
+
+int EnumTreeNode::getClusterLwrBound() { return ClusterDLB; }
+
+void EnumTreeNode::setTotalClusterCost(int Cost) {
+  assert(Cost >= 0);
+  ClusterTotalCost = Cost;
+}
+
+int EnumTreeNode::getTotalClusterCost() { return ClusterTotalCost; }
+
+bool EnumTreeNode::isClustering() { return enumrtr_->isClustering(); }
+/*****************************************************************************/
+
 bool EnumTreeNode::IsNxtCycleNew_() {
   if (enumrtr_->issuRate_ == 1) {
     return true;