@@ -544,6 +544,12 @@ struct BBInfo {
544544 const std::string infoString () const {
545545 return (Twine (" Index=" Twine (Index)).str ();
546546 }
547+
548+ // Empty function -- only applicable to UseBBInfo.
549+ void addOutEdge (PGOEdge *E LLVM_ATTRIBUTE_UNUSED) {}
550+
551+ // Empty function -- only applicable to UseBBInfo.
552+ void addInEdge (PGOEdge *E LLVM_ATTRIBUTE_UNUSED) {}
547553};
548554
549555// This class implements the CFG edges. Note the CFG can be a multi-graph.
@@ -748,7 +754,7 @@ void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
748754}
749755
750756// Collect all the BBs that will be instruments and return them in
751- // InstrumentBBs.
757+ // InstrumentBBs and setup InEdges/OutEdge for UseBBInfo .
752758template <class Edge , class BBInfo >
753759void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
754760 std::vector<BasicBlock *> &InstrumentBBs) {
@@ -763,6 +769,18 @@ void FuncPGOInstrumentation<Edge, BBInfo>::getInstrumentBBs(
763769 if (InstrBB)
764770 InstrumentBBs.push_back (InstrBB);
765771 }
772+
773+ // Set up InEdges/OutEdges for all BBs.
774+ for (auto &E : MST.AllEdges ) {
775+ if (E->Removed )
776+ continue ;
777+ const BasicBlock *SrcBB = E->SrcBB ;
778+ const BasicBlock *DestBB = E->DestBB ;
779+ BBInfo &SrcInfo = getBBInfo (SrcBB);
780+ BBInfo &DestInfo = getBBInfo (DestBB);
781+ SrcInfo.addOutEdge (E.get ());
782+ DestInfo.addInEdge (E.get ());
783+ }
766784}
767785
768786// Given a CFG E to be instrumented, find which BB to place the instrumented
@@ -780,34 +798,45 @@ BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
780798 if (DestBB == nullptr )
781799 return SrcBB;
782800
801+ auto canInstrument = [this ](BasicBlock *BB) -> BasicBlock * {
802+ // There are basic blocks (such as catchswitch) cannot be instrumented.
803+ // If the returned first insertion point is the end of BB, skip this BB.
804+ if (BB->getFirstInsertionPt () == BB->end ()) {
805+ LLVM_DEBUG (dbgs () << " Cannot instrument BB index=" getBBInfo (BB).Index
806+ << " \n "
807+ return nullptr ;
808+ }
809+ return BB;
810+ };
811+
783812 // Instrument the SrcBB if it has a single successor,
784813 // otherwise, the DestBB if this is not a critical edge.
785814 Instruction *TI = SrcBB->getTerminator ();
786815 if (TI->getNumSuccessors () <= 1 )
787- return SrcBB;
816+ return canInstrument ( SrcBB) ;
788817 if (!E->IsCritical )
789- return DestBB;
818+ return canInstrument ( DestBB) ;
790819
791- // For a critical edge, we have to split. Instrument the newly
792- // created BB.
793- IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
794- LLVM_DEBUG (dbgs () << " Split critical edge: " getBBInfo (SrcBB).Index
795- << " --> " getBBInfo (DestBB).Index << " \n "
796820 unsigned SuccNum = GetSuccessorNumber (SrcBB, DestBB);
797821 BasicBlock *InstrBB = SplitCriticalEdge (TI, SuccNum);
798822 if (!InstrBB) {
799823 LLVM_DEBUG (
800824 dbgs () << " Fail to split critical edge: not instrument this edge.\n "
801825 return nullptr ;
802826 }
827+ // For a critical edge, we have to split. Instrument the newly
828+ // created BB.
829+ IsCS ? NumOfCSPGOSplit++ : NumOfPGOSplit++;
830+ LLVM_DEBUG (dbgs () << " Split critical edge: " getBBInfo (SrcBB).Index
831+ << " --> " getBBInfo (DestBB).Index << " \n "
803832 // Need to add two new edges. First one: Add new edge of SrcBB->InstrBB.
804833 MST.addEdge (SrcBB, InstrBB, 0 );
805834 // Second one: Add new edge of InstrBB->DestBB.
806835 Edge &NewEdge1 = MST.addEdge (InstrBB, DestBB, 0 );
807836 NewEdge1.InMST = true ;
808837 E->Removed = true ;
809838
810- return InstrBB;
839+ return canInstrument ( InstrBB) ;
811840}
812841
813842// Visit all edge and instrument the edges not in MST, and do value profiling.
@@ -925,6 +954,18 @@ struct UseBBInfo : public BBInfo {
925954 return BBInfo::infoString ();
926955 return (Twine (BBInfo::infoString ()) + " Count=" Twine (CountValue)).str ();
927956 }
957+
958+ // Add an OutEdge and update the edge count.
959+ void addOutEdge (PGOUseEdge *E) {
960+ OutEdges.push_back (E);
961+ UnknownCountOutEdge++;
962+ }
963+
964+ // Add an InEdge and update the edge count.
965+ void addInEdge (PGOUseEdge *E) {
966+ InEdges.push_back (E);
967+ UnknownCountInEdge++;
968+ }
928969};
929970
930971} // end anonymous namespace
@@ -1069,24 +1110,50 @@ bool PGOUseFunc::setInstrumentedCounts(
10691110 if (NumCounters != CountFromProfile.size ()) {
10701111 return false ;
10711112 }
1113+ // Set the profile count to the Instrumented BBs.
10721114 uint32_t I = 0 ;
10731115 for (BasicBlock *InstrBB : InstrumentBBs) {
10741116 uint64_t CountValue = CountFromProfile[I++];
10751117 UseBBInfo &Info = getBBInfo (InstrBB);
10761118 Info.setBBInfoCount (CountValue);
1077- // If only one in-edge, the edge profile count should be the same as BB
1078- // profile count.
1079- if (Info.InEdges .size () == 1 ) {
1080- Info.InEdges [0 ]->setEdgeCount (CountValue);
1081- }
1119+ }
1120+ ProfileCountSize = CountFromProfile.size ();
1121+ CountPosition = I;
1122+
1123+ // Set the edge count and update the count of unknown edges for BBs.
1124+ auto setEdgeCount = [this ](PGOUseEdge *E, uint64_t Value) -> void {
1125+ E->setEdgeCount (Value);
1126+ this ->getBBInfo (E->SrcBB ).UnknownCountOutEdge --;
1127+ this ->getBBInfo (E->DestBB ).UnknownCountInEdge --;
1128+ };
1129+
1130+ // Set the profile count the Instrumented edges. There are BBs that not in
1131+ // MST but not instrumented. Need to set the edge count value so that we can
1132+ // populate the profile counts later.
1133+ for (auto &E : FuncInfo.MST .AllEdges ) {
1134+ if (E->Removed || E->InMST )
1135+ continue ;
1136+ const BasicBlock *SrcBB = E->SrcBB ;
1137+ UseBBInfo &SrcInfo = getBBInfo (SrcBB);
1138+
10821139 // If only one out-edge, the edge profile count should be the same as BB
10831140 // profile count.
1084- if (Info.OutEdges .size () == 1 ) {
1085- Info.OutEdges [0 ]->setEdgeCount (CountValue);
1141+ if (SrcInfo.CountValid && SrcInfo.OutEdges .size () == 1 )
1142+ setEdgeCount (E.get (), SrcInfo.CountValue );
1143+ else {
1144+ const BasicBlock *DestBB = E->DestBB ;
1145+ UseBBInfo &DestInfo = getBBInfo (DestBB);
1146+ // If only one in-edge, the edge profile count should be the same as BB
1147+ // profile count.
1148+ if (DestInfo.CountValid && DestInfo.InEdges .size () == 1 )
1149+ setEdgeCount (E.get (), DestInfo.CountValue );
10861150 }
1151+ if (E->CountValid )
1152+ continue ;
1153+ // E's count should have been set from profile. If not, this meenas E skips
1154+ // the instrumentation. We set the count to 0.
1155+ setEdgeCount (E.get (), 0 );
10871156 }
1088- ProfileCountSize = CountFromProfile.size ();
1089- CountPosition = I;
10901157 return true ;
10911158}
10921159
@@ -1180,26 +1247,6 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros)
11801247// Populate the counters from instrumented BBs to all BBs.
11811248// In the end of this operation, all BBs should have a valid count value.
11821249void PGOUseFunc::populateCounters () {
1183- // First set up Count variable for all BBs.
1184- for (auto &E : FuncInfo.MST .AllEdges ) {
1185- if (E->Removed )
1186- continue ;
1187-
1188- const BasicBlock *SrcBB = E->SrcBB ;
1189- const BasicBlock *DestBB = E->DestBB ;
1190- UseBBInfo &SrcInfo = getBBInfo (SrcBB);
1191- UseBBInfo &DestInfo = getBBInfo (DestBB);
1192- SrcInfo.OutEdges .push_back (E.get ());
1193- DestInfo.InEdges .push_back (E.get ());
1194- SrcInfo.UnknownCountOutEdge ++;
1195- DestInfo.UnknownCountInEdge ++;
1196-
1197- if (!E->CountValid )
1198- continue ;
1199- DestInfo.UnknownCountInEdge --;
1200- SrcInfo.UnknownCountOutEdge --;
1201- }
1202-
12031250 bool Changes = true ;
12041251 unsigned NumPasses = 0 ;
12051252 while (Changes) {
0 commit comments