Skip to content

Commit

Permalink
Correct-way to fix picture out-of-order when input h.264 stream conta…
Browse files Browse the repository at this point in the history 
…ins H.264 spec incomplied POC information for any stream without B-frame. Related to issue 3312. (cisco#3382)
  • Loading branch information
@xiaotiansf
xiaotiansf authored Dec 17, 2021
1 parent 2830556 commit dc17cee
Show file tree
Hide file tree
Showing 5 changed files with 104 additions and 122 deletions.
1 change: 1 addition & 0 deletions codec/decoder/core/inc/decoder_context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ typedef struct tagPictReoderingStatus {
int32_t iLastGOPRemainPicts;
int32_t iLastWrittenPOC;
int32_t iLargestBufferedPicIndex;
bool bHasBSlice;
} SPictReoderingStatus, *PPictReoderingStatus;

/*
Expand Down
1 change: 1 addition & 0 deletions codec/decoder/core/src/decoder.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ void ResetReorderingPictureBuffers (PPictReoderingStatus pPictReoderingStatus, P
pPictInfo[i].iPOC = IMinInt32;
}
pPictInfo->sBufferInfo.iBufferStatus = 0;
pPictReoderingStatus->bHasBSlice = false;
}
}

Expand Down
3 changes: 2 additions & 1 deletion codec/decoder/plus/inc/welsDecoderExt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,8 @@ class CWelsDecoder : public ISVCDecoder {
int ThreadDecodeFrameInternal (const unsigned char* kpSrc, const int kiSrcLen, unsigned char** ppDst,
SBufferInfo* pDstInfo);
void BufferingReadyPicture (PWelsDecoderContext pCtx, unsigned char** ppDst, SBufferInfo* pDstInfo);
void ReleaseBufferedReadyPicture (PWelsDecoderContext pCtx, unsigned char** ppDst, SBufferInfo* pDstInfo);
void ReleaseBufferedReadyPictureReorder (PWelsDecoderContext pCtx, unsigned char** ppDst, SBufferInfo* pDstInfo, bool isFlush = false);
void ReleaseBufferedReadyPictureNoReorder (PWelsDecoderContext pCtx, unsigned char** ppDst, SBufferInfo* pDstInfo);

void OpenDecoderThreads();
void CloseDecoderThreads();
Expand Down
209 changes: 94 additions & 115 deletions codec/decoder/plus/src/welsDecoderExt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -697,8 +697,15 @@ DECODING_STATE CWelsDecoder::DecodeFrameNoDelay (const unsigned char* kpSrc,
if (m_sReoderingStatus.iNumOfPicts) {
WAIT_EVENT (&m_sBufferingEvent, WELS_DEC_THREAD_WAIT_INFINITE);
RESET_EVENT (&m_sReleaseBufferEvent);
ReleaseBufferedReadyPicture (NULL, ppDst, pDstInfo);
SET_EVENT (&m_sReleaseBufferEvent);
if (!m_sReoderingStatus.bHasBSlice) {
if (m_sReoderingStatus.iNumOfPicts > 1) {
ReleaseBufferedReadyPictureNoReorder (NULL, ppDst, pDstInfo);
}
}
else {
ReleaseBufferedReadyPictureReorder (NULL, ppDst, pDstInfo);
}
SET_EVENT(&m_sReleaseBufferEvent);
}
return (DECODING_STATE)iRet;
}
Expand Down Expand Up @@ -865,7 +872,6 @@ DECODING_STATE CWelsDecoder::DecodeFrame2WithCtx (PWelsDecoderContext pDecContex
pDecContext->dDecTime += (iEnd - iStart) / 1e3;

OutputStatisticsLog (*pDecContext->pDecoderStatistics);

if (GetThreadCount (pDecContext) >= 1) {
WAIT_EVENT (&m_sReleaseBufferEvent, WELS_DEC_THREAD_WAIT_INFINITE);
RESET_EVENT (&m_sBufferingEvent);
Expand Down Expand Up @@ -922,69 +928,13 @@ DECODING_STATE CWelsDecoder::FlushFrame (unsigned char** ppDst,
}
}
if (bEndOfStreamFlag && m_sReoderingStatus.iNumOfPicts > 0) {
m_sReoderingStatus.iMinPOC = IMinInt32;
if (m_bIsBaseline) {
uint32_t uiDecodingTimeStamp = 0;
int32_t firstValidIdx = -1;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sPictInfoList[i].iPOC > IMinInt32) {
uiDecodingTimeStamp = m_sPictInfoList[i].uiDecodingTimeStamp;
m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
m_sReoderingStatus.iPictInfoIndex = i;
firstValidIdx = i;
break;
}
}
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (i == firstValidIdx) continue;
if (m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].uiDecodingTimeStamp < uiDecodingTimeStamp) {
uiDecodingTimeStamp = m_sPictInfoList[i].uiDecodingTimeStamp;
m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
m_sReoderingStatus.iPictInfoIndex = i;
}
}
} else {
int32_t firstValidIdx = -1;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sReoderingStatus.iMinPOC == IMinInt32 && m_sPictInfoList[i].iPOC > IMinInt32) {
m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
m_sReoderingStatus.iPictInfoIndex = i;
firstValidIdx = i;
break;
}
}
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (i == firstValidIdx) continue;
if (m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].iPOC < m_sReoderingStatus.iMinPOC) {
m_sReoderingStatus.iMinPOC = m_sPictInfoList[i].iPOC;
m_sReoderingStatus.iPictInfoIndex = i;
}
}
if (!m_sReoderingStatus.bHasBSlice) {
ReleaseBufferedReadyPictureNoReorder (NULL, ppDst, pDstInfo);
}
}
if (m_sReoderingStatus.iMinPOC > IMinInt32) {
m_sReoderingStatus.iLastWrittenPOC = m_sReoderingStatus.iMinPOC;
#if defined (_DEBUG)
#ifdef _MOTION_VECTOR_DUMP_
fprintf (stderr, "Output POC: #%d uiDecodingTimeStamp=%d\n", m_sReoderingStatus.iLastWrittenPOC,
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].uiDecodingTimeStamp);
#endif
#endif
memcpy (pDstInfo, &m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].sBufferInfo, sizeof (SBufferInfo));
ppDst[0] = pDstInfo->pDst[0];
ppDst[1] = pDstInfo->pDst[1];
ppDst[2] = pDstInfo->pDst[2];
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC = IMinInt32;
PPicBuff pPicBuff = m_iThreadCount <= 1 ? m_pDecThrCtx[0].pCtx->pPicBuff : m_pPicBuff;
if (m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx < pPicBuff->iCapacity) {
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
else {
ReleaseBufferedReadyPictureReorder (NULL, ppDst, pDstInfo, true);
}
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
m_sReoderingStatus.iMinPOC = IMinInt32;
--m_sReoderingStatus.iNumOfPicts;
}

return dsErrorFree;
}

Expand Down Expand Up @@ -1040,6 +990,9 @@ void CWelsDecoder::BufferingReadyPicture (PWelsDecoderContext pCtx, unsigned cha
}
m_bIsBaseline = pCtx->pSps->uiProfileIdc == 66 || pCtx->pSps->uiProfileIdc == 83;
if (!m_bIsBaseline) {
if (pCtx->pSliceHeader->eSliceType == B_SLICE) {
m_sReoderingStatus.bHasBSlice = true;
}
if (m_sReoderingStatus.iNumOfPicts && pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb
&& pCtx->pLastDecPicInfo->pPreviousDecodedPictureInDpb->bNewSeqBegin) {
m_sReoderingStatus.iLastGOPRemainPicts = m_sReoderingStatus.iNumOfPicts;
Expand Down Expand Up @@ -1089,13 +1042,13 @@ void CWelsDecoder::BufferingReadyPicture (PWelsDecoderContext pCtx, unsigned cha
}
}

void CWelsDecoder::ReleaseBufferedReadyPicture (PWelsDecoderContext pCtx, unsigned char** ppDst,
SBufferInfo* pDstInfo) {
void CWelsDecoder::ReleaseBufferedReadyPictureReorder (PWelsDecoderContext pCtx, unsigned char** ppDst,
SBufferInfo* pDstInfo, bool isFlush) {
PPicBuff pPicBuff = pCtx ? pCtx->pPicBuff : m_pPicBuff;
if (pCtx == NULL && m_iThreadCount <= 1) {
pCtx = m_pDecThrCtx[0].pCtx;
}
if (!m_bIsBaseline && m_sReoderingStatus.iLastGOPRemainPicts > 0) {
if (m_sReoderingStatus.iLastGOPRemainPicts > 0) {
m_sReoderingStatus.iMinPOC = IMinInt32;
int32_t firstValidIdx = -1;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
Expand Down Expand Up @@ -1126,8 +1079,10 @@ void CWelsDecoder::ReleaseBufferedReadyPicture (PWelsDecoderContext pCtx, unsign
ppDst[1] = pDstInfo->pDst[1];
ppDst[2] = pDstInfo->pDst[2];
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC = IMinInt32;
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
if (pPicBuff != NULL) {
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
}
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
m_sReoderingStatus.iMinPOC = IMinInt32;
--m_sReoderingStatus.iNumOfPicts;
Expand All @@ -1137,42 +1092,6 @@ void CWelsDecoder::ReleaseBufferedReadyPicture (PWelsDecoderContext pCtx, unsign
}
return;
}
if (m_sReoderingStatus.iNumOfPicts && m_bIsBaseline) {
uint32_t uiDecodingTimeStamp = 0;
int32_t firstValidIdx = -1;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sPictInfoList[i].iPOC > IMinInt32) {
uiDecodingTimeStamp = m_sPictInfoList[i].uiDecodingTimeStamp;
m_sReoderingStatus.iPictInfoIndex = i;
firstValidIdx = i;
break;
}
}
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (i == firstValidIdx) continue;
if (m_sPictInfoList[i].iPOC > IMinInt32 && m_sPictInfoList[i].uiDecodingTimeStamp < uiDecodingTimeStamp) {
uiDecodingTimeStamp = m_sPictInfoList[i].uiDecodingTimeStamp;
m_sReoderingStatus.iPictInfoIndex = i;
}
}
if (uiDecodingTimeStamp > 0) {
#if defined (_DEBUG)
#ifdef _MOTION_VECTOR_DUMP_
fprintf (stderr, "Output POC: #%d uiDecodingTimeStamp=%d\n", m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC,
uiDecodingTimeStamp);
#endif
#endif
memcpy (pDstInfo, &m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].sBufferInfo, sizeof (SBufferInfo));
ppDst[0] = pDstInfo->pDst[0];
ppDst[1] = pDstInfo->pDst[1];
ppDst[2] = pDstInfo->pDst[2];
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC = IMinInt32;
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
--m_sReoderingStatus.iNumOfPicts;
}
return;
}
if (m_sReoderingStatus.iNumOfPicts > 0) {
m_sReoderingStatus.iMinPOC = IMinInt32;
int32_t firstValidIdx = -1;
Expand All @@ -1193,10 +1112,13 @@ void CWelsDecoder::ReleaseBufferedReadyPicture (PWelsDecoderContext pCtx, unsign
}
}
if (m_sReoderingStatus.iMinPOC > IMinInt32) {
int32_t iLastPOC = pCtx != NULL ? pCtx->pSliceHeader->iPicOrderCntLsb : m_sPictInfoList[m_iLastBufferedIdx].iPOC;
bool isReady = (m_sReoderingStatus.iLastWrittenPOC > IMinInt32
&& m_sReoderingStatus.iMinPOC - m_sReoderingStatus.iLastWrittenPOC <= 1)
|| m_sReoderingStatus.iMinPOC < iLastPOC;
bool isReady = true;
if (!isFlush) {
int32_t iLastPOC = pCtx != NULL ? pCtx->pSliceHeader->iPicOrderCntLsb : m_sPictInfoList[m_iLastBufferedIdx].iPOC;
isReady = (m_sReoderingStatus.iLastWrittenPOC > IMinInt32
&& m_sReoderingStatus.iMinPOC - m_sReoderingStatus.iLastWrittenPOC <= 1)
|| m_sReoderingStatus.iMinPOC < iLastPOC;
}
if (isReady) {
m_sReoderingStatus.iLastWrittenPOC = m_sReoderingStatus.iMinPOC;
#if defined (_DEBUG)
Expand All @@ -1210,23 +1132,80 @@ void CWelsDecoder::ReleaseBufferedReadyPicture (PWelsDecoderContext pCtx, unsign
ppDst[1] = pDstInfo->pDst[1];
ppDst[2] = pDstInfo->pDst[2];
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC = IMinInt32;
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
if (pPicBuff != NULL) {
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
}
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
m_sReoderingStatus.iMinPOC = IMinInt32;
--m_sReoderingStatus.iNumOfPicts;
}
}
}

DECODING_STATE CWelsDecoder::ReorderPicturesInDisplay (PWelsDecoderContext pDecContext, unsigned char** ppDst,
SBufferInfo* pDstInfo) {
//if there is no b-frame, no ordering based on values of POCs is necessary.
//The function is added to force to avoid picture reordering because some h.264 streams do not follow H.264 POC specifications.
void CWelsDecoder::ReleaseBufferedReadyPictureNoReorder(PWelsDecoderContext pCtx, unsigned char** ppDst, SBufferInfo* pDstInfo)
{
int32_t firstValidIdx = -1;
uint32_t uiDecodingTimeStamp = 0;
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (m_sPictInfoList[i].iPOC != IMinInt32) {
uiDecodingTimeStamp = m_sPictInfoList[i].uiDecodingTimeStamp;
m_sReoderingStatus.iPictInfoIndex = i;
firstValidIdx = i;
break;
}
}
for (int32_t i = 0; i <= m_sReoderingStatus.iLargestBufferedPicIndex; ++i) {
if (i == firstValidIdx) continue;
if (m_sPictInfoList[i].iPOC != IMinInt32 && m_sPictInfoList[i].uiDecodingTimeStamp < uiDecodingTimeStamp) {
uiDecodingTimeStamp = m_sPictInfoList[i].uiDecodingTimeStamp;
m_sReoderingStatus.iPictInfoIndex = i;
}
}
if (uiDecodingTimeStamp > 0) {
#if defined (_DEBUG)
#ifdef _MOTION_VECTOR_DUMP_
fprintf(stderr, "Output POC: #%d uiDecodingTimeStamp=%d\n", m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC,
uiDecodingTimeStamp);
#endif
#endif
m_sReoderingStatus.iLastWrittenPOC = m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC;
memcpy(pDstInfo, &m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].sBufferInfo, sizeof(SBufferInfo));
ppDst[0] = pDstInfo->pDst[0];
ppDst[1] = pDstInfo->pDst[1];
ppDst[2] = pDstInfo->pDst[2];
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPOC = IMinInt32;
if (pCtx || m_pPicBuff) {
PPicBuff pPicBuff = pCtx ? pCtx->pPicBuff : m_pPicBuff;
PPicture pPic = pPicBuff->ppPic[m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].iPicBuffIdx];
--pPic->iRefCount;
}
if (m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP) {
--m_sReoderingStatus.iLastGOPRemainPicts;
m_sPictInfoList[m_sReoderingStatus.iPictInfoIndex].bLastGOP = false;
}
--m_sReoderingStatus.iNumOfPicts;
}
return;
}

DECODING_STATE CWelsDecoder::ReorderPicturesInDisplay(PWelsDecoderContext pDecContext, unsigned char** ppDst,
SBufferInfo* pDstInfo) {
DECODING_STATE iRet = dsErrorFree;
if ((pDstInfo->iBufferStatus == 1) && (pDecContext->pPps->bEntropyCodingModeFlag)) {
if (pDecContext->pSps != NULL) {
m_bIsBaseline = pDecContext->pSps->uiProfileIdc == 66 || pDecContext->pSps->uiProfileIdc == 83;
if (!m_bIsBaseline) {
BufferingReadyPicture (pDecContext, ppDst, pDstInfo);
ReleaseBufferedReadyPicture (pDecContext, ppDst, pDstInfo);
if (pDstInfo->iBufferStatus == 1) {
BufferingReadyPicture(pDecContext, ppDst, pDstInfo);
if (!m_sReoderingStatus.bHasBSlice && m_sReoderingStatus.iNumOfPicts > 1) {
ReleaseBufferedReadyPictureNoReorder (pDecContext, ppDst, pDstInfo);
}
else {
ReleaseBufferedReadyPictureReorder (pDecContext, ppDst, pDstInfo);
}
}
}
}
return iRet;
Expand Down
12 changes: 6 additions & 6 deletions test/api/decoder_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,18 +127,18 @@ static const FileParam kFileParamArray[] = {
{"res/test_cif_I_CABAC_slice.264", "19121bc67f2b13fb8f030504fc0827e1ac6d0fdb"},
{"res/test_cif_P_CABAC_slice.264", "521bbd0ba2422369b724c7054545cf107a56f959"},
{"res/test_qcif_cabac.264", "587d1d05943f3cd416bf69469975fdee05361e69"},
{"res/test_scalinglist_jm.264", "f690a3af2896a53360215fb5d35016bfd41499b3"},
{"res/test_scalinglist_jm.264", "992a25b4ec98db4a16d61c097e614eb16afe3478"},
{"res/test_vd_1d.264", "5827d2338b79ff82cd091c707823e466197281d3"},
{"res/test_vd_rc.264", "eea02e97bfec89d0418593a8abaaf55d02eaa1ca"},
{"res/Cisco_Men_whisper_640x320_CABAC_Bframe_9.264", "931ba1caf075e7b47445c1f4410ade77a46048f6"},
{"res/Cisco_Men_whisper_640x320_CAVLC_Bframe_9.264", "ff501efe475fda239ab6acab81d958ea06dc61dd"},
{"res/Cisco_Adobe_PDF_sample_a_1024x768_CAVLC_Bframe_9.264", "01d9ddef71ea2e33d4686a38a7726b8d949af31c"},
{"res/Cisco_Men_whisper_640x320_CAVLC_Bframe_9.264", "9819c0345abdd4faedbaf8f8c4dadb7749515e4d"},
{"res/Cisco_Adobe_PDF_sample_a_1024x768_CAVLC_Bframe_9.264", "9d758d9e6f4dead0d7b361f3ddf2ee009d0ea190"},
{"res/VID_1280x544_cabac_temporal_direct.264", "b7f04399f38a90c866f0b518d1dd93c823d5d91f"},
{"res/VID_1280x720_cabac_temporal_direct.264", "dabc1d0d44921a5c72ed2d4fde1d602465249c97"},
{"res/VID_1920x1080_cabac_temporal_direct.264", "6e719adb650cee4ca99a45242685d261257c04cc"},
{"res/VID_1280x544_cavlc_temporal_direct.264", "f1ad1c6b05a6fd5f2fc67db8c3d0dcd0248c16e2"},
{"res/VID_1280x720_cavlc_temporal_direct.264", "d2bc970528b5498a80f56b0e1fce051fa2969205"},
{"res/VID_1920x1080_cavlc_temporal_direct.264", "abfea849e52b51e8013a36f4772d5480e8d49bf5"},
{"res/VID_1280x544_cavlc_temporal_direct.264", "33bfa44b4a3c87fe28354cace1d4b99a03d2967d"},
{"res/VID_1280x720_cavlc_temporal_direct.264", "4face6b5d73a378b6e564a831b49311c230158e4"},
{"res/VID_1920x1080_cavlc_temporal_direct.264", "b35dc99604ea2a1fda5b84d1b9098cb7565dec8f"},
};

INSTANTIATE_TEST_CASE_P (DecodeFile, DecoderOutputTest,
Expand Down

0 comments on commit dc17cee

Please sign in to comment.