Skip to content

Commit

Permalink
videoio/FFmpeg: added CV_16UC1 read/write support
Browse files Browse the repository at this point in the history
  • Loading branch information
@mshabunin
mshabunin committed Dec 7, 2022
1 parent cb8f1dc commit 6ad2165
Show file tree
Hide file tree
Showing 5 changed files with 175 additions and 23 deletions.
11 changes: 6 additions & 5 deletions modules/videoio/src/cap_ffmpeg.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
#define icvReleaseCapture_FFMPEG_p cvReleaseCapture_FFMPEG
#define icvGrabFrame_FFMPEG_p cvGrabFrame_FFMPEG
#define icvRetrieveFrame_FFMPEG_p cvRetrieveFrame_FFMPEG
#define icvRetrieveFrame2_FFMPEG_p cvRetrieveFrame2_FFMPEG
#define icvSetCaptureProperty_FFMPEG_p cvSetCaptureProperty_FFMPEG
#define icvGetCaptureProperty_FFMPEG_p cvGetCaptureProperty_FFMPEG
#define icvCreateVideoWriter_FFMPEG_p cvCreateVideoWriter_FFMPEG
Expand Down Expand Up @@ -90,7 +91,7 @@ class CvCapture_FFMPEG_proxy CV_FINAL : public cv::IVideoCapture
virtual bool retrieveFrame(int flag, cv::OutputArray frame) CV_OVERRIDE
{
unsigned char* data = 0;
int step=0, width=0, height=0, cn=0;
int step=0, width=0, height=0, cn=0, depth=0;

if (!ffmpegCapture)
return false;
Expand All @@ -103,15 +104,15 @@ class CvCapture_FFMPEG_proxy CV_FINAL : public cv::IVideoCapture
}

if (flag == 0) {
if (!icvRetrieveFrame_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn))
if (!icvRetrieveFrame2_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn, &depth))
return false;
}
else {
if (!ffmpegCapture->retrieveFrame(flag, &data, &step, &width, &height, &cn))
if (!ffmpegCapture->retrieveFrame(flag, &data, &step, &width, &height, &cn, &depth))
return false;
}

cv::Mat tmp(height, width, CV_MAKETYPE(CV_8U, cn), data, step);
cv::Mat tmp(height, width, CV_MAKETYPE(depth, cn), data, step);
applyMetadataRotation(*this, tmp);
tmp.copyTo(frame);

Expand Down Expand Up @@ -165,7 +166,7 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL :
{
if(!ffmpegWriter)
return;
CV_Assert(image.depth() == CV_8U);
CV_Assert(image.depth() == CV_8U || image.depth() == CV_16U);

// if UMat, try GPU to GPU copy using OpenCL extensions
if (image.isUMat()) {
Expand Down
94 changes: 77 additions & 17 deletions modules/videoio/src/cap_ffmpeg_impl.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,6 @@ struct Image_FFMPEG
int step;
int width;
int height;
int cn;
};


Expand Down Expand Up @@ -556,7 +555,7 @@ struct CvCapture_FFMPEG
double getProperty(int) const;
bool setProperty(int, double);
bool grabFrame();
bool retrieveFrame(int flag, unsigned char** data, int* step, int* width, int* height, int* cn);
bool retrieveFrame(int flag, unsigned char** data, int* step, int* width, int* height, int* cn, int* depth);
bool retrieveHWFrame(cv::OutputArray output);
void rotateFrame(cv::Mat &mat) const;

Expand Down Expand Up @@ -614,6 +613,7 @@ struct CvCapture_FFMPEG
bool processRawPacket();
bool rawMode;
bool rawModeInitialized;
bool convertRGB;
AVPacket packet_filtered;
#if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(58, 20, 100)
AVBSFContext* bsfc;
Expand Down Expand Up @@ -666,6 +666,7 @@ void CvCapture_FFMPEG::init()

rawMode = false;
rawModeInitialized = false;
convertRGB = true;
memset(&packet_filtered, 0, sizeof(packet_filtered));
av_init_packet(&packet_filtered);
bsfc = NULL;
Expand Down Expand Up @@ -1042,6 +1043,15 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters&

if (!params.empty())
{
convertRGB = params.get<bool>(CAP_PROP_CONVERT_RGB, true);
if (!convertRGB)
{
CV_LOG_WARNING(NULL, "VIDEOIO/FFMPEG: BGR conversion turned OFF, decoded frame will be "
"returned in its original format. "
"Multiplanar formats are not supported by the backend. "
"Only GRAY8/GRAY16LE pixel formats have been tested. "
"Use at your own risk.");
}
if (params.has(CAP_PROP_FORMAT))
{
int value = params.get<int>(CAP_PROP_FORMAT);
Expand Down Expand Up @@ -1309,7 +1319,6 @@ bool CvCapture_FFMPEG::open(const char* _filename, const VideoCaptureParameters&

frame.width = context->width;
frame.height = context->height;
frame.cn = 3;
frame.step = 0;
frame.data = NULL;
get_rotation_angle();
Expand Down Expand Up @@ -1560,7 +1569,7 @@ bool CvCapture_FFMPEG::grabFrame()
return valid;
}

bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step, int* width, int* height, int* cn)
bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step, int* width, int* height, int* cn, int* depth)
{
if (!video_st || !context)
return false;
Expand All @@ -1581,6 +1590,7 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step,
*width = *step;
*height = 1;
*cn = 1;
*depth = CV_8U;
return ret;
}

Expand All @@ -1600,6 +1610,21 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step,
if (!sw_picture || !sw_picture->data[0])
return false;

CV_LOG_DEBUG(NULL, "Input picture format: " << av_get_pix_fmt_name((AVPixelFormat)sw_picture->format));
const AVPixelFormat result_format = convertRGB ? AV_PIX_FMT_BGR24 : (AVPixelFormat)sw_picture->format;
switch (result_format)
{
case AV_PIX_FMT_BGR24: *depth = CV_8U; *cn = 3; break;
case AV_PIX_FMT_GRAY8: *depth = CV_8U; *cn = 1; break;
case AV_PIX_FMT_GRAY16LE: *depth = CV_16U; *cn = 1; break;
default:
CV_LOG_WARNING(NULL, "Unknown/unsupported picture format: " << av_get_pix_fmt_name(result_format)
<< ", will be treated as 8UC1.");
*depth = CV_8U;
*cn = 1;
break; // TODO: return false?
}

if( img_convert_ctx == NULL ||
frame.width != video_st->CV_FFMPEG_CODEC_FIELD->width ||
frame.height != video_st->CV_FFMPEG_CODEC_FIELD->height ||
Expand All @@ -1614,7 +1639,7 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step,
buffer_width, buffer_height,
(AVPixelFormat)sw_picture->format,
buffer_width, buffer_height,
AV_PIX_FMT_BGR24,
result_format,
SWS_BICUBIC,
NULL, NULL, NULL
);
Expand All @@ -1624,7 +1649,7 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step,

#if USE_AV_FRAME_GET_BUFFER
av_frame_unref(&rgb_picture);
rgb_picture.format = AV_PIX_FMT_BGR24;
rgb_picture.format = result_format;
rgb_picture.width = buffer_width;
rgb_picture.height = buffer_height;
if (0 != av_frame_get_buffer(&rgb_picture, 32))
Expand All @@ -1636,14 +1661,13 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step,
int aligns[AV_NUM_DATA_POINTERS];
avcodec_align_dimensions2(video_st->codec, &buffer_width, &buffer_height, aligns);
rgb_picture.data[0] = (uint8_t*)realloc(rgb_picture.data[0],
_opencv_ffmpeg_av_image_get_buffer_size( AV_PIX_FMT_BGR24,
_opencv_ffmpeg_av_image_get_buffer_size( result_format,
buffer_width, buffer_height ));
_opencv_ffmpeg_av_image_fill_arrays(&rgb_picture, rgb_picture.data[0],
AV_PIX_FMT_BGR24, buffer_width, buffer_height );
result_format, buffer_width, buffer_height );
#endif
frame.width = video_st->CV_FFMPEG_CODEC_FIELD->width;
frame.height = video_st->CV_FFMPEG_CODEC_FIELD->height;
frame.cn = 3;
frame.data = rgb_picture.data[0];
frame.step = rgb_picture.linesize[0];
}
Expand All @@ -1661,7 +1685,6 @@ bool CvCapture_FFMPEG::retrieveFrame(int flag, unsigned char** data, int* step,
*step = frame.step;
*width = frame.width;
*height = frame.height;
*cn = frame.cn;

#if USE_AV_HW_CODECS
if (sw_picture != picture)
Expand Down Expand Up @@ -1766,6 +1789,8 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const
if (rawMode)
return -1;
break;
case CAP_PROP_CONVERT_RGB:
return convertRGB;
case CAP_PROP_LRF_HAS_KEY_FRAME: {
const AVPacket& p = bsfc ? packet_filtered : packet;
return ((p.flags & AV_PKT_FLAG_KEY) != 0) ? 1 : 0;
Expand Down Expand Up @@ -1984,6 +2009,9 @@ bool CvCapture_FFMPEG::setProperty( int property_id, double value )
if (value == -1)
return setRaw();
return false;
case CAP_PROP_CONVERT_RGB:
convertRGB = (value != 0);
return true;
case CAP_PROP_ORIENTATION_AUTO:
#if LIBAVUTIL_BUILD >= CALC_FFMPEG_VERSION(52, 94, 100)
rotation_auto = value != 0 ? true : false;
Expand Down Expand Up @@ -2218,7 +2246,6 @@ static AVCodecContext * icv_configure_video_stream_FFMPEG(AVFormatContext *oc,

c->gop_size = 12; /* emit one intra frame every twelve frames at most */
c->pix_fmt = pixel_format;

if (c->codec_id == CV_CODEC(CODEC_ID_MPEG2VIDEO)) {
c->max_b_frames = 2;
}
Expand Down Expand Up @@ -2351,12 +2378,15 @@ bool CvVideoWriter_FFMPEG::writeFrame( const unsigned char* data, int step, int
return false;
}
}
else if (input_pix_fmt == AV_PIX_FMT_GRAY8) {
else if (input_pix_fmt == AV_PIX_FMT_GRAY8 || input_pix_fmt == AV_PIX_FMT_GRAY16LE) {
if (cn != 1) {
return false;
}
}
else {
CV_LOG_WARNING(NULL, "Input data does not match selected pixel format: "
<< av_get_pix_fmt_name(input_pix_fmt)
<< ", number of channels: " << cn);
CV_Assert(false);
}

Expand Down Expand Up @@ -2656,6 +2686,14 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
close();

const bool is_color = params.get(VIDEOWRITER_PROP_IS_COLOR, true);
const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U);
const bool is_supported = depth == CV_8U || (depth == CV_16U && !is_color);
if (!is_supported)
{
CV_LOG_WARNING(NULL, "Unsupported depth/isColor combination is selected, "
"only CV_8UC1/CV_8UC3/CV_16UC1 are supported.");
return false;
}
if (params.has(VIDEOWRITER_PROP_HW_ACCELERATION))
{
va_type = params.get<VideoAccelerationType>(VIDEOWRITER_PROP_HW_ACCELERATION, VIDEO_ACCELERATION_NONE);
Expand Down Expand Up @@ -2713,12 +2751,28 @@ bool CvVideoWriter_FFMPEG::open( const char * filename, int fourcc,
return false;

/* determine optimal pixel format */
if (is_color) {
input_pix_fmt = AV_PIX_FMT_BGR24;
if (is_color)
{
switch (depth)
{
case CV_8U: input_pix_fmt = AV_PIX_FMT_BGR24; break;
default:
CV_LOG_WARNING(NULL, "Unsupported input depth for color image: " << depth);
return false;
}
}
else {
input_pix_fmt = AV_PIX_FMT_GRAY8;
else
{
switch (depth)
{
case CV_8U: input_pix_fmt = AV_PIX_FMT_GRAY8; break;
case CV_16U: input_pix_fmt = AV_PIX_FMT_GRAY16LE; break;
default:
CV_LOG_WARNING(NULL, "Unsupported input depth for grayscale image: " << depth);
return false;
}
}
CV_LOG_DEBUG(NULL, "Selected pixel format: " << av_get_pix_fmt_name(input_pix_fmt));

if (fourcc == -1)
{
Expand Down Expand Up @@ -3158,7 +3212,13 @@ int cvGrabFrame_FFMPEG(CvCapture_FFMPEG* capture)

int cvRetrieveFrame_FFMPEG(CvCapture_FFMPEG* capture, unsigned char** data, int* step, int* width, int* height, int* cn)
{
return capture->retrieveFrame(0, data, step, width, height, cn);
int depth = CV_8U;
return cvRetrieveFrame2_FFMPEG(capture, data, step, width, height, cn, &depth);
}

int cvRetrieveFrame2_FFMPEG(CvCapture_FFMPEG* capture, unsigned char** data, int* step, int* width, int* height, int* cn, int* depth)
{
return capture->retrieveFrame(0, data, step, width, height, cn, depth);
}

static CvVideoWriter_FFMPEG* cvCreateVideoWriterWithParams_FFMPEG( const char* filename, int fourcc, double fps,
Expand Down
2 changes: 2 additions & 0 deletions modules/videoio/src/cap_ffmpeg_legacy_api.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ OPENCV_FFMPEG_API double cvGetCaptureProperty_FFMPEG(struct CvCapture_FFMPEG* ca
OPENCV_FFMPEG_API int cvGrabFrame_FFMPEG(struct CvCapture_FFMPEG* cap);
OPENCV_FFMPEG_API int cvRetrieveFrame_FFMPEG(struct CvCapture_FFMPEG* capture, unsigned char** data,
int* step, int* width, int* height, int* cn);
OPENCV_FFMPEG_API int cvRetrieveFrame2_FFMPEG(struct CvCapture_FFMPEG* capture, unsigned char** data,
int* step, int* width, int* height, int* cn, int* depth);
OPENCV_FFMPEG_API void cvReleaseCapture_FFMPEG(struct CvCapture_FFMPEG** cap);

OPENCV_FFMPEG_API struct CvVideoWriter_FFMPEG* cvCreateVideoWriter_FFMPEG(const char* filename,
Expand Down
Loading

0 comments on commit 6ad2165

Please sign in to comment.