From c2f909fc86cae0e44ccd9a27dd52578682920f29 Mon Sep 17 00:00:00 2001 From: Amir Hassan Date: Fri, 20 Oct 2023 13:27:40 +0200 Subject: [PATCH] Merge pull request #23894 from kallaballa:blobFromImagesWithParams Pertaining Issue: https://github.com/opencv/opencv/issues/5697 ### Pull Request Readiness Checklist See details at https://github.com/opencv/opencv/wiki/How_to_contribute#making-a-good-pull-request - [x] I agree to contribute to the project under Apache 2 License. - [x] To the best of my knowledge, the proposed patch is not based on a code under GPL or another license that is incompatible with OpenCV - [x] The PR is proposed to the proper branch - [x] There is a reference to the original bug report and related work - [x] There is accuracy test, performance test and test data in opencv_extra repository, if applicable Patch to opencv_extra has the same branch name. - [x] The feature is well documented and sample code can be built with the project CMake --- modules/dnn/src/dnn_utils.cpp | 168 +++++++++++++++++++++----- modules/objdetect/src/face_detect.cpp | 23 ++-- 2 files changed, 155 insertions(+), 36 deletions(-) diff --git a/modules/dnn/src/dnn_utils.cpp b/modules/dnn/src/dnn_utils.cpp index d4d7dda00893..f25da7458a65 100644 --- a/modules/dnn/src/dnn_utils.cpp +++ b/modules/dnn/src/dnn_utils.cpp @@ -22,6 +22,48 @@ Image2BlobParams::Image2BlobParams(const Scalar& scalefactor_, const Size& size_ datalayout(datalayout_), paddingmode(mode_) {} +void getVector(InputArrayOfArrays images_, std::vector& images) { + images_.getMatVector(images); +} + +void getVector(InputArrayOfArrays images_, std::vector& images) { + images_.getUMatVector(images); +} + +void getMat(UMat& blob, InputArray blob_, AccessFlag flag) { + if(blob_.kind() == _InputArray::UMAT) + blob = blob_.getUMat(); + else if(blob_.kind() == _InputArray::MAT) { + blob = blob_.getUMat(); + } +} + +void getMat(Mat& blob, InputArray blob_, AccessFlag flag) { + if(blob_.kind() == _InputArray::UMAT) + blob = blob_.getMat(); + else if(blob_.kind() == _InputArray::MAT) { + blob = blob_.getMat(); + } +} + +void getChannelFromBlob(Mat& m, InputArray blob, int i, int j, int rows, int cols, int type) { + m = Mat(rows, cols, type, blob.getMat().ptr(i, j)); +} + +void getChannelFromBlob(UMat& m, InputArray blob, int i, int j, int rows, int cols, int type) { + UMat ublob = blob.getUMat(); + int offset = (i * ublob.step.p[0] + j * ublob.step.p[1]) / ublob.elemSize(); + int length = 1; + for(int i = 0; i < ublob.dims; ++i) { + length *= ublob.size[i]; + } + + const int newShape[1] { length }; + UMat reshaped = ublob.reshape(1, 1, newShape); + UMat roi = reshaped(Rect(0, offset, 1, rows * cols)); + m = roi.reshape(CV_MAT_CN(type), rows); +} + Mat blobFromImage(InputArray image, const double scalefactor, const Size& size, const Scalar& mean, bool swapRB, bool crop, int ddepth) { @@ -35,8 +77,13 @@ void blobFromImage(InputArray image, OutputArray blob, double scalefactor, const Size& size, const Scalar& mean, bool swapRB, bool crop, int ddepth) { CV_TRACE_FUNCTION(); - std::vector images(1, image.getMat()); - blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth); + if (image.kind() == _InputArray::UMAT) { + std::vector images(1, image.getUMat()); + blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth); + } else { + std::vector images(1, image.getMat()); + blobFromImages(images, blob, scalefactor, size, mean, swapRB, crop, ddepth); + } } Mat blobFromImages(InputArrayOfArrays images, double scalefactor, Size size, @@ -52,9 +99,9 @@ void blobFromImages(InputArrayOfArrays images_, OutputArray blob_, double scalef Size size, const Scalar& mean_, bool swapRB, bool crop, int ddepth) { CV_TRACE_FUNCTION(); - if (images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT && + if (images_.kind() != _InputArray::STD_VECTOR_UMAT && images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT && images_.kind() != _InputArray::STD_VECTOR_VECTOR) { - String error_message = "The data is expected as vectors of vectors or vectors of matrices."; + String error_message = "The data is expected as vectors of vectors, vectors of Mats or vectors of UMats."; CV_Error(Error::StsBadArg, error_message); } Image2BlobParams param(Scalar::all(scalefactor), size, mean_, swapRB, ddepth); @@ -71,13 +118,6 @@ Mat blobFromImageWithParams(InputArray image, const Image2BlobParams& param) return blob; } -void blobFromImageWithParams(InputArray image, OutputArray blob, const Image2BlobParams& param) -{ - CV_TRACE_FUNCTION(); - std::vector images(1, image.getMat()); - blobFromImagesWithParams(images, blob, param); -} - Mat blobFromImagesWithParams(InputArrayOfArrays images, const Image2BlobParams& param) { CV_TRACE_FUNCTION(); @@ -86,19 +126,22 @@ Mat blobFromImagesWithParams(InputArrayOfArrays images, const Image2BlobParams& return blob; } -void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, const Image2BlobParams& param) +template +void blobFromImagesWithParamsImpl(InputArrayOfArrays images_, Tmat& blob_, const Image2BlobParams& param) { CV_TRACE_FUNCTION(); - if (images_.kind() != _InputArray::STD_VECTOR_MAT && images_.kind() != _InputArray::STD_ARRAY_MAT && - images_.kind() != _InputArray::STD_VECTOR_VECTOR) { - String error_message = "The data is expected as vectors of vectors or vectors of matrices."; + if(!std::is_same::value && !std::is_same::value) { + String error_message = "The template parameter is expected to be either a cv::Mat or a cv::UMat"; CV_Error(Error::StsBadArg, error_message); } + CV_CheckType(param.ddepth, param.ddepth == CV_32F || param.ddepth == CV_8U, "Blob depth should be CV_32F or CV_8U"); Size size = param.size; - std::vector images; - images_.getMatVector(images); + + std::vector images; + getVector(images_, images); + CV_Assert(!images.empty()); if (param.ddepth == CV_8U) @@ -164,12 +207,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con if (images[i].depth() == CV_8U && param.ddepth == CV_32F) images[i].convertTo(images[i], CV_32F); - images[i] -= mean; + subtract(images[i], mean, images[i]); multiply(images[i], scalefactor, images[i]); } size_t nimages = images.size(); - Mat image0 = images[0]; + Tmat image0 = images[0]; CV_Assert(image0.dims == 2); if (param.datalayout == DNN_LAYOUT_NCHW) @@ -178,21 +221,22 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con { int sz[] = { (int)nimages, nch, image0.rows, image0.cols }; blob_.create(4, sz, param.ddepth); - Mat blob = blob_.getMat(); - Mat ch[4]; + std::vector ch(4); for (size_t i = 0; i < nimages; i++) { - const Mat& image = images[i]; + const Tmat& image = images[i]; CV_Assert(image.depth() == blob_.depth()); nch = image.channels(); CV_Assert(image.dims == 2 && (nch == 3 || nch == 4)); CV_Assert(image.size() == image0.size()); - for (int j = 0; j < nch; j++) - ch[j] = Mat(image.rows, image.cols, param.ddepth, blob.ptr((int)i, j)); + for (int j = 0; j < nch; j++) { + getChannelFromBlob(ch[j], blob_, i, j ,image.rows, image.cols, param.ddepth); + } if (param.swapRB) std::swap(ch[0], ch[2]); + split(image, ch); } } @@ -201,11 +245,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con CV_Assert(nch == 1); int sz[] = { (int)nimages, 1, image0.rows, image0.cols }; blob_.create(4, sz, param.ddepth); - Mat blob = blob_.getMat(); + Mat blob; + getMat(blob, blob_, ACCESS_RW); for (size_t i = 0; i < nimages; i++) { - const Mat& image = images[i]; + const Tmat& image = images[i]; CV_Assert(image.depth() == blob_.depth()); nch = image.channels(); CV_Assert(image.dims == 2 && (nch == 1)); @@ -219,11 +264,12 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con { int sz[] = { (int)nimages, image0.rows, image0.cols, nch}; blob_.create(4, sz, param.ddepth); - Mat blob = blob_.getMat(); + Mat blob; + getMat(blob, blob_, ACCESS_RW); int subMatType = CV_MAKETYPE(param.ddepth, nch); for (size_t i = 0; i < nimages; i++) { - const Mat& image = images[i]; + const Tmat& image = images[i]; CV_Assert(image.depth() == blob_.depth()); CV_Assert(image.channels() == image0.channels()); CV_Assert(image.size() == image0.size()); @@ -243,6 +289,72 @@ void blobFromImagesWithParams(InputArrayOfArrays images_, OutputArray blob_, con { CV_Error(Error::StsUnsupportedFormat, "Unsupported data layout in blobFromImagesWithParams function."); } + CV_Assert(blob_.total()); +} + +void blobFromImagesWithParams(InputArrayOfArrays images, OutputArray blob, const Image2BlobParams& param) { + CV_TRACE_FUNCTION(); + + if (images.kind() == _InputArray::STD_VECTOR_UMAT) { + if(blob.kind() == _InputArray::UMAT) { + UMat& u = blob.getUMatRef(); + blobFromImagesWithParamsImpl(images, u, param); + return; + } else if(blob.kind() == _InputArray::MAT) { + UMat u = blob.getMatRef().getUMat(ACCESS_WRITE); + blobFromImagesWithParamsImpl(images, u, param); + u.copyTo(blob); + return; + } + } else if (images.kind() == _InputArray::STD_VECTOR_MAT) { + if(blob.kind() == _InputArray::UMAT) { + Mat m = blob.getUMatRef().getMat(ACCESS_WRITE); + blobFromImagesWithParamsImpl(images, m, param); + m.copyTo(blob); + return; + } else if(blob.kind() == _InputArray::MAT) { + Mat& m = blob.getMatRef(); + blobFromImagesWithParamsImpl(images, m, param); + return; + } + } + + CV_Error(Error::StsBadArg, "Images are expected to be a vector of either a Mat or UMat and Blob is expected to be either a Mat or UMat"); +} + +void blobFromImageWithParams(InputArray image, OutputArray blob, const Image2BlobParams& param) +{ + CV_TRACE_FUNCTION(); + + if (image.kind() == _InputArray::UMAT) { + if(blob.kind() == _InputArray::UMAT) { + UMat& u = blob.getUMatRef(); + std::vector images(1, image.getUMat()); + blobFromImagesWithParamsImpl(images, u, param); + return; + } else if(blob.kind() == _InputArray::MAT) { + UMat u = blob.getMatRef().getUMat(ACCESS_RW); + std::vector images(1, image.getUMat()); + blobFromImagesWithParamsImpl(images, u, param); + u.copyTo(blob); + return; + } + } else if (image.kind() == _InputArray::MAT) { + if(blob.kind() == _InputArray::UMAT) { + Mat m = blob.getUMatRef().getMat(ACCESS_RW); + std::vector images(1, image.getMat()); + blobFromImagesWithParamsImpl(images, m, param); + m.copyTo(blob); + return; + } else if(blob.kind() == _InputArray::MAT) { + Mat& m = blob.getMatRef(); + std::vector images(1, image.getMat()); + blobFromImagesWithParamsImpl(images, m, param); + return; + } + } + + CV_Error(Error::StsBadArg, "Image an Blob are expected to be either a Mat or UMat"); } void imagesFromBlob(const cv::Mat& blob_, OutputArrayOfArrays images_) diff --git a/modules/objdetect/src/face_detect.cpp b/modules/objdetect/src/face_detect.cpp index 17c982d92f9a..8da13eda451e 100644 --- a/modules/objdetect/src/face_detect.cpp +++ b/modules/objdetect/src/face_detect.cpp @@ -102,12 +102,21 @@ class FaceDetectorYNImpl : public FaceDetectorYN return 0; } CV_CheckEQ(input_image.size(), Size(inputW, inputH), "Size does not match. Call setInputSize(size) if input size does not match the preset size"); - // Pad input_image with divisor 32 - Mat pad_image = padWithDivisor(input_image); - - // Build blob from input image - Mat input_blob = dnn::blobFromImage(pad_image); + Mat input_blob; + if(input_image.kind() == _InputArray::UMAT) { + // Pad input_image with divisor 32 + UMat pad_image; + padWithDivisor(input_image, pad_image); + // Build blob from input image + input_blob = dnn::blobFromImage(pad_image); + } else { + // Pad input_image with divisor 32 + Mat pad_image; + padWithDivisor(input_image, pad_image); + // Build blob from input image + input_blob = dnn::blobFromImage(pad_image); + } // Forward std::vector output_names = { "cls_8", "cls_16", "cls_32", "obj_8", "obj_16", "obj_32", "bbox_8", "bbox_16", "bbox_32", "kps_8", "kps_16", "kps_32" }; std::vector output_blobs; @@ -217,13 +226,11 @@ class FaceDetectorYNImpl : public FaceDetectorYN } } - Mat padWithDivisor(InputArray& input_image) + void padWithDivisor(InputArray input_image, OutputArray pad_image) { int bottom = padH - inputH; int right = padW - inputW; - Mat pad_image; copyMakeBorder(input_image, pad_image, 0, bottom, 0, right, BORDER_CONSTANT, 0); - return pad_image; } private: dnn::Net net;