RGB to Gray ConvertColor Preprocessor (openvinotoolkit#13855)

* Add RGB to Gray ConvertColor Preprocessor

* Revert some changes

* Additional updates

* Polish RGBToGray implementation

Move Conv computations to FP32 and add Round for INT cases

* Extend tests - add comparison with OpenCV

* ClangFormat

* Return commented code

* Make `pre_post_process` tests parametrized

* Add shape calculation for RGB ColorFormat. Make RGB->GRAY 4D only

* Update ov_core_unit_tests

* Update ov_template_func_tests

* Clang Formatting

* Fix python preprocess tests

* Fix warning: truncation from 'double' to 'float'
of RGB->GRAY weights

* Fix flake8 issues

* Fix `convert_color_asserts` test

* Treat GRAY as NHWC

* Resolve review issues with tests

* Fix Py preprocess tests after rebase

* Clang Format

* Add case if C < 0

* Add reference link to RGB coefficients

* Add doxygen documentation for ColorFormat enum

src/bindings/python/tests/test_graph/test_preprocess.py

@@ -199,7 +199,7 @@ def custom_postprocess(output: Output):
 
 
 def test_graph_preprocess_spatial_static_shape():
-    shape = [2, 2, 2]
+    shape = [3, 2, 2]
     parameter_a = ops.parameter(shape, dtype=np.int32, name="A")
     model = parameter_a
     function = Model(model, [parameter_a], "TestFunction")
@@ -210,7 +210,7 @@ def test_graph_preprocess_spatial_static_shape():
     ppp = PrePostProcessor(function)
     inp = ppp.input()
     inp.tensor().set_layout(layout).set_spatial_static_shape(2, 2).set_color_format(color_format)
-    inp.preprocess().convert_element_type(Type.f32).mean([1., 2.])
+    inp.preprocess().convert_element_type(Type.f32).mean([1., 2., 3.])
     inp.model().set_layout(layout)
     out = ppp.output()
     out.tensor().set_layout(layout).set_element_type(Type.f32)
@@ -227,7 +227,7 @@ def test_graph_preprocess_spatial_static_shape():
     ]
     assert len(model_operators) == 7
     assert function.get_output_size() == 1
-    assert list(function.get_output_shape(0)) == [2, 2, 2]
+    assert list(function.get_output_shape(0)) == [3, 2, 2]
     assert function.get_output_element_type(0) == Type.f32
     for op in expected_ops:
         assert op in model_operators
@@ -363,9 +363,10 @@ def test_graph_preprocess_set_memory_type():
      (ResizeAlgorithm.RESIZE_NEAREST, ColorFormat.BGR, ColorFormat.I420_THREE_PLANES, True),
      (ResizeAlgorithm.RESIZE_NEAREST, ColorFormat.BGR, ColorFormat.NV12_SINGLE_PLANE, True),
      (ResizeAlgorithm.RESIZE_NEAREST, ColorFormat.BGR, ColorFormat.NV12_TWO_PLANES, True),
-     (ResizeAlgorithm.RESIZE_NEAREST, ColorFormat.BGR, ColorFormat.UNDEFINED, True)])
+     (ResizeAlgorithm.RESIZE_NEAREST, ColorFormat.BGR, ColorFormat.UNDEFINED, True),
+     ])
 def test_graph_preprocess_steps(algorithm, color_format1, color_format2, is_failing):
-    shape = [1, 1, 3, 3]
+    shape = [1, 3, 3, 3]
     parameter_a = ops.parameter(shape, dtype=np.float32, name="A")
     model = parameter_a
     function = Model(model, [parameter_a], "TestFunction")
@@ -394,7 +395,7 @@ def test_graph_preprocess_steps(algorithm, color_format1, color_format2, is_fail
         ]
         assert len(model_operators) == 16
         assert function.get_output_size() == 1
-        assert list(function.get_output_shape(0)) == [1, 1, 3, 3]
+        assert list(function.get_output_shape(0)) == [1, 3, 3, 3]
         assert function.get_output_element_type(0) == Type.f32
         for op in expected_ops:
             assert op in model_operators

src/core/include/openvino/core/preprocess/color_format.hpp

@@ -9,19 +9,16 @@ namespace preprocess {
 
 /// \brief Color format enumeration for conversion
 enum class ColorFormat {
-    UNDEFINED,
-    NV12_SINGLE_PLANE,  // Image in NV12 format as single tensor
-    /// \brief Image in NV12 format represented as separate tensors for Y and UV planes.
-    NV12_TWO_PLANES,
-    I420_SINGLE_PLANE,  // Image in I420 (YUV) format as single tensor
-    /// \brief Image in I420 format represented as separate tensors for Y, U and V planes.
-    I420_THREE_PLANES,
-    RGB,
-    BGR,
-    /// \brief Image in RGBX interleaved format (4 channels)
-    RGBX,
-    /// \brief Image in BGRX interleaved format (4 channels)
-    BGRX
+    UNDEFINED,          //!< Undefined color format
+    NV12_SINGLE_PLANE,  //!< Image in NV12 format represented as separate tensors for Y and UV planes.
+    NV12_TWO_PLANES,    //!< Image in NV12 format represented as separate tensors for Y and UV planes.
+    I420_SINGLE_PLANE,  //!< Image in I420 (YUV) format as single tensor
+    I420_THREE_PLANES,  //!< Image in I420 format represented as separate tensors for Y, U and V planes.
+    RGB,                //!< Image in RGB interleaved format (3 channels)
+    BGR,                //!< Image in BGR interleaved format (3 channels)
+    GRAY,               //!< Image in GRAY format (1 channel)
+    RGBX,               //!< Image in RGBX interleaved format (4 channels)
+    BGRX                //!< Image in BGRX interleaved format (4 channels)
 };
 
 }  // namespace preprocess

src/core/src/preprocess/color_utils.cpp

@@ -20,13 +20,18 @@ std::unique_ptr<ColorFormatInfo> ColorFormatInfo::get(ColorFormat format) {
         res.reset(new ColorFormatInfoI420_ThreePlanes(format));
         break;
     case ColorFormat::RGB:
+        res.reset(new ColorFormatRGB(format));
+        break;
     case ColorFormat::BGR:
-        res.reset(new ColorFormatNHWC(format));
+        res.reset(new ColorFormatBGR(format));
         break;
     case ColorFormat::RGBX:
     case ColorFormat::BGRX:
         res.reset(new ColorFormatInfo_RGBX_Base(format));
         break;
+    case ColorFormat::GRAY:
+        res.reset(new ColorFormatNHWC(format));
+        break;
     default:
         res.reset(new ColorFormatInfo(format));
         break;

src/core/src/preprocess/color_utils.hpp

@@ -43,6 +43,9 @@ inline std::string color_format_name(ColorFormat format) {
     case ColorFormat::BGRX:
         name = "BGRX";
         break;
+    case ColorFormat::GRAY:
+        name = "GRAY";
+        break;
     default:
         name = "Unknown";
         break;
@@ -65,16 +68,17 @@ class ColorFormatInfo {
         return {};
     }
 
-    // Calculate shape of plane based image shape in NHWC format
-    PartialShape shape(size_t plane_num, const PartialShape& image_src_shape) const {
+    PartialShape shape(size_t plane_num, const PartialShape& src_shape, const Layout& src_layout) const {
         OPENVINO_ASSERT(plane_num < planes_count(),
                         "Internal error: incorrect plane number specified for color format");
-        return calculate_shape(plane_num, image_src_shape);
+        return calculate_shape(plane_num, src_shape, src_layout);
     }
 
 protected:
-    virtual PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const {
-        return image_shape;
+    virtual PartialShape calculate_shape(size_t plane_num,
+                                         const PartialShape& src_shape,
+                                         const Layout& src_layout) const {
+        return src_shape;
     }
 
     explicit ColorFormatInfo(ColorFormat format) : m_format(format) {}
@@ -91,15 +95,38 @@ class ColorFormatNHWC : public ColorFormatInfo {
     }
 };
 
+// Assume that it should work with NHWC and NCHW cases, but default is NHWC
+class ColorFormatRGB : public ColorFormatNHWC {
+public:
+    explicit ColorFormatRGB(ColorFormat format) : ColorFormatNHWC(format) {}
+
+protected:
+    PartialShape calculate_shape(size_t plane_num,
+                                 const PartialShape& src_shape,
+                                 const Layout& src_layout) const override {
+        PartialShape result = src_shape;
+        if (src_shape.rank().is_static()) {
+            auto c_idx = ov::layout::channels_idx(src_layout);
+            c_idx = c_idx < 0 ? c_idx + src_shape.size() : c_idx;
+            result[c_idx] = 3;
+        }
+        return result;
+    }
+};
+
+using ColorFormatBGR = ColorFormatRGB;
+
 // Applicable for both NV12 and I420 formats
 class ColorFormatInfoYUV420_Single : public ColorFormatNHWC {
 public:
     explicit ColorFormatInfoYUV420_Single(ColorFormat format) : ColorFormatNHWC(format) {}
 
 protected:
-    PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override {
-        PartialShape result = image_shape;
-        if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) {
+    PartialShape calculate_shape(size_t plane_num,
+                                 const PartialShape& src_shape,
+                                 const Layout& src_layout) const override {
+        PartialShape result = src_shape;
+        if (src_shape.rank().is_static() && src_shape.rank().get_length() == 4) {
             result[3] = 1;
             if (result[1].is_static()) {
                 result[1] = result[1].get_length() * 3 / 2;
@@ -118,9 +145,11 @@ class ColorFormatInfoNV12_TwoPlanes : public ColorFormatNHWC {
     }
 
 protected:
-    PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override {
-        PartialShape result = image_shape;
-        if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) {
+    PartialShape calculate_shape(size_t plane_num,
+                                 const PartialShape& src_shape,
+                                 const Layout& src_layout) const override {
+        PartialShape result = src_shape;
+        if (src_shape.rank().is_static() && src_shape.rank().get_length() == 4) {
             if (plane_num == 0) {
                 result[3] = 1;
                 return result;
@@ -148,9 +177,11 @@ class ColorFormatInfoI420_ThreePlanes : public ColorFormatNHWC {
     }
 
 protected:
-    PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override {
-        PartialShape result = image_shape;
-        if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) {
+    PartialShape calculate_shape(size_t plane_num,
+                                 const PartialShape& src_shape,
+                                 const Layout& src_layout) const override {
+        PartialShape result = src_shape;
+        if (src_shape.rank().is_static() && src_shape.rank().get_length() == 4) {
             result[3] = 1;  //  Number of channels is always 1 for I420 planes
             if (plane_num == 0) {
                 return result;
@@ -173,9 +204,11 @@ class ColorFormatInfo_RGBX_Base : public ColorFormatNHWC {
     explicit ColorFormatInfo_RGBX_Base(ColorFormat format) : ColorFormatNHWC(format) {}
 
 protected:
-    PartialShape calculate_shape(size_t plane_num, const PartialShape& image_shape) const override {
-        PartialShape result = image_shape;
-        if (image_shape.rank().is_static() && image_shape.rank().get_length() == 4) {
+    PartialShape calculate_shape(size_t plane_num,
+                                 const PartialShape& src_shape,
+                                 const Layout& src_layout) const override {
+        PartialShape result = src_shape;
+        if (src_shape.rank().is_static() && src_shape.rank().get_length() == 4) {
             result[3] = 4;
             return result;
         }

src/core/src/preprocess/preprocess_impls.cpp

@@ -95,7 +95,7 @@ InputInfo::InputInfoImpl::InputInfoData InputInfo::InputInfoImpl::create_new_par
 
     // Create separate parameter for each plane. Shape is based on color format
     for (size_t plane = 0; plane < color_info->planes_count(); plane++) {
-        auto plane_shape = color_info->shape(plane, new_param_shape);
+        auto plane_shape = color_info->shape(plane, new_param_shape, res.m_tensor_layout);
         auto plane_param = std::make_shared<opset8::Parameter>(tensor_elem_type, plane_shape);
         if (plane < get_tensor_data()->planes_sub_names().size()) {
             std::unordered_set<std::string> plane_tensor_names;
@@ -163,7 +163,7 @@ bool InputInfo::InputInfoImpl::build(const std::shared_ptr<Model>& model,
     context.model_shape() = data.m_param->get_partial_shape();
     context.target_element_type() = data.m_param->get_element_type();
 
-    // 2. Apply preprocessing
+    // Apply preprocessing
     auto nodes = data.as_nodes();
     for (const auto& action : get_preprocess()->actions()) {
         auto action_result = action.m_op(nodes, model, context);
@@ -176,11 +176,12 @@ bool InputInfo::InputInfoImpl::build(const std::shared_ptr<Model>& model,
                     "preprocessing operation. Current format is '",
                     color_format_name(context.color_format()),
                     "'");
-    OPENVINO_ASSERT(is_rgb_family(context.color_format()) || context.color_format() == ColorFormat::UNDEFINED,
-                    "model shall have RGB/BGR color format. Consider add 'convert_color' preprocessing operation "
+    OPENVINO_ASSERT(is_rgb_family(context.color_format()) || context.color_format() == ColorFormat::GRAY ||
+                        context.color_format() == ColorFormat::UNDEFINED,
+                    "model shall have RGB/BGR/GRAY color format. Consider add 'convert_color' preprocessing operation "
                     "to convert current color format '",
                     color_format_name(context.color_format()),
-                    "'to RGB/BGR");
+                    "'to RGB/BGR/GRAY");
 
     // Implicit: Convert element type + layout to user's tensor implicitly
     auto implicit_steps = create_implicit_steps(context, nodes[0].get_element_type());
@@ -193,6 +194,7 @@ bool InputInfo::InputInfoImpl::build(const std::shared_ptr<Model>& model,
     if (node.get_partial_shape() != context.model_shape()) {
         need_validate = true;  // Trigger revalidation if input parameter shape is changed
     }
+
     // Check final shape
     OPENVINO_ASSERT(node.get_partial_shape().compatible(context.model_shape()),
                     "Resulting shape '",

src/core/src/preprocess/preprocess_steps_impl.cpp

@@ -428,6 +428,64 @@ void PreStepsList::add_convert_color_impl(const ColorFormat& dst_format) {
                 context.color_format() = dst_format;
                 return res;
             }
+            if ((context.color_format() == ColorFormat::RGB || context.color_format() == ColorFormat::BGR) &&
+                (dst_format == ColorFormat::GRAY)) {
+                auto node = nodes[0];
+                auto elem_type = node.get_element_type();
+                auto shape = node.get_partial_shape();
+                OPENVINO_ASSERT(shape.size() == 4,
+                                "Input shape size should be equal to 4, actual size: ",
+                                shape.size());
+                auto channels_idx = get_and_check_channels_idx(context.layout(), shape);
+                OPENVINO_ASSERT(shape[channels_idx] == 3,
+                                "Channels dimesion should be equal to 3, actual value: ",
+                                shape[channels_idx]);
+
+                auto is_transposed = false, is_converted = false;
+                if (channels_idx + 1 == shape.size()) {
+                    // Transpose N...C  to NC...
+                    auto permutation = layout::utils::find_permutation(context.layout(), shape, ov::Layout{"NC..."});
+                    auto perm_constant =
+                        op::v0::Constant::create<int64_t>(element::i64, Shape{permutation.size()}, permutation);
+                    node = std::make_shared<op::v1::Transpose>(node, perm_constant);
+                    is_transposed = true;
+                }
+                if (elem_type.is_integral_number()) {
+                    // Compute in floats due weights are floats
+                    node = std::make_shared<op::v0::Convert>(node, element::f32);
+                    is_converted = true;
+                }
+
+                // RGB coefficients were used from https://docs.opencv.org/3.4/de/d25/imgproc_color_conversions.html
+                auto weights_data = context.color_format() == ColorFormat::RGB
+                                        ? std::vector<float>{0.299f, 0.587f, 0.114f}
+                                        : std::vector<float>{0.114f, 0.587f, 0.299f};
+                auto weights_shape = ov::Shape(shape.size(), 1);
+                weights_shape[1] = 3;  // Set kernel layout to [1, 3, 1, ...]
+                auto weights_node = std::make_shared<ov::op::v0::Constant>(element::f32, weights_shape, weights_data);
+                node = std::make_shared<ov::op::v1::Convolution>(node,
+                                                                 weights_node,
+                                                                 ov::Strides(weights_shape.size() - 2, 1),
+                                                                 ov::CoordinateDiff(weights_shape.size() - 2, 0),
+                                                                 ov::CoordinateDiff(weights_shape.size() - 2, 0),
+                                                                 ov::Strides(weights_shape.size() - 2, 1));
+
+                if (is_converted) {
+                    // Round values according to OpenCV rule before converting to integral values
+                    auto round_val =
+                        std::make_shared<ov::op::v5::Round>(node, ov::op::v5::Round::RoundMode::HALF_TO_EVEN);
+                    node = std::make_shared<op::v0::Convert>(round_val, elem_type);
+                }
+                if (is_transposed) {
+                    // Return NC... to N...C
+                    auto permutation = layout::utils::find_permutation(ov::Layout{"NC..."}, shape, context.layout());
+                    auto perm_constant =
+                        op::v0::Constant::create<int64_t>(element::i64, Shape{permutation.size()}, permutation);
+                    node = std::make_shared<op::v1::Transpose>(node, perm_constant);
+                }
+                context.color_format() = dst_format;
+                return std::make_tuple(std::vector<Output<Node>>{node}, true);
+            }
             if (context.color_format() == ColorFormat::RGBX) {
                 if (dst_format == ColorFormat::RGB) {
                     auto res = cut_last_channel(nodes, function, context);