Processing failed, potentially unsupported config

[f-alemauro]

Hello!
I have an issue with OAK-1 running an ImageManip node.

My device: OAK-1, tried with normal and "barebone" (https://store.opencv.ai/collections/usb/products/oak-1)

DepthAi version: 2.11.1.0

OS: Windows 10

Python: 3.9.2

The error I get is the following:

Processing failed, potentially unsupported config

Here is a simple code to reproduce the error

import cv2
import depthai as dai
import numpy as np


def to_planar(arr: np.ndarray, shape: tuple) -> list:
    return [val for channel in cv2.resize(arr, shape).transpose(2, 0, 1) for y_col in channel for val in y_col]


w_frame = 1280
h_frame = 720
xmin = 539
ymin = 258
xmax = xmin+207  # <--- W
ymax = ymin+207  # <--- H
cropxmin_norm = xmin / w_frame
cropymin_norm = ymin / h_frame
cropxmax_norm = xmax / w_frame
cropymax_norm = ymax / h_frame

width = xmax - xmin
height = ymax - ymin
area = width * height
aspect_ratio = width / height

print("W: {} - H: {}".format(width, height))
print("A: {} - AR: {}".format(area, aspect_ratio))
pipeline = dai.Pipeline()


xlinkIn = pipeline.create(dai.node.XLinkIn)
xlinkIn.setMaxDataSize(w_frame*h_frame*3)
xlinkIn.setStreamName("input")

# creating preview out node
rgbOut = pipeline.createXLinkOut()
rgbOut.setStreamName("rgb")

rgbOutRes = pipeline.createXLinkOut()
rgbOutRes.setStreamName("rgb_res")

# create a resize node
resize_manip = pipeline.create(dai.node.ImageManip)
resize_manip.initialConfig.setCropRect(cropxmin_norm, cropymin_norm, cropxmax_norm, cropymax_norm)
resize_manip.initialConfig.setResizeThumbnail(300, 300, 0, 0, 0)
resize_manip.initialConfig.setKeepAspectRatio(False)

# linking
xlinkIn.out.link(resize_manip.inputImage)
xlinkIn.out.link(rgbOut.input)
resize_manip.out.link(rgbOutRes.input)


with dai.Device(pipeline) as device:
    info = device.getDeviceInfo()
    inputImage = device.getInputQueue("input")
    qRgb = device.getOutputQueue(name="rgb", maxSize=8, blocking=True)
    qRgb_res = device.getOutputQueue(name="rgb_res", maxSize=8, blocking=True)

    img_data = cv2.imread("img.jpg")

    img = dai.ImgFrame()
    img.setData(to_planar(img_data, (w_frame, h_frame)))
    img.setType(dai.RawImgFrame.Type.BGR888p)
    img.setWidth(w_frame)
    img.setHeight(h_frame)
    inputImage.send(img)
    inRgb = qRgb.get()
    if inRgb is not None:
        c = inRgb.getCvFrame()
        cv2.imshow('RGB', inRgb.getCvFrame())
    inRgb_res = qRgb_res.get()
    if inRgb_res is not None:
        cv2.imshow('RGB_RES', inRgb_res.getCvFrame())
    cv2.waitKey(12)

This code simply load the image to the device, the device should perform a crop and a resize with thumbnail.
At the beginning of the code I set the dimension of the rectangle to be cropped

xmin = 539
ymin = 258
xmax = xmin+207  # <--- W
ymax = ymin+207  # <--- H

With that exact (207, 207) value this script gives that error. If I change to, eg, (207, 208) no more error!

Here is the image I'm using to test this:

[Erol444]

Hello, sorry for the delay. We have recently refactored whole ImageManip, could you try with the new version as well?
python3 -mpip install depthai==2.13.2.0.dev0+87552f67f0820a6eef058550af2c27724795fefe --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/
If that doesn't work, I'm sure @themarpe will be able to spot the issue.
Thanks, Erik

[f-alemauro]

Great, thank, I'll definitely try it!
I'll let you know

[f-alemauro]

So, tried with that deptahi version

python3 -mpip install depthai==2.13.2.0.dev0+87552f67f0820a6eef058550af2c27724795fefe --extra-index-url https://artifacts.luxonis.com/artifactory/luxonis-python-snapshot-local/

and the code I posted now works. No more image manipulation error appears.

To test it thoroughly I wrote this code

import cv2
import depthai as dai
import numpy as np


def to_planar(arr: np.ndarray, shape: tuple) -> list:
    return [val for channel in cv2.resize(arr, shape).transpose(2, 0, 1) for y_col in channel for val in y_col]


w_frame = 1280
h_frame = 720
pipeline = dai.Pipeline()


xlinkIn = pipeline.create(dai.node.XLinkIn)
xlinkIn.setMaxDataSize(w_frame*h_frame*3)
xlinkIn.setStreamName("input")

rgbOutRes = pipeline.createXLinkOut()
rgbOutRes.setStreamName("rgb_res")


script = pipeline.create(dai.node.Script)
xlinkIn.out.link(script.inputs['img'])
script.setScript("""
import time
img = node.io['img'].get()
for w in range (200,500):
    for h in range (10,500):
        node.warn(f"WxH: {w}x{h}")
        cfg = ImageManipConfig()
        cfg.setCropRect(0, 0, w/1280, h/720)
        cfg.setResizeThumbnail(100, 100, 0, 0, 0)
        cfg.setKeepAspectRatio(False) #  <- this MUST be false otherwise the sensor crash when using setResizeThumbnail
        node.io['manip_cfg'].send(cfg)
        node.io['manip_img'].send(img)    
        time.sleep(0.5)
""")


# create a resize node
resize_manip = pipeline.create(dai.node.ImageManip)
script.outputs['manip_img'].link(resize_manip.inputImage)
script.outputs['manip_cfg'].link(resize_manip.inputConfig)
resize_manip.setWaitForConfigInput(True)

# linking
resize_manip.out.link(rgbOutRes.input)


with dai.Device(pipeline) as device:
    info = device.getDeviceInfo()
    inputImage = device.getInputQueue("input")
    qRgb_res = device.getOutputQueue(name="rgb_res", maxSize=8, blocking=True)

    img_data = cv2.imread("img.jpg")

    img = dai.ImgFrame()
    img.setData(to_planar(img_data, (w_frame, h_frame)))
    img.setType(dai.RawImgFrame.Type.BGR888p)
    img.setWidth(w_frame)
    img.setHeight(h_frame)
    inputImage.send(img)
    for w in range(10, 500):
        for h in range(10, 500):
            inRgb_res = qRgb_res.get()
            if inRgb_res is not None:
                cv2.imshow('RGB_RES', inRgb_res.getCvFrame())
            cv2.waitKey(10)

This code runs on depthai version 2.13.1.0, but gives ImageManip error.
The same code does not runs (freeze exactly after 9 cicles) on depthai version 2.13.2.0.dev0+87552f67f0820a6eef058550af2c27724795fefe

[Erol444]

Hello @f-alemauro, I have just tried your script with depthai==2.13.3.0 and I think it works (not sure what it should do). Even after 2 min doesn't seem to freeze as you mentioned. Could you try 2.13.3 as well?
Thanks, Erik

[f-alemauro]

Ok, with depthai==2.13.3.0 it does not freeze anymore. But... unfortunately it makes the previous error happen again (Processing failed, potentially unsupported config)

So now we have:
TEST A (the first I gave you) fails (ImageManip error) on depthai==2.11.1.0
TEST A works on depthai==2.13.2.0.dev0+87552f67f0820a6eef058550af2c27724795fefe
TEST B fails (freezes) on depthai==2.13.2.0.dev0+87552f67f0820a6eef058550af2c27724795fefe
TEST A fails (ImageManip error) on depthai==2.13.3.0
TEST B works on depthai==2.13.3.0

To be more precise: TEST B is a more "advanced" version of TEST A.
The host send an image to the device, the device has a pipeline with a script node that loop through various image cropping configurations (i.e. different cropping sizes) and send image and config to an ImageManip node.
This would be a quick&dirty way to try understanding what is causing the ImageManip node to fail: is it due to the cropping resolution? Is it due to the cropped area? Is it due to the aspect ratio of the cropped area?

[themarpe]

@f-alemauro sorry for late response from my end
image_manip_refactor branch in in progress of being merged into mainline, and I'd like to cover this kind of edge cases as best as possible. Above code snippet does that relate to TEST A or B? Can you send over both, so I can test against before mainlining the improvements and fix if anything still doesn't work as expected.

[f-alemauro]

Here they are, TEST_A and TEST_B
TESTS.zip

Please let me know if I can help you debugging or testing or whatsoever ;)

[carsonfenimore]

I am seeing this still on depthai 2.14.1.0 when i do a crop AND resize in a single imagemanip. If I break up the crop and resize into separate imagemanips, it sometimes works - however I have even run into issues in this scenario which i don't understand.

The following code fails for every frame - unless you comment out the line below the "BROKEN IMAGE MANIP" comment. The error you get is "Processing failed, potentially unsupported config". I tried the 2.13 branches above to no avail.

I suspect this is a problem in the device-side code - is that right? Which, if true, is unfortunate as we can't see that code to help fix it.

#!/usr/bin/env python3
import cv2
import depthai as dai
import time
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('-f', '--file', help="Image or movie to read as source of frames, instead of camera", default=None)
args = parser.parse_args()

pipeline = dai.Pipeline()

if args.file:
    xinRgb = pipeline.create(dai.node.XLinkIn)
    xinRgb.setMaxDataSize(1920*1080*3)
    xinRgb.setStreamName("rgbIn")

xoutRgb = pipeline.create(dai.node.XLinkOut)
xoutRgb.setStreamName("rgb")

if not args.file:
    camRgb = pipeline.create(dai.node.ColorCamera)
    camRgb.setPreviewSize(1920, 1080)
    camRgb.setInterleaved(False)
    camRgb.setColorOrder(dai.ColorCameraProperties.ColorOrder.RGB)

copyPreview = pipeline.create(dai.node.ImageManip)
copyPreview.setNumFramesPool(22)
copyPreview.initialConfig.setKeepAspectRatio(False)
copyPreview.setMaxOutputFrameSize(1920*1080*3)
# BROKEN IMAGE MANIP
copyPreview.initialConfig.setResize(112,112)
copyPreview.initialConfig.setCropRect( 0.5357448469359066, 0.08127486485257948, 0.5553122867558863,0.12383386128773913 )
copyPreview.initialConfig.setFrameType(dai.RawImgFrame.Type.RGB888p)
if args.file:
    fileToPush = cv2.imread(args.file)
    w,h = (1280,720)
    fileToPush = cv2.resize(fileToPush, (w, h), interpolation=cv2.INTER_AREA)
    daiFrame = dai.ImgFrame()
    daiFrame.setTimestamp(time.perf_counter())
    daiFrame.setWidth(w)
    daiFrame.setHeight(h)
    daiFrame.setData(fileToPush.transpose(2, 0, 1))
    xinRgb.out.link(copyPreview.inputImage)
else:
    camRgb.preview.link(copyPreview.inputImage)

copyPreview2 = pipeline.create(dai.node.ImageManip)
copyPreview2.setNumFramesPool(22)
copyPreview2.initialConfig.setKeepAspectRatio(False)
copyPreview2.setMaxOutputFrameSize(1920*1080*3)
copyPreview2.initialConfig.setResize(112,112)
copyPreview2.initialConfig.setFrameType(dai.RawImgFrame.Type.RGB888p)
copyPreview.out.link(copyPreview2.inputImage)
copyPreview2.out.link(xoutRgb.input)

with dai.Device(pipeline) as device:

    qRgb = device.getOutputQueue(name="rgb", maxSize=4, blocking=False)
    if args.file:
        qRgbIn = device.getInputQueue(name="rgbIn")

    timeOfLastSend = 0
    maxSendPeriod = 1/60.
    daiSendCounter = 0
    while True:
        if args.file and (time.time() - timeOfLastSend) > maxSendPeriod:
            daiFrame.setSequenceNum(daiSendCounter)
            qRgbIn.send(daiFrame)
            timeOfLastSend = time.time()
            daiSendCounter += 1

        inRgb = qRgb.tryGet()
        if inRgb is not None:
            frame = inRgb.getCvFrame()
            cv2.imshow("rgb", frame)

        if cv2.waitKey(1) == ord('q'):
            break

[f-alemauro]

I tried also with depthai == 2.14.1.0 and depthai == 2.14.0.0, unfortunately the problem is not solved.
Would be really cool if you could share some of the imagemanip code to test it together 👌