Skip to content

Commit

Permalink
Add fixes and tests for different layers
Browse files Browse the repository at this point in the history
  • Loading branch information
@sl-sergei
sl-sergei committed Nov 17, 2020
1 parent 2b82f8f commit 32e7ef8
Show file tree
Hide file tree
Showing 7 changed files with 240 additions and 28 deletions.
3 changes: 3 additions & 0 deletions modules/dnn/include/opencv2/dnn/dnn.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,9 +354,12 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
const int requiredOutputs,
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const;

virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const {CV_UNUSED(inputs); CV_UNUSED(outputs); return 0;}

virtual bool updateMemoryShapes(const std::vector<MatShape> &inputs);

CV_PROP String name; //!< Name of the layer instance, can be used for logging or other internal purposes.
CV_PROP String type; //!< Type name which was used for creating layer by layer factory.
CV_PROP int preferableTarget; //!< prefer target for layer forwarding
Expand Down
55 changes: 54 additions & 1 deletion modules/dnn/src/dnn.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1119,6 +1119,7 @@ struct Net::Impl : public detail::NetImplBase
preferableBackend = DNN_BACKEND_DEFAULT;
preferableTarget = DNN_TARGET_CPU;
skipInfEngineInit = false;
hasDynamicShapes = false;
}

Ptr<DataLayer> netInputLayer;
Expand All @@ -1130,6 +1131,7 @@ struct Net::Impl : public detail::NetImplBase
int preferableTarget;
String halideConfigFile;
bool skipInfEngineInit;
bool hasDynamicShapes;
// Map host data to backend specific wrapper.
std::map<void*, Ptr<BackendWrapper> > backendWrappers;

Expand Down Expand Up @@ -3074,6 +3076,46 @@ struct Net::Impl : public detail::NetImplBase
shapes = inOutShapes[layerId];
}

void updateLayersShapes()
{
CV_Assert(!layers[0].outputBlobs.empty());
ShapesVec inputShapes;
for(int i = 0; i < layers[0].outputBlobs.size(); i++)
{
Mat& inp = layers[0].outputBlobs[i];
CV_Assert(inp.total());
if (preferableBackend == DNN_BACKEND_OPENCV &&
preferableTarget == DNN_TARGET_OPENCL_FP16)
{
layers[0].outputBlobs[i].create(inp.dims, inp.size, CV_16S);
}
inputShapes.push_back(shape(inp));
}
LayersShapesMap layersShapes;
layersShapes[0].in = inputShapes;
for (MapIdToLayerData::iterator it = layers.begin();
it != layers.end(); it++)
{
int layerId = it->first;
std::vector<LayerPin>& inputLayerIds = it->second.inputBlobsId;
if (layersShapes[layerId].in.empty())
{
for(int i = 0; i < inputLayerIds.size(); i++)
{
int inputLayerId = inputLayerIds[i].lid;
LayersShapesMap::iterator inputIt = layersShapes.find(inputLayerId);
if(inputIt == layersShapes.end() || inputIt->second.out.empty())
{
getLayerShapesRecursively(inputLayerId, layersShapes);
}
const MatShape& shape = layersShapes[inputLayerId].out[inputLayerIds[i].oid];
layersShapes[layerId].in.push_back(shape);
}
it->second.layerInstance->updateMemoryShapes(layersShapes[layerId].in);
}
}
}

LayerPin getLatestLayerPin(const std::vector<LayerPin>& pins)
{
return *std::max_element(pins.begin(), pins.end());
Expand Down Expand Up @@ -3487,6 +3529,8 @@ int Net::addLayer(const String &name, const String &type, LayerParams &params)
int id = ++impl->lastLayerId;
impl->layerNameToId.insert(std::make_pair(name, id));
impl->layers.insert(std::make_pair(id, LayerData(id, name, type, params)));
if (params.get<bool>("has_dynamic_shapes", false))
impl->hasDynamicShapes = true;

return id;
}
Expand Down Expand Up @@ -3818,8 +3862,13 @@ void Net::setInput(InputArray blob, const String& name, double scalefactor, cons
bool oldShape = prevShape == blobShape;

blob_.copyTo(impl->netInputLayer->inputsData[pin.oid]);
if (!oldShape)
if (!oldShape) {
ld.outputBlobs[pin.oid] = impl->netInputLayer->inputsData[pin.oid];
if (impl->hasDynamicShapes)
{
impl->updateLayersShapes();
}
}

if (!ld.outputBlobsWrappers[pin.oid].empty())
{
Expand Down Expand Up @@ -4746,6 +4795,10 @@ bool Layer::getMemoryShapes(const std::vector<MatShape> &inputs,
return false;
}

bool Layer::updateMemoryShapes(const std::vector<MatShape> &inputs)
{
return true;
}
//////////////////////////////////////////////////////////////////////////

static Mutex& getLayerFactoryMutex()
Expand Down
49 changes: 35 additions & 14 deletions modules/dnn/src/layers/pooling_layer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ class PoolingLayerImpl CV_FINAL : public PoolingLayer
stride = Size(1, 1);
pad_t = pad_l = pad_b = pad_r = 0;

hasDynamicShapes = params.get<bool>("has_dynamic_shapes", false);
shapesInitialized = !hasDynamicShapes;

if (params.has("pool") || params.has("kernel_size") ||
params.has("kernel_w") || params.has("kernel_h"))
{
Expand Down Expand Up @@ -1043,26 +1046,34 @@ virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inp
outShape.push_back(pooledSize.height);
outShape.push_back(pooledSize.width);
}
else if (padMode.empty())
else
{
for (int i = 0; i < local_kernel.size(); i++) {
float dst = (float)(inpShape[i] + pads_begin[i] + pads_end[i] - local_kernel[i]) / strides[i];
outShape.push_back(1 + (ceilMode ? ceil(dst) : floor(dst)));
if (hasDynamicShapes && !shapesInitialized)
{
//Just copy input shapes for width and height to prevent errors on loading stage
for (int i = 0; i < inpShape.size(); i++)
outShape.push_back(inpShape[i]);
}
else if (padMode.empty())
{
for (int i = 0; i < local_kernel.size(); i++) {
float dst = (float) (inpShape[i] + pads_begin[i] + pads_end[i] - local_kernel[i]) / strides[i];
outShape.push_back(1 + (ceilMode ? ceil(dst) : floor(dst)));
}

// If we have padding, ensure that the last pooling starts strictly
// inside the image (instead of at the padding); otherwise clip the last.
for (int i = 0; i < pads_end.size(); i++) {
if (pads_end[i] && (outShape[2 + i] - 1) * strides[i] >= inpShape[i] + pads_end[i]) {
--outShape[2 + i];
CV_Assert((outShape[2 + i] - 1) * strides[i] < inpShape[i] + pads_end[i]);
// If we have padding, ensure that the last pooling starts strictly
// inside the image (instead of at the padding); otherwise clip the last.
for (int i = 0; i < pads_end.size(); i++) {
if (pads_end[i] && (outShape[2 + i] - 1) * strides[i] >= inpShape[i] + pads_end[i]) {
--outShape[2 + i];
CV_Assert((outShape[2 + i] - 1) * strides[i] < inpShape[i] + pads_end[i]);
}
}
} else {
getConvPoolOutParams(inpShape, local_kernel, strides, padMode,
std::vector<size_t>(local_kernel.size(), 1), outShape);
}
}
else
{
getConvPoolOutParams(inpShape, local_kernel, strides, padMode, std::vector<size_t>(local_kernel.size(), 1), outShape);
}
if (type == ROI)
{
CV_Assert(inputs.size() == 2);
Expand All @@ -1083,6 +1094,14 @@ virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inp
return false;
}

bool updateMemoryShapes(const std::vector<MatShape> &inputs) CV_OVERRIDE
{
int dims = inputs[0].size();
CV_Assert(inputs[0][dims - 1] > 0 && inputs[0][dims - 2] > 0);
shapesInitialized = true;
return true;
}

virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
const std::vector<MatShape> &outputs) const CV_OVERRIDE
{
Expand Down Expand Up @@ -1114,6 +1133,8 @@ virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inp
ROI, // RoI pooling, https://arxiv.org/pdf/1504.08083.pdf
PSROI // Position-sensitive RoI pooling, https://arxiv.org/pdf/1605.06409.pdf
};
bool hasDynamicShapes;
bool shapesInitialized;
};

Ptr<PoolingLayer> PoolingLayer::create(const LayerParams& params)
Expand Down
51 changes: 49 additions & 2 deletions modules/dnn/src/layers/reshape_layer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,9 @@ class ReshapeLayerImpl CV_FINAL : public ReshapeLayer
setParamsFrom(params);
int axis = params.get<int>("axis", 0);
int numAxes = params.get<int>("num_axes", -1);
hasDynamicShapes = params.get<bool>("has_dynamic_shapes", false);
shapesInitialized = !hasDynamicShapes;

CV_Assert(numAxes >= -1);
newShapeRange = (numAxes == -1) ? Range(axis, INT_MAX) : Range(axis, axis + numAxes);

Expand All @@ -176,6 +179,25 @@ class ReshapeLayerImpl CV_FINAL : public ReshapeLayer
for (i = 0; i < dims; i++)
newShapeDesc[i] = paramShape.get<int>(i);
}
if (hasDynamicShapes)
{
dynamicShapes.clear();
inputIndices.clear();
if (params.has("dynamic_axes")) {
CV_Assert(params.has("input_indices"));
const DictValue &dynamicAxes = params.get("dynamic_axes");
const DictValue &dynamicInputShapes = params.get("input_indices");
int i, dims = dynamicAxes.size();
CV_Assert(dims == dynamicInputShapes.size());
CV_Assert(dims > 0);
dynamicShapes.resize(dims);
inputIndices.resize(dims);
for (i = 0; i < dims; i++) {
dynamicShapes[i] = dynamicAxes.get<int>(i);
inputIndices[i] = dynamicInputShapes.get<int>(i);
}
}
}
}

virtual bool supportBackend(int backendId) CV_OVERRIDE
Expand All @@ -189,13 +211,21 @@ class ReshapeLayerImpl CV_FINAL : public ReshapeLayer
std::vector<MatShape> &outputs,
std::vector<MatShape> &internals) const CV_OVERRIDE
{

if (inputs.size() == 1 || inputs.size() == requiredOutputs)
{
outputs.clear();
for (size_t i = 0; i < inputs.size(); i++)
{
outputs.push_back(MatShape());
computeShapeByReshapeMask(inputs[i], newShapeDesc, newShapeRange, outputs.back());
if (hasDynamicShapes && !shapesInitialized)
{
outputs.push_back(newShapeDesc);
}
else
{
outputs.push_back(MatShape());
computeShapeByReshapeMask(inputs[i], newShapeDesc, newShapeRange, outputs.back());
}
}
}
else
Expand All @@ -206,6 +236,19 @@ class ReshapeLayerImpl CV_FINAL : public ReshapeLayer
return true;
}

bool updateMemoryShapes(const std::vector<MatShape> &inputs) CV_OVERRIDE
{
if (hasDynamicShapes)
{
for (int i = 0; i < dynamicShapes.size(); ++i)
{
newShapeDesc[dynamicShapes[i]] = inputs[0][inputIndices[i]];
}
}
shapesInitialized = true;
return true;
}

void finalize(InputArrayOfArrays, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
{
std::vector<Mat> outputs;
Expand Down Expand Up @@ -287,6 +330,10 @@ class ReshapeLayerImpl CV_FINAL : public ReshapeLayer

private:
std::vector<MatShape> outShapes;
std::vector<int> dynamicShapes; // Which axes shapes are dynamic and require reinitialization with new input
std::vector<int> inputIndices; // Which axes from input are needed to compute correct output shape
bool hasDynamicShapes;
bool shapesInitialized;
};

Ptr<ReshapeLayer> ReshapeLayer::create(const LayerParams& params)
Expand Down
13 changes: 12 additions & 1 deletion modules/dnn/src/layers/slice_layer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ class SliceLayerImpl : public SliceLayer
setParamsFrom(params);
axis = params.get<int>("axis", 1);
num_split = params.get<int>("num_split", 0);
hasDynamicShapes = params.get<bool>("has_dynamic_shapes", false);
shapesInitialized = !hasDynamicShapes;
if (params.has("slice_point"))
{
CV_Assert(!params.has("begin") && !params.has("size") && !params.has("end"));
Expand Down Expand Up @@ -143,7 +145,8 @@ class SliceLayerImpl : public SliceLayer
CV_Assert(sliceRanges[i].size() <= inpShape.size());
for (int j = 0; j < sliceRanges[i].size(); ++j)
{
outputs[i][j] = clamp(sliceRanges[i][j], inpShape[j]).size();
if (shapesInitialized || inpShape[j] > 0)
outputs[i][j] = clamp(sliceRanges[i][j], inpShape[j]).size();
}
}
}
Expand All @@ -158,6 +161,12 @@ class SliceLayerImpl : public SliceLayer
return false;
}

bool updateMemoryShapes(const std::vector<MatShape> &inputs) CV_OVERRIDE
{
shapesInitialized = true;
return true;
}

void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
{
#ifdef HAVE_OPENCL
Expand Down Expand Up @@ -564,6 +573,8 @@ class SliceLayerImpl : public SliceLayer
protected:
// The actual non-negative values determined from @p sliceRanges depends on input size.
std::vector<std::vector<Range> > finalSliceRanges;
bool hasDynamicShapes;
bool shapesInitialized;
};

class CropLayerImpl CV_FINAL : public SliceLayerImpl
Expand Down
Loading

0 comments on commit 32e7ef8

Please sign in to comment.