Skip to content

Commit

Permalink
Merge pull request opencv#23047 from fengyuentau:layer_norm
Browse files Browse the repository at this point in the history 
dnn: add layer normalization for vision transformers

* add layer norm onnx parser, impl and tests

* add onnx graph simplifier for layer norm expanded

* handle the case when constants are of type Initializer

* add test case for layer norm expanded with initializers

* use CV_Assert & CV_CheckType in place of CV_Assert_N; use forward_fallback for OCL_FP16

* use const ref / ref in parameters of invoker::run; extract inner const if from nested loop; use size_t in place of ull

* template hasBias

* remove trailing whitespace

* use pointer parameter with null check; move normSize division & mean_square division outside of loop; use std::max to ensure positive value before std::sqrt

* refactor implementation, optimize parallel_for

* disable layer norm expanded

* remove the removal of layer norm optional outputs
  • Loading branch information
@fengyuentau
fengyuentau authored Jan 27, 2023
1 parent 52855a3 commit 4d918ba
Show file tree
Hide file tree
Showing 7 changed files with 636 additions and 0 deletions.
10 changes: 10 additions & 0 deletions modules/dnn/include/opencv2/dnn/all_layers.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1085,6 +1085,16 @@ CV__DNN_INLINE_NS_BEGIN
static Ptr<TileLayer> create(const LayerParams& params);
};

class CV_EXPORTS LayerNormLayer : public Layer
{
public:
bool hasBias;
int axis;
float epsilon;

static Ptr<LayerNormLayer> create(const LayerParams& params);
};

//! @}
//! @}
CV__DNN_INLINE_NS_END
Expand Down
208 changes: 208 additions & 0 deletions modules/dnn/perf/perf_layer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,12 +417,220 @@ PERF_TEST_P_(Layer_ScatterND, DISABLED_ScatterND_add)
test_layer({N, C, H , W}, "add");
}

struct Layer_LayerNorm : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());

Mat x(x_shape, CV_32FC1);
Mat scale(x_shape.back(), 1, CV_32FC1);
Mat b(x_shape.back(), 1, CV_32FC1);

randu(x, 0.f, 1.f);
randu(scale, 0.f, 1.f);
randu(b, 0.f, 1.f);


Net net;
LayerParams lp;
lp.type = "LayerNormalization";
lp.name = "testLayer";
lp.set("axis", 2);
lp.set("hasBias", true);
int id = net.addLayerToPrev(lp.name, lp.type, lp);
net.connect(0, 0, id, 0);
net.connect(0, 1, id, 1);
net.connect(0, 2, id, 2);

// warmup
{
std::vector<String> inpNames(3);
inpNames[0] = "x";
inpNames[1] = "scale";
inpNames[2] = "b";
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setInput(scale, inpNames[1]);
net.setInput(b, inpNames[2]);

net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}

TEST_CYCLE()
{
Mat res = net.forward();
}

SANITY_CHECK_NOTHING();
}

int N = 1;
int H = 50;
int W = 768;
};

PERF_TEST_P_(Layer_LayerNorm, LayerNorm)
{
test_layer({N, H ,W});
}

struct Layer_LayerNormExpanded : public TestBaseWithParam<tuple<Backend, Target> >
{
void test_layer(const std::vector<int>& x_shape)
{
int backendId = get<0>(GetParam());
int targetId = get<1>(GetParam());

Mat x(x_shape, CV_32FC1);
Mat scale(1, x_shape.back(), CV_32FC1); // transpose to pass shape check
Mat b(1, x_shape.back(), CV_32FC1); // transpose to pass shape check

randu(x, 0.f, 1.f);
randu(scale, 0.f, 1.f);
randu(b, 0.f, 1.f);

// sub graph structure:
// -> ReduceMean -> -> Pow(2) -> ReduceMean -> Add(epsilon) -> Sqrt ->
// x Sub Div -> Mul(scale) -> Add(bias)
// ---------------> ------------------------------------------------->

Net net;

LayerParams lp_rm;
lp_rm.type = "Reduce";
lp_rm.name = "reducemean1";
lp_rm.set("reduce", "AVE");
std::vector<int> deleteDims(1, x_shape.back());
lp_rm.set("deleted_dims", DictValue::arrayInt(&deleteDims[0], deleteDims.size()));
std::vector<int> targetDims(x_shape.begin(), x_shape.end());
targetDims[x_shape.size() - 1] = 1;
lp_rm.set("target_dims", DictValue::arrayInt(&targetDims[0], targetDims.size()));
int id_rm = net.addLayerToPrev(lp_rm.name, lp_rm.type, lp_rm);
net.connect(0, 0, id_rm, 0);

LayerParams lp_sub;
lp_sub.type = "NaryEltwise";
lp_sub.name = "sub1";
lp_sub.set("operation", "sub");
int id_sub = net.addLayer(lp_sub.name, lp_sub.type, lp_sub);
net.connect(0, 0, id_sub, 0);
net.connect(id_rm, 0, id_sub, 1);

Mat pow_const(1, 1, CV_32FC1);
pow_const.at<float>(0) = 2.f;
LayerParams lp_pow_const;
lp_pow_const.type = "Const";
lp_pow_const.name = "const1";
lp_pow_const.blobs.push_back(pow_const);
int id_pow_const = net.addLayer(lp_pow_const.name, lp_pow_const.type, lp_pow_const);
LayerParams lp_pow;
lp_pow.type = "NaryEltwise";
lp_pow.name = "pow1";
lp_pow.set("operation", "pow");
int id_pow = net.addLayer(lp_pow.name, lp_pow.type, lp_pow);
net.connect(id_sub, 0, id_pow, 0);
net.connect(id_pow_const, 0, id_pow, 1);

LayerParams lp_rm1;
lp_rm1.type = "Reduce";
lp_rm1.name = "reducemean2";
lp_rm1.set("reduce", "AVE");
lp_rm1.set("deleted_dims", DictValue::arrayInt(&deleteDims[0], deleteDims.size()));
lp_rm1.set("target_dims", DictValue::arrayInt(&targetDims[0], targetDims.size()));
int id_rm1 = net.addLayer(lp_rm1.name, lp_rm1.type, lp_rm1);
net.connect(id_pow, 0, id_rm1, 0);

Mat add_const(1, 1, CV_32F);
add_const.at<float>(0) = 1e-5;
LayerParams lp_add_const;
lp_add_const.type = "Const";
lp_add_const.name = "const2";
lp_add_const.blobs.push_back(add_const);
int id_add_const = net.addLayer(lp_add_const.name, lp_add_const.type, lp_add_const);
LayerParams lp_add;
lp_add.type = "NaryEltwise";
lp_add.name = "add1";
lp_add.set("operation", "add");
int id_add = net.addLayer(lp_add.name, lp_add.type, lp_add);
net.connect(id_rm1, 0, id_add, 0);
net.connect(id_add_const, 0, id_add, 1);

LayerParams lp_sqrt;
lp_sqrt.type = "Sqrt";
lp_sqrt.name = "sqrt1";
int id_sqrt = net.addLayer(lp_sqrt.name, lp_sqrt.type, lp_sqrt);
net.connect(id_add, 0, id_sqrt, 0);

LayerParams lp_div;
lp_div.type = "NaryEltwise";
lp_div.name = "div1";
lp_div.set("operation", "div");
int id_div = net.addLayer(lp_div.name, lp_div.type, lp_div);
net.connect(id_sub, 0, id_div, 0);
net.connect(id_sqrt, 0, id_div, 1);

LayerParams lp_mul;
lp_mul.type = "NaryEltwise";
lp_mul.name = "mul1";
lp_mul.set("operation", "mul");
int id_mul = net.addLayer(lp_mul.name, lp_mul.type, lp_mul);
net.connect(id_div, 0, id_mul, 0);
net.connect(0, 1, id_mul, 1);

LayerParams lp_add1;
lp_add1.type = "NaryEltwise";
lp_add1.name = "add2";
lp_add1.set("operation", "add");
int id_add1 = net.addLayer(lp_add1.name, lp_add1.type, lp_add1);
net.connect(id_mul, 0, id_add1, 0);
net.connect(0, 2, id_add1, 1);

// warmup
{
std::vector<String> inpNames(3);
inpNames[0] = "x";
inpNames[1] = "scale";
inpNames[2] = "b";
net.setInputsNames(inpNames);
net.setInput(x, inpNames[0]);
net.setInput(scale, inpNames[1]);
net.setInput(b, inpNames[2]);

net.setPreferableBackend(backendId);
net.setPreferableTarget(targetId);
Mat out = net.forward();
}

TEST_CYCLE()
{
Mat res = net.forward();
}

SANITY_CHECK_NOTHING();
}

int N = 1;
int H = 50;
int W = 768;
};

PERF_TEST_P_(Layer_LayerNormExpanded, DISABLED_LayerNormExpanded)
{
test_layer({N, H ,W});
}

INSTANTIATE_TEST_CASE_P(/**/, Layer_Slice, dnnBackendsAndTargets(false, false));
INSTANTIATE_TEST_CASE_P(/**/, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
#ifdef HAVE_CUDA
INSTANTIATE_TEST_CASE_P(CUDA, Layer_NaryEltwise, testing::Values(std::make_tuple(DNN_BACKEND_CUDA, DNN_TARGET_CUDA)));
#endif
INSTANTIATE_TEST_CASE_P(/**/, Layer_Scatter, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_ScatterND, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNorm, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));
INSTANTIATE_TEST_CASE_P(/**/, Layer_LayerNormExpanded, testing::Values(std::make_tuple(DNN_BACKEND_OPENCV, DNN_TARGET_CPU)));

} // namespace
1 change: 1 addition & 0 deletions modules/dnn/src/init.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ void initializeLayerFactory()
CV_DNN_REGISTER_LAYER_CLASS(Arg, ArgLayer);
CV_DNN_REGISTER_LAYER_CLASS(Reciprocal, ReciprocalLayer);
CV_DNN_REGISTER_LAYER_CLASS(Gather, GatherLayer);
CV_DNN_REGISTER_LAYER_CLASS(LayerNormalization, LayerNormLayer);

CV_DNN_REGISTER_LAYER_CLASS(Crop, CropLayer);
CV_DNN_REGISTER_LAYER_CLASS(Eltwise, EltwiseLayer);
Expand Down
Loading

0 comments on commit 4d918ba

Please sign in to comment.