Skip to content

Commit 45365e4

Browse files
vpisarevvpisarev
committed
Merge pull request opencv#9691 from dkurt:padding_layer_refactoring
2 parents 139b327 + 222149b commit 45365e4

File tree

7 files changed

+166
-113
lines changed

7 files changed

+166
-113
lines changed

‎modules/dnn/include/opencv2/dnn/all_layers.hpp

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,25 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
337337
static Ptr<PermuteLayer> create(const LayerParams& params);
338338
};
339339

340+
/**
341+
* @brief Adds extra values for specific axes.
342+
* @param paddings Vector of paddings in format
343+
* @code
344+
* [ pad_before, pad_after, // [0]th dimension
345+
* pad_before, pad_after, // [1]st dimension
346+
* ...
347+
* pad_before, pad_after ] // [n]th dimension
348+
* @endcode
349+
* that represents number of padded values at every dimension
350+
* starting from the first one. The rest of dimensions won't
351+
* be padded.
352+
* @param value Value to be padded. Defaults to zero.
353+
* @param input_dims Torch's parameter. If @p input_dims is not equal to the
354+
* actual input dimensionality then the `[0]th` dimension
355+
* is considered as a batch dimension and @p paddings are shifted
356+
* to a one dimension. Defaults to `-1` that means padding
357+
* corresponding to @p paddings.
358+
*/
340359
class CV_EXPORTS PaddingLayer : public Layer
341360
{
342361
public:

‎modules/dnn/src/layers/padding_layer.cpp

Lines changed: 59 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
// It is subject to the license terms in the LICENSE file found in the top-level directory
33
// of this distribution and at http://opencv.org/license.html.
44

5-
// Copyright (C) 2016, Intel Corporation, all rights reserved.
5+
// Copyright (C) 2017, Intel Corporation, all rights reserved.
66
// Third party copyrights are property of their respective owners.
77

88
/*
@@ -24,104 +24,105 @@ class PaddingLayerImpl : public PaddingLayer
2424
PaddingLayerImpl(const LayerParams &params)
2525
{
2626
setParamsFrom(params);
27-
paddingDim = params.get<int>("padding_dim");
28-
padding = params.get<int>("padding");
29-
inputDims = params.get<int>("input_dims", 0);
30-
index = params.get<int>("index", 0);
31-
paddingValue = params.get<double>("value", 0);
32-
33-
if(paddingDim < 0 || padding < 0)
34-
CV_Error(cv::Error::StsNotImplemented, "Negative padding and dim aren't supported");
27+
paddingValue = params.get<float>("value", 0);
28+
inputDims = params.get<int>("input_dims", -1);
29+
30+
CV_Assert(params.has("paddings"));
31+
const DictValue& paddingsParam = params.get("paddings");
32+
CV_Assert((paddingsParam.size() & 1) == 0);
33+
34+
paddings.resize(paddingsParam.size() / 2);
35+
for (int i = 0; i < paddings.size(); ++i)
36+
{
37+
paddings[i].first = paddingsParam.get<int>(i * 2); // Pad before.
38+
paddings[i].second = paddingsParam.get<int>(i * 2 + 1); // Pad after.
39+
CV_Assert(paddings[i].first >= 0, paddings[i].second >= 0);
40+
}
3541
}
3642

3743
bool getMemoryShapes(const std::vector<MatShape> &inputs,
3844
const int requiredOutputs,
3945
std::vector<MatShape> &outputs,
4046
std::vector<MatShape> &internals) const
4147
{
42-
outputs.clear();
43-
for(int i = 0; i < inputs.size(); i++)
48+
CV_Assert(inputs.size() == 1);
49+
const MatShape& inpShape = inputs[0];
50+
CV_Assert(inpShape.size() >= paddings.size());
51+
CV_Assert(inputDims == -1 || inpShape.size() == inputDims || inpShape.size() > paddings.size());
52+
53+
outputs.resize(1, inpShape);
54+
int offset = (inputDims == -1 ? 0 : (inpShape.size() > inputDims ? 1 : 0));
55+
for (int i = 0; i < paddings.size(); ++i)
4456
{
45-
MatShape shape = inputs[i];
46-
int dim = getPadDim(shape);
47-
CV_Assert(dim < shape.size());
57+
outputs[0][offset + i] = inpShape[offset + i] + paddings[i].first + paddings[i].second;
58+
}
59+
return false;
60+
}
4861

49-
shape[dim] += padding;
50-
outputs.push_back(shape);
62+
void finalize(const std::vector<Mat*> &inputs, std::vector<Mat> &outputs)
63+
{
64+
// Compute dstRanges.
65+
const MatSize& inpShape = inputs[0]->size;
66+
dstRanges.resize(paddings.size());
67+
68+
int offset = 0;
69+
if (inputDims != -1 && inputs[0]->dims != inputDims)
70+
{
71+
dstRanges.insert(dstRanges.begin(), Range::all());
72+
offset = 1;
5173
}
5274

53-
return false;
75+
for (int i = 0; i < paddings.size(); ++i)
76+
{
77+
dstRanges[offset + i].start = paddings[i].first;
78+
dstRanges[offset + i].end = paddings[i].first + inpShape[offset + i];
79+
}
80+
81+
// Add the rest of dimensions.
82+
for (int i = dstRanges.size(); i < inputs[0]->dims; ++i)
83+
dstRanges.push_back(Range::all());
5484
}
5585

5686
virtual bool supportBackend(int backendId)
5787
{
5888
return backendId == DNN_BACKEND_DEFAULT ||
59-
backendId == DNN_BACKEND_HALIDE && haveHalide();
89+
backendId == DNN_BACKEND_HALIDE && haveHalide() && dstRanges.size() == 4;
6090
}
6191

6292
void forward(std::vector<Mat*> &inputs, std::vector<Mat> &outputs, std::vector<Mat> &internals)
6393
{
6494
CV_TRACE_FUNCTION();
6595
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
6696

67-
for(int i = 0; i < inputs.size(); i++)
68-
{
69-
outputs[i] = paddingValue;
70-
const Mat& inp = *inputs[i];
71-
Mat& out = outputs[i];
72-
int dims = inp.dims;
73-
MatShape inShape(inp.size.p, inp.size.p + dims);
74-
MatShape outShape(out.size.p, out.size.p + dims);
75-
int dim = getPadDim(inShape);
76-
77-
int actualIndex = index;
78-
if(index == 0)
79-
actualIndex = inShape[dim];
80-
81-
std::vector<std::pair<Range, Range> > srcDstRanges;
82-
srcDstRanges.push_back(std::make_pair(Range(0, actualIndex), Range(0, actualIndex)));
83-
srcDstRanges.push_back(std::make_pair(Range(actualIndex, inShape[dim]),
84-
Range(actualIndex + padding, outShape[dim])));
85-
86-
std::vector<Range> srcRanges(dims, Range::all()), dstRanges = srcRanges;
87-
88-
for(int j = 0; j < srcDstRanges.size(); j++)
89-
{
90-
if(!srcDstRanges[j].first.empty())
91-
{
92-
srcRanges[dim] = srcDstRanges[j].first;
93-
dstRanges[dim] = srcDstRanges[j].second;
94-
Mat dst = out(&dstRanges[0]);
95-
Mat src = inp(&srcRanges[0]).clone();
96-
src.copyTo(dst);
97-
}
98-
}
99-
}
100-
}
101-
102-
int getPadDim(const MatShape& shape) const
103-
{
104-
return inputDims > 0 && (int)shape.size() > inputDims ? paddingDim + 1 : paddingDim;
97+
outputs[0].setTo(paddingValue);
98+
inputs[0]->copyTo(outputs[0](dstRanges));
10599
}
106100

107101
virtual Ptr<BackendNode> initHalide(const std::vector<Ptr<BackendWrapper> > &inputs)
108102
{
109103
#ifdef HAVE_HALIDE
110104
int inW, inH, inC, inN;
105+
int minN = std::max(dstRanges[0].start, 0);
106+
int minC = std::max(dstRanges[1].start, 0);
107+
int minY = std::max(dstRanges[2].start, 0);
108+
int minX = std::max(dstRanges[3].start, 0);
111109
Halide::Buffer<float> inputBuffer = halideBuffer(inputs[0]);
112110
getCanonicalSize(inputBuffer, &inW, &inH, &inC, &inN);
113111

114112
Halide::Var x("x"), y("y"), c("c"), n("n");
115113
Halide::Func top = (name.empty() ? Halide::Func() : Halide::Func(name));
116114
Halide::Func padded =
117115
Halide::BoundaryConditions::constant_exterior(inputBuffer, paddingValue);
118-
top(x, y, c, n) = padded(x, y, c, n);
116+
top(x, y, c, n) = padded(x - minX, y - minY, c - minC, n - minN);
119117
return Ptr<BackendNode>(new HalideBackendNode(top));
120118
#endif // HAVE_HALIDE
121119
return Ptr<BackendNode>();
122120
}
123121

124-
int paddingDim, padding, inputDims, index;
122+
private:
123+
std::vector<std::pair<int, int> > paddings; // Pairs pad before, pad after.
124+
std::vector<Range> dstRanges;
125+
int inputDims;
125126
float paddingValue;
126127
};
127128

‎modules/dnn/src/tensorflow/tf_importer.cpp

Lines changed: 20 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -931,51 +931,28 @@ void TFImporter::populateNet(Net dstNet)
931931
}
932932
else if (type == "Pad")
933933
{
934-
tensorflow::TensorProto paddings = getConstBlob(layer, value_id, 1);
935-
MatShape shape;
936-
blobShapeFromTensor(paddings, shape);
937-
if (shape[0] != 4)
938-
CV_Error(Error::StsError, "Expected NHWC data format");
939-
940-
// Copy tensor with paddings.
941-
std::vector<int32_t> values(shape[0] * 2);
942-
CV_Assert(sizeof(int32_t) * values.size() ==
943-
paddings.tensor_content().size());
944-
memcpy(&values[0], &paddings.tensor_content()[0],
945-
paddings.tensor_content().size());
946-
947-
// Allow only one padding operation per layer.
948-
bool padded = false;
949-
for (int i = 0; i < values.size(); ++i)
934+
Mat paddings = getTensorContent(getConstBlob(layer, value_id, 1));
935+
CV_Assert(paddings.type() == CV_32SC1);
936+
if (paddings.total() == 8)
950937
{
951-
if (values[i])
952-
{
953-
if (padded)
954-
CV_Error(Error::StsError,
955-
"Only single padding operation per layer is supported");
956-
padded = true;
957-
958-
int axis = i / 2;
959-
// Remap NHWC to NCHW.
960-
// 0 -> 0
961-
// 1 -> 2
962-
// 2 -> 3
963-
// 3 -> 1
964-
if (axis != 0)
965-
axis = axis % 3 + 1;
966-
967-
layerParams.set("padding_dim", axis);
968-
if (i % 2) // Pad after
969-
layerParams.set("padding", values[i]);
970-
else // Pad before
971-
layerParams.set("padding", -1 * values[i]);
972-
973-
int id = dstNet.addLayer(name, "Padding", layerParams);
974-
layer_id[name] = id;
975-
976-
connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0);
977-
}
938+
// Perhabs, we have NHWC padding dimensions order.
939+
// N H W C
940+
// 0 1 2 3 4 5 6 7
941+
std::swap(*paddings.ptr<int32_t>(0, 2), *paddings.ptr<int32_t>(0, 6));
942+
std::swap(*paddings.ptr<int32_t>(0, 3), *paddings.ptr<int32_t>(0, 7));
943+
// N C W H
944+
// 0 1 2 3 4 5 6 7
945+
std::swap(*paddings.ptr<int32_t>(0, 4), *paddings.ptr<int32_t>(0, 6));
946+
std::swap(*paddings.ptr<int32_t>(0, 5), *paddings.ptr<int32_t>(0, 7));
947+
// N C H W
948+
// 0 1 2 3 4 5 6 7
978949
}
950+
layerParams.set("paddings", DictValue::arrayInt<int*>((int*)paddings.data, paddings.total()));
951+
952+
int id = dstNet.addLayer(name, "Padding", layerParams);
953+
layer_id[name] = id;
954+
955+
connect(layer_id, dstNet, parsePin(layer.input(0)), id, 0);
979956
}
980957
else if (type == "FusedBatchNorm")
981958
{

‎modules/dnn/src/torch/torch_importer.cpp

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -714,23 +714,25 @@ struct TorchImporter : public ::cv::dnn::Importer
714714
readTorchTable(scalarParams, tensorParams);
715715
newModule->apiType = "Padding";
716716

717-
CV_Assert(scalarParams.has("pad") &&
718-
scalarParams.has("dim"));
717+
CV_Assert(scalarParams.has("pad") && scalarParams.has("dim"));
718+
if (scalarParams.has("index") && scalarParams.get<int>("index") != 1)
719+
CV_Error(Error::StsNotImplemented, "Padding with offset is not implemented");
719720

720-
layerParams.set("padding_dim",
721-
static_cast<int>(scalarParams.get<double>("dim") - 1));
722-
layerParams.set("padding", static_cast<int>(scalarParams.get<double>("pad")));
721+
if (scalarParams.has("value"))
722+
layerParams.set("value", scalarParams.get<float>("value"));
723723

724724
if (scalarParams.has("nInputDim"))
725-
layerParams.set("input_dims",
726-
static_cast<int>(scalarParams.get<double>("nInputDim")));
725+
layerParams.set("input_dims", scalarParams.get<int>("nInputDim"));
727726

728-
if (scalarParams.has("value"))
729-
layerParams.set("value", scalarParams.get<double>("value"));
727+
int dim = scalarParams.get<int>("dim") - 1; // In Lua we start from 1.
728+
int pad = scalarParams.get<int>("pad");
730729

731-
if (scalarParams.has("index"))
732-
layerParams.set("index",
733-
static_cast<int>(scalarParams.get<double>("index") - 1));
730+
std::vector<int> paddings((dim + 1) * 2, 0);
731+
if (pad > 0)
732+
paddings[dim * 2 + 1] = pad; // Pad after (right).
733+
else
734+
paddings[dim * 2] = -pad; // Pad before (left).
735+
layerParams.set("paddings", DictValue::arrayInt<int*>(&paddings[0], paddings.size()));
734736

735737
curModule->modules.push_back(newModule);
736738
}
@@ -867,6 +869,31 @@ struct TorchImporter : public ::cv::dnn::Importer
867869
layerParams.set("scale", scalarParams.get<float>("constant_scalar"));
868870
curModule->modules.push_back(newModule);
869871
}
872+
else if (nnName == "SpatialZeroPadding")
873+
{
874+
readTorchTable(scalarParams, tensorParams);
875+
CV_Assert(scalarParams.has("pad_l"), scalarParams.has("pad_r"),
876+
scalarParams.has("pad_t"), scalarParams.has("pad_b"));
877+
int padTop = scalarParams.get<int>("pad_t");
878+
int padLeft = scalarParams.get<int>("pad_l");
879+
int padRight = scalarParams.get<int>("pad_r");
880+
int padBottom = scalarParams.get<int>("pad_b");
881+
if (padTop < 0 || padLeft < 0 || padRight < 0 || padBottom < 0)
882+
CV_Error(Error::StsNotImplemented, "SpatialZeroPadding in cropping mode is not implemented");
883+
884+
newModule->apiType = "Padding";
885+
886+
// Torch's SpatialZeroPadding works with 3- or 4-dimensional input.
887+
// So we add parameter input_dims=3 to ignore batch dimension if it will be.
888+
std::vector<int> paddings(6, 0); // CHW
889+
paddings[2] = padTop;
890+
paddings[3] = padBottom;
891+
paddings[4] = padLeft;
892+
paddings[5] = padRight;
893+
layerParams.set("paddings", DictValue::arrayInt<int*>(&paddings[0], paddings.size()));
894+
layerParams.set("input_dims", 3);
895+
curModule->modules.push_back(newModule);
896+
}
870897
else
871898
{
872899
CV_Error(Error::StsNotImplemented, "Unknown nn class \"" + className + "\"");

‎modules/dnn/test/test_halide_layers.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,28 @@ static void test(LayerParams& params, Mat& input)
3434
normAssert(outputDefault, outputHalide);
3535
}
3636

37+
////////////////////////////////////////////////////////////////////////////////
38+
// Padding
39+
////////////////////////////////////////////////////////////////////////////////
40+
TEST(Padding_Halide, Accuracy)
41+
{
42+
static const int kNumRuns = 10;
43+
std::vector<int> paddings(8);
44+
for (int t = 0; t < kNumRuns; ++t)
45+
{
46+
for (int i = 0; i < paddings.size(); ++i)
47+
paddings[i] = rand() % 5;
48+
49+
LayerParams lp;
50+
lp.set("paddings", DictValue::arrayInt<int*>(&paddings[0], paddings.size()));
51+
lp.type = "Padding";
52+
lp.name = "testLayer";
53+
54+
Mat input({1 + rand() % 10, 1 + rand() % 10, 1 + rand() % 10, 1 + rand() % 10}, CV_32F);
55+
test(lp, input);
56+
}
57+
}
58+
3759
////////////////////////////////////////////////////////////////////////////////
3860
// Convolution
3961
////////////////////////////////////////////////////////////////////////////////

‎modules/dnn/test/test_tf_importer.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ TEST(Test_TensorFlow, padding)
103103
{
104104
runTensorFlowNet("padding_same");
105105
runTensorFlowNet("padding_valid");
106+
runTensorFlowNet("spatial_padding");
106107
}
107108

108109
TEST(Test_TensorFlow, eltwise_add_mul)

‎modules/dnn/test/test_torch_importer.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,12 @@ TEST(Torch_Importer, net_normalize)
190190
runTorchNet("net_normalize", "", false, true);
191191
}
192192

193+
TEST(Torch_Importer, net_padding)
194+
{
195+
runTorchNet("net_padding", "", false, true);
196+
runTorchNet("net_spatial_zero_padding", "", false, true);
197+
}
198+
193199
TEST(Torch_Importer, ENet_accuracy)
194200
{
195201
Net net;

0 commit comments

Comments
 (0)