Skip to content

Commit a65b5df

Browse files
alalekalalek
committed
Merge pull request opencv#10416 from fenrus75:avx512
2 parents 2370c8a + 898ca38 commit a65b5df

File tree

7 files changed

+129
-34
lines changed

7 files changed

+129
-34
lines changed

‎cmake/OpenCVCompilerOptimizations.cmake

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# SSE / SSE2 (always available on 64-bit CPUs)
33
# SSE3 / SSSE3
44
# SSE4_1 / SSE4_2 / POPCNT
5-
# AVX / AVX2 / AVX512
5+
# AVX / AVX2 / AVX_512F
66
# FMA3
77

88
# CPU_{opt}_SUPPORTED=ON/OFF - compiler support (possibly with additional flag)
@@ -26,7 +26,7 @@
2626
#
2727
# CPU_DISPATCH_FLAGS_${opt} - flags for source files compiled separately (<name>.avx2.cpp)
2828

29-
set(CPU_ALL_OPTIMIZATIONS "SSE;SSE2;SSE3;SSSE3;SSE4_1;SSE4_2;POPCNT;AVX;FP16;AVX2;FMA3") # without AVX512
29+
set(CPU_ALL_OPTIMIZATIONS "SSE;SSE2;SSE3;SSSE3;SSE4_1;SSE4_2;POPCNT;AVX;FP16;AVX2;FMA3;AVX_512F")
3030
list(APPEND CPU_ALL_OPTIMIZATIONS NEON VFPV3 FP16)
3131
list(APPEND CPU_ALL_OPTIMIZATIONS VSX)
3232
list(REMOVE_DUPLICATES CPU_ALL_OPTIMIZATIONS)
@@ -145,7 +145,7 @@ elseif(" ${CMAKE_CXX_FLAGS} " MATCHES " -march=native | -xHost | /QxHost ")
145145
endif()
146146

147147
if(X86 OR X86_64)
148-
ocv_update(CPU_KNOWN_OPTIMIZATIONS "SSE;SSE2;SSE3;SSSE3;SSE4_1;POPCNT;SSE4_2;FP16;FMA3;AVX;AVX2") # without AVX512
148+
ocv_update(CPU_KNOWN_OPTIMIZATIONS "SSE;SSE2;SSE3;SSSE3;SSE4_1;POPCNT;SSE4_2;FP16;FMA3;AVX;AVX2;AVX_512F")
149149

150150
ocv_update(CPU_SSE_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_sse.cpp")
151151
ocv_update(CPU_SSE2_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_sse2.cpp")
@@ -157,11 +157,11 @@ if(X86 OR X86_64)
157157
ocv_update(CPU_AVX_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_avx.cpp")
158158
ocv_update(CPU_AVX2_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_avx2.cpp")
159159
ocv_update(CPU_FP16_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_fp16.cpp")
160-
ocv_update(CPU_AVX512_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_avx512.cpp")
160+
ocv_update(CPU_AVX_512F_TEST_FILE "${OpenCV_SOURCE_DIR}/cmake/checks/cpu_avx512.cpp")
161161

162162
if(NOT OPENCV_CPU_OPT_IMPLIES_IGNORE)
163-
ocv_update(CPU_AVX512_IMPLIES "AVX2")
164-
ocv_update(CPU_AVX512_FORCE "") # Don't force other optimizations
163+
ocv_update(CPU_AVX_512F_IMPLIES "AVX2")
164+
ocv_update(CPU_AVX_512F_FORCE "") # Don't force other optimizations
165165
ocv_update(CPU_AVX2_IMPLIES "AVX;FMA3;FP16")
166166
ocv_update(CPU_FMA3_IMPLIES "AVX2")
167167
ocv_update(CPU_FMA3_FORCE "") # Don't force other optimizations
@@ -205,7 +205,7 @@ if(X86 OR X86_64)
205205
if(NOT X86_64) # x64 compiler doesn't support /arch:sse
206206
ocv_intel_compiler_optimization_option(SSE "-msse" "/arch:SSE")
207207
endif()
208-
#ocv_intel_compiler_optimization_option(AVX512 "-march=core-avx512")
208+
ocv_intel_compiler_optimization_option(AVX_512F "-march=common-avx512" "/arch:COMMON-AVX512")
209209
elseif(CMAKE_COMPILER_IS_GNUCXX)
210210
ocv_update(CPU_AVX2_FLAGS_ON "-mavx2")
211211
ocv_update(CPU_FP16_FLAGS_ON "-mf16c")
@@ -219,7 +219,8 @@ if(X86 OR X86_64)
219219
ocv_update(CPU_SSE2_FLAGS_ON "-msse2")
220220
ocv_update(CPU_SSE_FLAGS_ON "-msse")
221221
if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0")
222-
ocv_update(CPU_AVX512_FLAGS_ON "-mavx512f -mavx512pf -mavx512er -mavx512cd -mavx512vl -mavx512bw -mavx512dq -mavx512ifma -mavx512vbmi")
222+
# -mavx512f -mavx512pf -mavx512er -mavx512cd -mavx512vl -mavx512bw -mavx512dq -mavx512ifma -mavx512vbmi
223+
ocv_update(CPU_AVX_512F_FLAGS_ON "-mavx512f")
223224
endif()
224225
elseif(MSVC)
225226
ocv_update(CPU_AVX2_FLAGS_ON "/arch:AVX2")

‎modules/core/include/opencv2/core/cv_cpu_dispatch.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@
8282
# include <immintrin.h>
8383
# define CV_AVX2 1
8484
#endif
85+
#ifdef CV_CPU_COMPILE_AVX_512F
86+
# include <immintrin.h>
87+
# define CV_AVX_512F 1
88+
#endif
8589
#ifdef CV_CPU_COMPILE_FMA3
8690
# define CV_FMA3 1
8791
#endif

‎modules/core/include/opencv2/core/cv_cpu_helper.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,21 @@
165165
#endif
166166
#define __CV_CPU_DISPATCH_CHAIN_FMA3(fn, args, mode, ...) CV_CPU_CALL_FMA3(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
167167

168+
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_AVX_512F
169+
# define CV_TRY_AVX_512F 1
170+
# define CV_CPU_HAS_SUPPORT_AVX_512F 1
171+
# define CV_CPU_CALL_AVX_512F(fn, args) return (opt_AVX_512F::fn args)
172+
#elif !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_DISPATCH_COMPILE_AVX_512F
173+
# define CV_TRY_AVX_512F 1
174+
# define CV_CPU_HAS_SUPPORT_AVX_512F (cv::checkHardwareSupport(CV_CPU_AVX_512F))
175+
# define CV_CPU_CALL_AVX_512F(fn, args) if (CV_CPU_HAS_SUPPORT_AVX_512F) return (opt_AVX_512F::fn args)
176+
#else
177+
# define CV_TRY_AVX_512F 0
178+
# define CV_CPU_HAS_SUPPORT_AVX_512F 0
179+
# define CV_CPU_CALL_AVX_512F(fn, args)
180+
#endif
181+
#define __CV_CPU_DISPATCH_CHAIN_AVX_512F(fn, args, mode, ...) CV_CPU_CALL_AVX_512F(fn, args); __CV_EXPAND(__CV_CPU_DISPATCH_CHAIN_ ## mode(fn, args, __VA_ARGS__))
182+
168183
#if !defined CV_DISABLE_OPTIMIZATION && defined CV_ENABLE_INTRINSICS && defined CV_CPU_COMPILE_NEON
169184
# define CV_TRY_NEON 1
170185
# define CV_CPU_HAS_SUPPORT_NEON 1

‎modules/dnn/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ endif()
1313

1414
set(the_description "Deep neural network module. It allows to load models from different frameworks and to make forward pass")
1515

16-
ocv_add_dispatched_file("layers/layers_common" AVX AVX2)
16+
ocv_add_dispatched_file("layers/layers_common" AVX AVX2 AVX_512F)
1717

1818
ocv_add_module(dnn opencv_core opencv_imgproc WRAP python matlab java js)
1919
ocv_warnings_disable(CMAKE_CXX_FLAGS -Wno-shadow -Wno-parentheses -Wmaybe-uninitialized -Wsign-promo

‎modules/dnn/src/layers/convolution_layer.cpp

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,10 +345,11 @@ class ConvolutionLayerImpl : public BaseConvolutionLayerImpl
345345
bool is1x1_;
346346
bool useAVX;
347347
bool useAVX2;
348+
bool useAVX512;
348349

349350
ParallelConv()
350351
: input_(0), weights_(0), output_(0), ngroups_(0), nstripes_(0),
351-
biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false)
352+
biasvec_(0), reluslope_(0), activ_(0), is1x1_(false), useAVX(false), useAVX2(false), useAVX512(false)
352353
{}
353354

354355
static void run( const Mat& input, Mat& output, const Mat& weights,
@@ -383,6 +384,7 @@ class ConvolutionLayerImpl : public BaseConvolutionLayerImpl
383384
p.is1x1_ = kernel == Size(0,0) && pad == Size(0, 0);
384385
p.useAVX = checkHardwareSupport(CPU_AVX);
385386
p.useAVX2 = checkHardwareSupport(CPU_AVX2);
387+
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX_512F;
386388

387389
int ncn = std::min(inpCn, (int)BLK_SIZE_CN);
388390
p.ofstab_.resize(kernel.width*kernel.height*ncn);
@@ -562,6 +564,13 @@ class ConvolutionLayerImpl : public BaseConvolutionLayerImpl
562564
// now compute dot product of the weights
563565
// and im2row-transformed part of the tensor
564566
int bsz = ofs1 - ofs0;
567+
#if CV_TRY_AVX_512F
568+
/* AVX512 convolution requires an alignment of 16, and ROI is only there for larger vector sizes */
569+
if(useAVX512)
570+
opt_AVX_512F::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
571+
outShape, bsz, vsz, vsz_a, relu, cn0 == 0);
572+
else
573+
#endif
565574
#if CV_TRY_AVX2
566575
if(useAVX2)
567576
opt_AVX2::fastConv(wptr, wstep, biasptr, rowbuf0, data_out0 + ofs0,
@@ -1093,6 +1102,7 @@ class DeConvolutionLayerImpl : public BaseConvolutionLayerImpl
10931102
nstripes_ = nstripes;
10941103
useAVX = checkHardwareSupport(CPU_AVX);
10951104
useAVX2 = checkHardwareSupport(CPU_AVX2);
1105+
useAVX512 = CV_CPU_HAS_SUPPORT_AVX_512F;
10961106
}
10971107

10981108
void operator()(const Range& range_) const
@@ -1110,6 +1120,11 @@ class DeConvolutionLayerImpl : public BaseConvolutionLayerImpl
11101120
size_t bstep = b_->step1();
11111121
size_t cstep = c_->step1();
11121122

1123+
#if CV_TRY_AVX_512F
1124+
if( useAVX512 )
1125+
opt_AVX_512F::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
1126+
else
1127+
#endif
11131128
#if CV_TRY_AVX2
11141129
if( useAVX2 )
11151130
opt_AVX2::fastGEMM( aptr, astep, bptr, bstep, cptr, cstep, mmax, kmax, nmax );
@@ -1214,6 +1229,7 @@ class DeConvolutionLayerImpl : public BaseConvolutionLayerImpl
12141229
int nstripes_;
12151230
bool useAVX;
12161231
bool useAVX2;
1232+
bool useAVX512;
12171233
};
12181234

12191235
class Col2ImInvoker : public cv::ParallelLoopBody

‎modules/dnn/src/layers/fully_connected_layer.cpp

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ class FullyConnectedLayerImpl : public InnerProductLayer
139139
class FullyConnected : public ParallelLoopBody
140140
{
141141
public:
142-
FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false) {}
142+
FullyConnected() : srcMat(0), weights(0), biasMat(0), activ(0), dstMat(0), nstripes(0), useAVX(false), useAVX2(false), useAVX512(false) {}
143143

144144
static void run(const Mat& srcMat, const Mat& weights, const Mat& biasMat,
145145
Mat& dstMat, const ActivationLayer* activ, int nstripes)
@@ -161,6 +161,7 @@ class FullyConnectedLayerImpl : public InnerProductLayer
161161
p.activ = activ;
162162
p.useAVX = checkHardwareSupport(CPU_AVX);
163163
p.useAVX2 = checkHardwareSupport(CPU_AVX2);
164+
p.useAVX512 = CV_CPU_HAS_SUPPORT_AVX_512F;
164165

165166
parallel_for_(Range(0, nstripes), p, nstripes);
166167
}
@@ -195,6 +196,11 @@ class FullyConnectedLayerImpl : public InnerProductLayer
195196

196197
memcpy(sptr, sptr_, vecsize*sizeof(sptr[0]));
197198

199+
#if CV_TRY_AVX_512F
200+
if( useAVX512 )
201+
opt_AVX_512F::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize);
202+
else
203+
#endif
198204
#if CV_TRY_AVX2
199205
if( useAVX2 )
200206
opt_AVX2::fastGEMM1T( sptr, wptr, wstep, biasptr, dptr, nw, vecsize);
@@ -255,6 +261,7 @@ class FullyConnectedLayerImpl : public InnerProductLayer
255261
int nstripes;
256262
bool useAVX;
257263
bool useAVX2;
264+
bool useAVX512;
258265
};
259266

260267
#ifdef HAVE_OPENCL

‎modules/dnn/src/layers/layers_common.simd.hpp

Lines changed: 75 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ void fastConv( const float* weights, size_t wstep, const float* bias,
7272
int outCn = outShape[1];
7373
size_t outPlaneSize = outShape[2]*outShape[3];
7474
float r0 = 1.f, r1 = 1.f, r2 = 1.f;
75-
__m256 vr0 = _mm256_set1_ps(1.f), vr1 = vr0, vr2 = vr0, z = _mm256_setzero_ps();
75+
__m128 vr0 = _mm_set1_ps(1.f), vr1 = vr0, vr2 = vr0, z = _mm_setzero_ps();
7676

7777
// now compute dot product of the weights
7878
// and im2row-transformed part of the tensor
@@ -104,9 +104,9 @@ void fastConv( const float* weights, size_t wstep, const float* bias,
104104
r0 = relu[i];
105105
r1 = relu[i+1];
106106
r2 = relu[i+2];
107-
vr0 = _mm256_set1_ps(r0);
108-
vr1 = _mm256_set1_ps(r1);
109-
vr2 = _mm256_set1_ps(r2);
107+
vr0 = _mm_set1_ps(r0);
108+
vr1 = _mm_set1_ps(r1);
109+
vr2 = _mm_set1_ps(r2);
110110
}
111111

112112
int j = 0;
@@ -156,38 +156,38 @@ void fastConv( const float* weights, size_t wstep, const float* bias,
156156
t1 = _mm256_add_ps(t1, _mm256_permute2f128_ps(t1, t1, 1));
157157
t2 = _mm256_add_ps(t2, _mm256_permute2f128_ps(t2, t2, 1));
158158

159-
__m256 s0, s1, s2;
159+
__m128 s0, s1, s2;
160160

161161
if( initOutput )
162162
{
163-
s0 = _mm256_set1_ps(bias0);
164-
s1 = _mm256_set1_ps(bias1);
165-
s2 = _mm256_set1_ps(bias2);
163+
s0 = _mm_set1_ps(bias0);
164+
s1 = _mm_set1_ps(bias1);
165+
s2 = _mm_set1_ps(bias2);
166166
}
167167
else
168168
{
169-
s0 = _mm256_castps128_ps256(_mm_loadu_ps(outptr0 + j));
170-
s1 = _mm256_castps128_ps256(_mm_loadu_ps(outptr1 + j));
171-
s2 = _mm256_castps128_ps256(_mm_loadu_ps(outptr2 + j));
169+
s0 = _mm_loadu_ps(outptr0 + j);
170+
s1 = _mm_loadu_ps(outptr1 + j);
171+
s2 = _mm_loadu_ps(outptr2 + j);
172172
}
173173

174-
s0 = _mm256_add_ps(s0, t0);
175-
s1 = _mm256_add_ps(s1, t1);
176-
s2 = _mm256_add_ps(s2, t2);
174+
s0 = _mm_add_ps(s0, _mm256_castps256_ps128(t0));
175+
s1 = _mm_add_ps(s1, _mm256_castps256_ps128(t1));
176+
s2 = _mm_add_ps(s2, _mm256_castps256_ps128(t2));
177177

178178
if( relu )
179179
{
180-
__m256 m0 = _mm256_cmp_ps(s0, z, _CMP_GT_OS);
181-
__m256 m1 = _mm256_cmp_ps(s1, z, _CMP_GT_OS);
182-
__m256 m2 = _mm256_cmp_ps(s2, z, _CMP_GT_OS);
183-
s0 = _mm256_xor_ps(s0, _mm256_andnot_ps(m0, _mm256_xor_ps(_mm256_mul_ps(s0, vr0), s0)));
184-
s1 = _mm256_xor_ps(s1, _mm256_andnot_ps(m1, _mm256_xor_ps(_mm256_mul_ps(s1, vr1), s1)));
185-
s2 = _mm256_xor_ps(s2, _mm256_andnot_ps(m2, _mm256_xor_ps(_mm256_mul_ps(s2, vr2), s2)));
180+
__m128 m0 = _mm_cmp_ps(s0, z, _CMP_GT_OS);
181+
__m128 m1 = _mm_cmp_ps(s1, z, _CMP_GT_OS);
182+
__m128 m2 = _mm_cmp_ps(s2, z, _CMP_GT_OS);
183+
s0 = _mm_xor_ps(s0, _mm_andnot_ps(m0, _mm_xor_ps(_mm_mul_ps(s0, vr0), s0)));
184+
s1 = _mm_xor_ps(s1, _mm_andnot_ps(m1, _mm_xor_ps(_mm_mul_ps(s1, vr1), s1)));
185+
s2 = _mm_xor_ps(s2, _mm_andnot_ps(m2, _mm_xor_ps(_mm_mul_ps(s2, vr2), s2)));
186186
}
187187

188-
_mm_storeu_ps(outptr0 + j, _mm256_castps256_ps128(s0));
189-
_mm_storeu_ps(outptr1 + j, _mm256_castps256_ps128(s1));
190-
_mm_storeu_ps(outptr2 + j, _mm256_castps256_ps128(s2));
188+
_mm_storeu_ps(outptr0 + j, s0);
189+
_mm_storeu_ps(outptr1 + j, s1);
190+
_mm_storeu_ps(outptr2 + j, s2);
191191
}
192192

193193
for( ; j < blockSize; j++ )
@@ -294,11 +294,63 @@ void fastGEMM1T( const float* vec, const float* weights,
294294
_mm256_zeroupper();
295295
}
296296

297+
297298
void fastGEMM( const float* aptr, size_t astep, const float* bptr,
298299
size_t bstep, float* cptr, size_t cstep,
299300
int ma, int na, int nb )
300301
{
301302
int n = 0;
303+
304+
#if CV_AVX_512F
305+
for( ; n <= nb - 32; n += 32 )
306+
{
307+
for( int m = 0; m < ma; m += 4 )
308+
{
309+
const float* aptr0 = aptr + astep*m;
310+
const float* aptr1 = aptr + astep*std::min(m+1, ma-1);
311+
const float* aptr2 = aptr + astep*std::min(m+2, ma-1);
312+
const float* aptr3 = aptr + astep*std::min(m+3, ma-1);
313+
314+
float* cptr0 = cptr + cstep*m;
315+
float* cptr1 = cptr + cstep*std::min(m+1, ma-1);
316+
float* cptr2 = cptr + cstep*std::min(m+2, ma-1);
317+
float* cptr3 = cptr + cstep*std::min(m+3, ma-1);
318+
319+
__m512 d00 = _mm512_setzero_ps(), d01 = _mm512_setzero_ps();
320+
__m512 d10 = _mm512_setzero_ps(), d11 = _mm512_setzero_ps();
321+
__m512 d20 = _mm512_setzero_ps(), d21 = _mm512_setzero_ps();
322+
__m512 d30 = _mm512_setzero_ps(), d31 = _mm512_setzero_ps();
323+
324+
for( int k = 0; k < na; k++ )
325+
{
326+
__m512 a0 = _mm512_set1_ps(aptr0[k]);
327+
__m512 a1 = _mm512_set1_ps(aptr1[k]);
328+
__m512 a2 = _mm512_set1_ps(aptr2[k]);
329+
__m512 a3 = _mm512_set1_ps(aptr3[k]);
330+
__m512 b0 = _mm512_loadu_ps(bptr + k*bstep + n);
331+
__m512 b1 = _mm512_loadu_ps(bptr + k*bstep + n + 16);
332+
d00 = _mm512_fmadd_ps(a0, b0, d00);
333+
d01 = _mm512_fmadd_ps(a0, b1, d01);
334+
d10 = _mm512_fmadd_ps(a1, b0, d10);
335+
d11 = _mm512_fmadd_ps(a1, b1, d11);
336+
d20 = _mm512_fmadd_ps(a2, b0, d20);
337+
d21 = _mm512_fmadd_ps(a2, b1, d21);
338+
d30 = _mm512_fmadd_ps(a3, b0, d30);
339+
d31 = _mm512_fmadd_ps(a3, b1, d31);
340+
}
341+
342+
_mm512_storeu_ps(cptr0 + n, d00);
343+
_mm512_storeu_ps(cptr0 + n + 16, d01);
344+
_mm512_storeu_ps(cptr1 + n, d10);
345+
_mm512_storeu_ps(cptr1 + n + 16, d11);
346+
_mm512_storeu_ps(cptr2 + n, d20);
347+
_mm512_storeu_ps(cptr2 + n + 16, d21);
348+
_mm512_storeu_ps(cptr3 + n, d30);
349+
_mm512_storeu_ps(cptr3 + n + 16, d31);
350+
}
351+
}
352+
#endif
353+
302354
for( ; n <= nb - 16; n += 16 )
303355
{
304356
for( int m = 0; m < ma; m += 4 )

0 commit comments

Comments
 (0)