From b1dddf02ccf8dcfaadee4e8a5ed8726725ec1b93 Mon Sep 17 00:00:00 2001 From: AlexeyAB Date: Mon, 13 Aug 2018 02:43:45 +0300 Subject: [PATCH] Fixed AVX compiled bug --- Makefile | 2 +- src/convolutional_layer.c | 11 +++-- src/gemm.c | 90 +++++++++++++++++++++++++++------------ 3 files changed, 70 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index ba610865..ec74a813 100644 --- a/Makefile +++ b/Makefile @@ -57,7 +57,7 @@ ifeq ($(DEBUG), 1) OPTS= -O0 -g else ifeq ($(AVX), 1) -CFLAGS+= -ffp-contract=fast -mavx -msse4.1 -msse4a +CFLAGS+= -ffp-contract=fast -mavx -mavx2 -msse3 -msse4.1 -msse4.2 -msse4a endif endif diff --git a/src/convolutional_layer.c b/src/convolutional_layer.c index d236ec2e..927eb993 100644 --- a/src/convolutional_layer.c +++ b/src/convolutional_layer.c @@ -688,8 +688,8 @@ void forward_convolutional_layer(convolutional_layer l, network_state state) // t_input = calloc(t_intput_size, sizeof(float)); // im2col_cpu_custom_transpose(state.input, l.c, l.h, l.w, l.size, l.stride, l.pad, t_input, new_ldb); //} - if (l.xnor && l.size == 3 && l.stride == 1 && l.pad == 1) {} - else + //if (l.xnor && l.size == 3 && l.stride == 1 && l.pad == 1) {} + //else im2col_cpu_custom(state.input, l.c, l.h, l.w, l.size, l.stride, l.pad, b); @@ -772,7 +772,7 @@ void forward_convolutional_layer(convolutional_layer l, network_state state) } */ - + /* if (l.size == 3 && l.stride == 1 && l.pad == 1) { //binarize_weights(l.weights, l.n, l.c*l.size*l.size, l.binary_weights); @@ -783,12 +783,15 @@ void forward_convolutional_layer(convolutional_layer l, network_state state) l.binary_weights, state.input, l.output, l.mean_arr); } else { + */ //size_t ldb_align = 256; // 256 bit for AVX2 int ldb_align = l.lda_align; size_t new_ldb = k + (ldb_align - k%ldb_align); char *t_bit_input = NULL; size_t t_intput_size = binary_transpose_align_input(k, n, b, &t_bit_input, ldb_align); + //char *t_bit_input = calloc(new_ldb * n, sizeof(char)); // for im2col_cpu_custom_transpose() only + //float_to_bit(t_input, t_bit_input, new_ldb * n); // for im2col_cpu_custom_transpose() only gemm_nn_custom_bin_mean_transposed(m, n, k, 1, l.align_bit_weights, new_ldb, t_bit_input, new_ldb, c, n, l.mean_arr); @@ -796,7 +799,7 @@ void forward_convolutional_layer(convolutional_layer l, network_state state) //free(t_input); free(t_bit_input); - } + //} } diff --git a/src/gemm.c b/src/gemm.c index 580f4da2..b9098141 100644 --- a/src/gemm.c +++ b/src/gemm.c @@ -318,6 +318,22 @@ void gemm_nn_custom_bin_mean_transposed(int M, int N, int K, float ALPHA_UNUSED, #include #include +static inline __int32 _mm256_extract_epi64(__m256i a, const int index) { + return a.m256i_i64[index]; +} + +static inline __int32 _mm256_extract_epi32(__m256i a, const int index) { + return a.m256i_i32[index]; +} + +static inline float _castu32_f32(uint32_t a) { + return *((float *)&a); +} + +static inline float _mm256_extract_float32(__m256 a, const int index) { + return _castu32_f32(_mm256_extract_epi32(_mm256_castps_si256(a), index)); +} + #else // Linux GCC/Clang #include #include @@ -325,6 +341,14 @@ void gemm_nn_custom_bin_mean_transposed(int M, int N, int K, float ALPHA_UNUSED, #include #include +static inline float _castu32_f32(uint32_t a) { + return *((float *)&a); +} + +static inline float _mm256_extract_float32(__m256 a, const int index) { + return _castu32_f32(_mm256_extract_epi32(_mm256_castps_si256(a), index)); +} + void asm_cpuid(uint32_t* abcd, uint32_t eax) { uint32_t ebx = 0, edx = 0, ecx = 0; @@ -504,8 +528,10 @@ void convolution_2d(int w, int h, int ksize, int n, int c, int pad, int stride, } - __m256i all256_last_zero = _mm256_set1_epi32(0xFFFFFFFF); - all256_last_zero.m256i_i32[7] = 0; + //__m256i all256_last_zero = _mm256_set1_epi32(0xFFFFFFFF); + //all256_last_zero.m256i_i32[7] = 0; + __m256i all256_last_zero = + _mm256_set_epi32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x0); __m256i idx256 = _mm256_set_epi32(0, 7, 6, 5, 4, 3, 2, 1); //__m256 all256_sing1 = _mm256_set1_ps(0x80000000); @@ -561,15 +587,15 @@ void convolution_2d(int w, int h, int ksize, int n, int c, int pad, int stride, __m256 in = *((__m256*)&input[input_index]); __m256 w = _mm256_set1_ps(weights[weights_index]); //__m256 w_sign = _mm256_and_ps(w, _mm256_castsi256_ps(all256_sing1)); // check sign in 8 x 32-bit floats - __m256 xor = _mm256_xor_ps(w, in); - //printf("\n xor1 = %f, xor2 = %f \n", xor.m256_f32[0], xor.m256_f32[1]); - //printf("\n in = %f, w = %f, xor = %f \n", in.m256_f32[0], w_sign.m256_f32[0], xor.m256_f32[0]); + __m256 xor256 = _mm256_xor_ps(w, in); + //printf("\n xor256_1 = %f, xor256_2 = %f \n", xor256.m256_f32[0], xor256.m256_f32[1]); + //printf("\n in = %f, w = %f, xor256 = %f \n", in.m256_f32[0], w_sign.m256_f32[0], xor256.m256_f32[0]); - //__m256 pn1 = _mm256_and_ps(_mm256_castsi256_ps(all256i_one), xor); + //__m256 pn1 = _mm256_and_ps(_mm256_castsi256_ps(all256i_one), xor256); - //sum256 = xor; - sum256 = _mm256_add_ps(xor, sum256); + //sum256 = xor256; + sum256 = _mm256_add_ps(xor256, sum256); //printf("\n --- \n"); //printf("\n 0 = %f, 1 = %f, 2 = %f, 3 = %f, 4 = %f, 5 = %f, 6 = %f, 7 = %f \n", in.m256_f32[0], in.m256_f32[1], in.m256_f32[2], in.m256_f32[3], in.m256_f32[4], in.m256_f32[5], in.m256_f32[6], in.m256_f32[7]); @@ -638,10 +664,14 @@ static inline __m256i count256(__m256i v) { static inline int popcnt256_custom(__m256i n) { __m256i val = count256(n); - return val.m256i_i64[0] + - val.m256i_i64[1] + - val.m256i_i64[2] + - val.m256i_i64[3]; + //return val.m256i_i64[0] + + //val.m256i_i64[1] + + //val.m256i_i64[2] + + //val.m256i_i64[3]; + return _mm256_extract_epi64(val, 0) + + _mm256_extract_epi64(val, 1) + + _mm256_extract_epi64(val, 2) + + _mm256_extract_epi64(val, 3); } void gemm_nn_custom_bin_mean_transposed(int M, int N, int K, float ALPHA_UNUSED, @@ -686,10 +716,14 @@ void gemm_nn_custom_bin_mean_transposed(int M, int N, int K, float ALPHA_UNUSED, } // count of 1 bits - count = count_sum.m256i_i64[0] + - count_sum.m256i_i64[1] + - count_sum.m256i_i64[2] + - count_sum.m256i_i64[3]; + //count = count_sum.m256i_i64[0] + + // count_sum.m256i_i64[1] + + // count_sum.m256i_i64[2] + + // count_sum.m256i_i64[3]; + count = _mm256_extract_epi64(count_sum, 0) + + _mm256_extract_epi64(count_sum, 1) + + _mm256_extract_epi64(count_sum, 2) + + _mm256_extract_epi64(count_sum, 3); int f1 = (K % bit_step == 0) ? 0 : (bit_step - (K % bit_step)); count = count - f1; // remove extra bits (from empty space for align only) @@ -738,15 +772,15 @@ void im2col_cpu_custom_transpose(float* data_im, int col_index = (h * width_col + w)*ldb_align + c; // transposed & aligned //data_col[col_index] = data_im[im_col + width*(im_row + height*c_im)]; - __m256 src256 = _mm256_loadu_ps((__m256i *)(&data_im[im_col + width*(im_row + height*c_im)])); - data_col[col_index + ldb_align * 0] = src256.m256_f32[0]; - data_col[col_index + ldb_align * 1] = src256.m256_f32[1]; - data_col[col_index + ldb_align * 2] = src256.m256_f32[2]; - data_col[col_index + ldb_align * 3] = src256.m256_f32[3]; - data_col[col_index + ldb_align * 4] = src256.m256_f32[4]; - data_col[col_index + ldb_align * 5] = src256.m256_f32[5]; - data_col[col_index + ldb_align * 6] = src256.m256_f32[6]; - data_col[col_index + ldb_align * 7] = src256.m256_f32[7]; + __m256 src256 = _mm256_loadu_ps((float *)(&data_im[im_col + width*(im_row + height*c_im)])); + data_col[col_index + ldb_align * 0] = _mm256_extract_float32(src256, 0);// src256.m256_f32[0]; + data_col[col_index + ldb_align * 1] = _mm256_extract_float32(src256, 1);// src256.m256_f32[1]; + data_col[col_index + ldb_align * 2] = _mm256_extract_float32(src256, 2);// src256.m256_f32[2]; + data_col[col_index + ldb_align * 3] = _mm256_extract_float32(src256, 3);// src256.m256_f32[3]; + data_col[col_index + ldb_align * 4] = _mm256_extract_float32(src256, 4);// src256.m256_f32[4]; + data_col[col_index + ldb_align * 5] = _mm256_extract_float32(src256, 5);// src256.m256_f32[5]; + data_col[col_index + ldb_align * 6] = _mm256_extract_float32(src256, 6);// src256.m256_f32[6]; + data_col[col_index + ldb_align * 7] = _mm256_extract_float32(src256, 7);// src256.m256_f32[7]; //_mm256_storeu_ps(&data_col[col_index], src256); } @@ -853,7 +887,7 @@ void im2col_cpu_custom(float* data_im, int col_index = (c * height_col + h) * width_col + w; //data_col[col_index] = data_im[im_col + width*(im_row + height*c_im)]; - __m256 src256 = _mm256_loadu_ps((__m256i *)(&data_im[im_col + width*(im_row + height*c_im)])); + __m256 src256 = _mm256_loadu_ps((float *)(&data_im[im_col + width*(im_row + height*c_im)])); _mm256_storeu_ps(&data_col[col_index], src256); } @@ -931,13 +965,13 @@ void activate_array_cpu_custom(float *x, const int n, const ACTIVATION a) for (i = 0; i < n-8; i += 8) { //x[i] = (x[i]>0) ? x[i] : .1*x[i]; - __m256 src256 = _mm256_loadu_ps((__m256 *)(&x[i])); + __m256 src256 = _mm256_loadu_ps(&x[i]); __m256 mult256 = _mm256_mul_ps((src256), all256_01); // mult * 0.1 __m256i sign256 = _mm256_and_si256(_mm256_castps_si256(src256), all256_sing1); // check sign in 8 x 32-bit floats __m256 result256 = _mm256_blendv_ps(src256, mult256, _mm256_castsi256_ps(sign256)); // (sign>0) ? src : mult; - _mm256_storeu_ps((__m256 *)(&x[i]), result256); + _mm256_storeu_ps(&x[i], result256); } for (; i < n; ++i) {