From a9fef1bd66e6b2c40c344c1bdcd33bb1d209461c Mon Sep 17 00:00:00 2001 From: AlexeyAB Date: Sat, 11 Aug 2018 00:26:53 +0300 Subject: [PATCH] Bug fixes. Tested im2col_cpu_custom_transpose - bad way. --- src/convolutional_layer.c | 43 ++++++-- src/convolutional_layer.h | 2 +- src/gemm.c | 223 +++++++++++++++++++++++++++++++++++++- src/gemm.h | 7 ++ src/layer.h | 1 + src/network.c | 4 +- 6 files changed, 264 insertions(+), 16 deletions(-) diff --git a/src/convolutional_layer.c b/src/convolutional_layer.c index 3c9efdd4..7dc7dd29 100644 --- a/src/convolutional_layer.c +++ b/src/convolutional_layer.c @@ -593,11 +593,11 @@ void bit_to_float(unsigned char *src, float *dst, size_t size, size_t filters, f } } -void binary_align_weights(convolutional_layer *l, size_t lda_align) +void binary_align_weights(convolutional_layer *l) { int m = l->n; int k = l->size*l->size*l->c; - size_t new_lda = k + (lda_align - k%lda_align); // (k / 8 + 1) * 8; + size_t new_lda = k + (l->lda_align - k % l->lda_align); // (k / 8 + 1) * 8; binarize_weights(l->weights, m, k, l->binary_weights); @@ -680,7 +680,17 @@ void forward_convolutional_layer(convolutional_layer l, network_state state) for(i = 0; i < l.batch; ++i){ //im2col_cpu(state.input, l.c, l.h, l.w, l.size, l.stride, l.pad, b); - im2col_cpu_custom(state.input, l.c, l.h, l.w, l.size, l.stride, l.pad, b); + + //float *t_input = NULL; + //if (l.xnor) { + // size_t new_ldb = k + (l.lda_align - k%l.lda_align); + // size_t t_intput_size = new_ldb * n; + // t_input = calloc(t_intput_size, sizeof(float)); + // im2col_cpu_custom_transpose(state.input, l.c, l.h, l.w, l.size, l.stride, l.pad, t_input, new_ldb); + //} + //else + im2col_cpu_custom(state.input, l.c, l.h, l.w, l.size, l.stride, l.pad, b); + //gemm(0,0,m,n,k,1,a,k,b,n,1,c,n); //gemm_nn_custom(m, n, k, 1, a, k, b, n, c, n); @@ -760,19 +770,28 @@ void forward_convolutional_layer(convolutional_layer l, network_state state) free(align_weights); } */ - size_t ldb_align = 256; // 256 bit for AVX2 - size_t new_ldb = k + (ldb_align - k%ldb_align); - char *t_bit_input = NULL; - size_t t_intput_size = binary_transpose_align_input(k, n, b, &t_bit_input, ldb_align); - gemm_nn_custom_bin_mean_transposed(m, n, k, 1, l.align_bit_weights, new_ldb, t_bit_input, new_ldb, c, n, l.mean_arr); + /* + if (l.size == 3 && l.stride == 1 && l.pad == 1) { + convolution_2d(l.w, l.h, l.size, l.n, l.c, l.pad, l.stride, + l.weights, state.input, l.output); + } + else { + */ + //size_t ldb_align = 256; // 256 bit for AVX2 + int ldb_align = l.lda_align; + size_t new_ldb = k + (ldb_align - k%ldb_align); + char *t_bit_input = NULL; + size_t t_intput_size = binary_transpose_align_input(k, n, b, &t_bit_input, ldb_align); - //gemm_nn_custom_bin_mean_transposed(m, n, k, 1, bit_weights, k, t_bit_input, new_ldb, c, n, mean_arr); + gemm_nn_custom_bin_mean_transposed(m, n, k, 1, l.align_bit_weights, new_ldb, t_bit_input, new_ldb, c, n, l.mean_arr); - //free(t_input); - free(t_bit_input); + //gemm_nn_custom_bin_mean_transposed(m, n, k, 1, bit_weights, k, t_bit_input, new_ldb, c, n, mean_arr); + + //free(t_input); + free(t_bit_input); + //} - //free(align_bit_weights); } // for bit_input: (k * n) diff --git a/src/convolutional_layer.h b/src/convolutional_layer.h index b804afb8..8869a3c3 100644 --- a/src/convolutional_layer.h +++ b/src/convolutional_layer.h @@ -35,7 +35,7 @@ void binarize_weights(float *weights, int n, int size, float *binary); void swap_binary(convolutional_layer *l); void binarize_weights2(float *weights, int n, int size, char *binary, float *scales); -void binary_align_weights(convolutional_layer *l, size_t ldb_align); +void binary_align_weights(convolutional_layer *l); void backward_convolutional_layer(convolutional_layer layer, network_state state); diff --git a/src/gemm.c b/src/gemm.c index 4a7dad7e..75ce59c2 100644 --- a/src/gemm.c +++ b/src/gemm.c @@ -429,6 +429,56 @@ void gemm_nn(int M, int N, int K, float ALPHA, } +void convolution_2d(int w, int h, int ksize, int n, int c, int pad, int stride, + float *weights, float *input, float *output) +{ + int out_h = (h + 2 * pad - ksize) / stride + 1; // output_height=input_height for stride=1 and pad=1 + int out_w = (w + 2 * pad - ksize) / stride + 1; // output_width=input_width for stride=1 and pad=1 + int i, f, j; + + int fil; + // filter index +#pragma omp parallel for // "omp parallel for" - automatic parallelization of loop by using OpenMP + for (fil = 0; fil < n; ++fil) { + int chan, y, x, f_y, f_x; + // channel index + for (chan = 0; chan < c; ++chan) + // input - y + for (y = 0; y < h; ++y) + // input - x + for (x = 0; x < w; ++x) + { + int const output_index = fil*w*h + y*w + x; + int const weights_pre_index = fil*c*ksize*ksize + chan*ksize*ksize; + int const input_pre_index = chan*w*h; + float sum = 0; + + // filter - y + for (f_y = 0; f_y < ksize; ++f_y) + { + int input_y = y + f_y - pad; + // filter - x + for (f_x = 0; f_x < ksize; ++f_x) + { + int input_x = x + f_x - pad; + if (input_y < 0 || input_x < 0 || input_y >= h || input_x >= w) continue; + + int input_index = input_pre_index + input_y*w + input_x; + int weights_index = weights_pre_index + f_y*ksize + f_x; + + sum += input[input_index] * weights[weights_index]; + } + } + // l.output[filters][width][height] += + // state.input[channels][width][height] * + // l.weights[filters][channels][filter_width][filter_height]; + output[output_index] += sum; + } + } +} + + + // http://graphics.stanford.edu/~seander/bithacks.html // https://stackoverflow.com/questions/17354971/fast-counting-the-number-of-set-bits-in-m128i-register // https://arxiv.org/pdf/1611.07612.pdf @@ -539,6 +589,121 @@ static inline float im2col_get_pixel(float *im, int height, int width, int chann return im[col + width*(row + height*channel)]; } +//From Berkeley Vision's Caffe! +//https://github.com/BVLC/caffe/blob/master/LICENSE +void im2col_cpu_custom_transpose(float* data_im, + int channels, int height, int width, + int ksize, int stride, int pad, float* data_col, int ldb_align) +{ + int c, h, w; + int height_col = (height + 2 * pad - ksize) / stride + 1; + int width_col = (width + 2 * pad - ksize) / stride + 1; + int channels_col = channels * ksize * ksize; + + // optimized version + if (height_col == height && width_col == width && stride == 1 && pad == 1) + { +#pragma omp parallel for + for (c = 0; c < channels_col; ++c) { + int w_offset = c % ksize; + int h_offset = (c / ksize) % ksize; + int c_im = c / ksize / ksize; + for (h = pad; h < height_col - pad; ++h) { + for (w = pad; w < width_col - pad - 4; w+=8) { + int im_row = h_offset + h - pad; + int im_col = w_offset + w - pad; + //int col_index = (c * height_col + h) * width_col + w; + int col_index = (h * width_col + w)*ldb_align + c; // transposed & aligned + + //data_col[col_index] = data_im[im_col + width*(im_row + height*c_im)]; + __m256 src256 = _mm256_loadu_ps((__m256i *)(&data_im[im_col + width*(im_row + height*c_im)])); + data_col[col_index + ldb_align * 0] = src256.m256_f32[0]; + data_col[col_index + ldb_align * 1] = src256.m256_f32[1]; + data_col[col_index + ldb_align * 2] = src256.m256_f32[2]; + data_col[col_index + ldb_align * 3] = src256.m256_f32[3]; + data_col[col_index + ldb_align * 4] = src256.m256_f32[4]; + data_col[col_index + ldb_align * 5] = src256.m256_f32[5]; + data_col[col_index + ldb_align * 6] = src256.m256_f32[6]; + data_col[col_index + ldb_align * 7] = src256.m256_f32[7]; + + //_mm256_storeu_ps(&data_col[col_index], src256); + } + + for (; w < width_col - pad; ++w) { + int im_row = h_offset + h - pad; + int im_col = w_offset + w - pad; + int col_index = (h * width_col + w)*ldb_align + c; // transposed & aligned + data_col[col_index] = data_im[im_col + width*(im_row + height*c_im)]; + } + } + + { + w = 0; + for (h = 0; h < height_col; ++h) { + int im_row = h_offset + h; + int im_col = w_offset + w; + int col_index = (h * width_col + w)*ldb_align + c; // transposed & aligned + data_col[col_index] = im2col_get_pixel(data_im, height, width, channels, + im_row, im_col, c_im, pad); + } + } + + { + w = width_col - 1; + for (h = 0; h < height_col; ++h) { + int im_row = h_offset + h; + int im_col = w_offset + w; + int col_index = (h * width_col + w)*ldb_align + c; // transposed & aligned + data_col[col_index] = im2col_get_pixel(data_im, height, width, channels, + im_row, im_col, c_im, pad); + } + } + + { + h = 0; + for (w = 0; w < width_col; ++w) { + int im_row = h_offset + h; + int im_col = w_offset + w; + int col_index = (h * width_col + w)*ldb_align + c; // transposed & aligned + data_col[col_index] = im2col_get_pixel(data_im, height, width, channels, + im_row, im_col, c_im, pad); + } + } + + { + h = height_col - 1; + for (w = 0; w < width_col; ++w) { + int im_row = h_offset + h; + int im_col = w_offset + w; + int col_index = (h * width_col + w)*ldb_align + c; // transposed & aligned + data_col[col_index] = im2col_get_pixel(data_im, height, width, channels, + im_row, im_col, c_im, pad); + } + } + } + + } + else { + #pragma omp parallel for + for (c = 0; c < channels_col; ++c) { + int w_offset = c % ksize; + int h_offset = (c / ksize) % ksize; + int c_im = c / ksize / ksize; + for (h = 0; h < height_col; ++h) { + for (w = 0; w < width_col; ++w) { + int im_row = h_offset + h * stride; + int im_col = w_offset + w * stride; + + int col_index = (h * width_col + w)*ldb_align + c; // transposed & aligned + data_col[col_index] = im2col_get_pixel(data_im, height, width, channels, + im_row, im_col, c_im, pad); + } + } + } + } +} + + //From Berkeley Vision's Caffe! //https://github.com/BVLC/caffe/blob/master/LICENSE void im2col_cpu_custom(float* data_im, @@ -641,7 +806,7 @@ void activate_array_cpu_custom(float *x, const int n, const ACTIVATION a) __m256i all256_sing1 = _mm256_set_epi32(0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000, 0x80000000); __m256 all256_01 = _mm256_set1_ps(0.1F); - for (i = 0; i < n; i += 8) { + for (i = 0; i < n-8; i += 8) { //x[i] = (x[i]>0) ? x[i] : .1*x[i]; __m256 src256 = _mm256_loadu_ps((__m256 *)(&x[i])); @@ -755,6 +920,55 @@ void gemm_nn(int M, int N, int K, float ALPHA, } } + +void convolution_2d(int w, int h, int ksize, int n, int c, int pad, int stride, + float *weights, float *input, float *output) +{ + int out_h = (h + 2 * pad - ksize) / stride + 1; // output_height=input_height for stride=1 and pad=1 + int out_w = (w + 2 * pad - ksize) / stride + 1; // output_width=input_width for stride=1 and pad=1 + int i, f, j; + + int fil; + // filter index +#pragma omp parallel for // "omp parallel for" - automatic parallelization of loop by using OpenMP + for (fil = 0; fil < n; ++fil) { + int chan, y, x, f_y, f_x; + // channel index + for (chan = 0; chan < c; ++chan) + // input - y + for (y = 0; y < h; ++y) + // input - x + for (x = 0; x < w; ++x) + { + int const output_index = fil*w*h + y*w + x; + int const weights_pre_index = fil*c*ksize*ksize + chan*ksize*ksize; + int const input_pre_index = chan*w*h; + float sum = 0; + + // filter - y + for (f_y = 0; f_y < ksize; ++f_y) + { + int input_y = y + f_y - pad; + // filter - x + for (f_x = 0; f_x < ksize; ++f_x) + { + int input_x = x + f_x - pad; + if (input_y < 0 || input_x < 0 || input_y >= h || input_x >= w) continue; + + int input_index = input_pre_index + input_y*w + input_x; + int weights_index = weights_pre_index + f_y*ksize + f_x; + + sum += input[input_index] * weights[weights_index]; + } + } + // l.output[filters][width][height] += + // state.input[channels][width][height] * + // l.weights[filters][channels][filter_width][filter_height]; + output[output_index] += sum; + } + } +} + void gemm_nn_custom_bin_mean_transposed(int M, int N, int K, float ALPHA_UNUSED, unsigned char *A, int lda, unsigned char *B, int ldb, @@ -791,6 +1005,13 @@ void gemm_nn_custom_bin_mean_transposed(int M, int N, int K, float ALPHA_UNUSED, } } +void im2col_cpu_custom_transpose(float* data_im, + int channels, int height, int width, + int ksize, int stride, int pad, float* data_col, int ldb_align) +{ + printf("\n im2col_cpu_custom_transpose() isn't implemented without AVX \n"); +} + //From Berkeley Vision's Caffe! //https://github.com/BVLC/caffe/blob/master/LICENSE void im2col_cpu_custom(float* data_im, diff --git a/src/gemm.h b/src/gemm.h index c71cd24b..62dbe30b 100644 --- a/src/gemm.h +++ b/src/gemm.h @@ -4,6 +4,9 @@ #include #include +void convolution_2d(int w, int h, int ksize, int n, int c, int pad, int stride, + float *weights, float *input, float *output); + static inline void set_bit(unsigned char *const dst, size_t index) { size_t dst_i = index / 8; int dst_shift = index % 8; @@ -31,6 +34,10 @@ void im2col_cpu_custom(float* data_im, int channels, int height, int width, int ksize, int stride, int pad, float* data_col); +void im2col_cpu_custom_transpose(float* data_im, + int channels, int height, int width, + int ksize, int stride, int pad, float* data_col, int ldb_align); + void activate_array_cpu_custom(float *x, const int n, const ACTIVATION a); diff --git a/src/layer.h b/src/layer.h index 224f77a1..bd8518ae 100644 --- a/src/layer.h +++ b/src/layer.h @@ -181,6 +181,7 @@ struct layer{ char *align_bit_weights; float *mean_arr; + int lda_align; float *col_image; int * input_layers; diff --git a/src/network.c b/src/network.c index 345ce687..2ad51411 100644 --- a/src/network.c +++ b/src/network.c @@ -861,9 +861,9 @@ void calculate_binary_weights(network net) if (l->xnor) { //printf("\n %d \n", j); - size_t ldb_align = 256; // 256bit for AVX2 + l->lda_align = 256; // 256bit for AVX2 - binary_align_weights(l, ldb_align); + binary_align_weights(l); } } }