From 9564549857231c3ffda602fd8efea091894ff6e9 Mon Sep 17 00:00:00 2001 From: Abe Miessler Date: Sat, 3 Jun 2017 15:58:21 -0700 Subject: [PATCH 1/7] adding missing include to get pthread_t type working --- include/darknet.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/darknet.h b/include/darknet.h index 986d4309..3f5e9b74 100644 --- a/include/darknet.h +++ b/include/darknet.h @@ -1,6 +1,7 @@ #ifndef DARKNET_API #define DARKNET_API #include +#include extern int gpu_index; From e9f3b79776b69818b3f53b96e35aaadc63596a93 Mon Sep 17 00:00:00 2001 From: Yao Lu Date: Tue, 6 Jun 2017 16:50:19 -0700 Subject: [PATCH 2/7] Fix GRU, Add LSTM --- Makefile | 4 +- include/darknet.h | 33 ++- src/gru_layer.c | 526 +++++++++++++++++++--------------------------- src/gru_layer.h | 8 +- src/lstm_layer.c | 365 ++++++++++++++++++++++++++++++++ src/lstm_layer.h | 20 ++ src/network.c | 2 + src/parser.c | 65 ++++-- 8 files changed, 696 insertions(+), 327 deletions(-) create mode 100644 src/lstm_layer.c create mode 100644 src/lstm_layer.h diff --git a/Makefile b/Makefile index 36a451cc..9ef36b84 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -GPU=0 +GPU=1 CUDNN=0 OPENCV=0 DEBUG=0 @@ -51,7 +51,7 @@ CFLAGS+= -DCUDNN LDFLAGS+= -lcudnn endif -OBJ=gemm.o utils.o cuda.o deconvolutional_layer.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o detection_layer.o route_layer.o box.o normalization_layer.o avgpool_layer.o layer.o local_layer.o shortcut_layer.o activation_layer.o rnn_layer.o gru_layer.o crnn_layer.o demo.o batchnorm_layer.o region_layer.o reorg_layer.o tree.o +OBJ=gemm.o utils.o cuda.o deconvolutional_layer.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o detection_layer.o route_layer.o box.o normalization_layer.o avgpool_layer.o layer.o local_layer.o shortcut_layer.o activation_layer.o rnn_layer.o gru_layer.o crnn_layer.o demo.o batchnorm_layer.o region_layer.o reorg_layer.o tree.o lstm_layer.o EXECOBJA=captcha.o lsd.o super.o voxel.o art.o tag.o cifar.o go.o rnn.o rnn_vid.o compare.o segmenter.o regressor.o classifier.o coco.o dice.o yolo.o detector.o writing.o nightmare.o swag.o darknet.o ifeq ($(GPU), 1) LDFLAGS+= -lstdc++ diff --git a/include/darknet.h b/include/darknet.h index 3f5e9b74..f2ef660a 100644 --- a/include/darknet.h +++ b/include/darknet.h @@ -63,6 +63,7 @@ typedef enum { ACTIVE, RNN, GRU, + LSTM, CRNN, BATCHNORM, NETWORK, @@ -185,7 +186,7 @@ struct layer{ float * forgot_state; float * forgot_delta; float * state_delta; - + float * concat; float * concat_delta; @@ -251,6 +252,21 @@ struct layer{ struct layer *input_h_layer; struct layer *state_h_layer; + + struct layer *wz; + struct layer *uz; + struct layer *wr; + struct layer *ur; + struct layer *wh; + struct layer *uh; + struct layer *uo; + struct layer *wo; + struct layer *uf; + struct layer *wf; + struct layer *ui; + struct layer *wi; + struct layer *ug; + struct layer *wg; tree *softmax_tree; @@ -263,6 +279,21 @@ struct layer{ float *r_gpu; float *h_gpu; + float *temp_gpu; + float *temp2_gpu; + float *temp3_gpu; + + float *dh_gpu; + float *hh_gpu; + float *prev_cell_gpu; + float *cell_gpu; + float *f_gpu; + float *i_gpu; + float *g_gpu; + float *o_gpu; + float *c_gpu; + float *dc_gpu; + float *m_gpu; float *v_gpu; float *bias_m_gpu; diff --git a/src/gru_layer.c b/src/gru_layer.c index 7139f798..78964817 100644 --- a/src/gru_layer.c +++ b/src/gru_layer.c @@ -12,195 +12,100 @@ static void increment_layer(layer *l, int steps) { - int num = l->outputs*l->batch*steps; - l->output += num; - l->delta += num; - l->x += num; - l->x_norm += num; + int num = l->outputs*l->batch*steps; + l->output += num; + l->delta += num; + l->x += num; + l->x_norm += num; #ifdef GPU - l->output_gpu += num; - l->delta_gpu += num; - l->x_gpu += num; - l->x_norm_gpu += num; + l->output_gpu += num; + l->delta_gpu += num; + l->x_gpu += num; + l->x_norm_gpu += num; #endif } layer make_gru_layer(int batch, int inputs, int outputs, int steps, int batch_normalize) { - fprintf(stderr, "GRU Layer: %d inputs, %d outputs\n", inputs, outputs); - batch = batch / steps; - layer l = {0}; - l.batch = batch; - l.type = GRU; - l.steps = steps; - l.inputs = inputs; + fprintf(stderr, "GRU Layer: %d inputs, %d outputs\n", inputs, outputs); + batch = batch / steps; + layer l = { 0 }; + l.batch = batch; + l.type = GRU; + l.steps = steps; + l.inputs = inputs; - l.input_z_layer = malloc(sizeof(layer)); - fprintf(stderr, "\t\t"); - *(l.input_z_layer) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); - l.input_z_layer->batch = batch; + l.wz = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.wz) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); + l.wz->batch = batch; - l.state_z_layer = malloc(sizeof(layer)); - fprintf(stderr, "\t\t"); - *(l.state_z_layer) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); - l.state_z_layer->batch = batch; + l.uz = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.uz) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); + l.uz->batch = batch; + l.wr = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.wr) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); + l.wr->batch = batch; + l.ur = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.ur) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); + l.ur->batch = batch; - l.input_r_layer = malloc(sizeof(layer)); - fprintf(stderr, "\t\t"); - *(l.input_r_layer) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); - l.input_r_layer->batch = batch; + l.wh = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.wh) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); + l.wh->batch = batch; - l.state_r_layer = malloc(sizeof(layer)); - fprintf(stderr, "\t\t"); - *(l.state_r_layer) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); - l.state_r_layer->batch = batch; + l.uh = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.uh) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); + l.uh->batch = batch; + l.batch_normalize = batch_normalize; + l.outputs = outputs; + l.output = calloc(outputs*batch*steps, sizeof(float)); + l.state = calloc(outputs*batch, sizeof(float)); - l.input_h_layer = malloc(sizeof(layer)); - fprintf(stderr, "\t\t"); - *(l.input_h_layer) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); - l.input_h_layer->batch = batch; - - l.state_h_layer = malloc(sizeof(layer)); - fprintf(stderr, "\t\t"); - *(l.state_h_layer) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); - l.state_h_layer->batch = batch; - -#ifdef CUDNN - cudnnSetTensor4dDescriptor(l.input_z_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.input_z_layer->out_c, l.input_z_layer->out_h, l.input_z_layer->out_w); - cudnnSetTensor4dDescriptor(l.input_h_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.input_h_layer->out_c, l.input_h_layer->out_h, l.input_h_layer->out_w); - cudnnSetTensor4dDescriptor(l.input_r_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.input_r_layer->out_c, l.input_r_layer->out_h, l.input_r_layer->out_w); - cudnnSetTensor4dDescriptor(l.state_z_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.state_z_layer->out_c, l.state_z_layer->out_h, l.state_z_layer->out_w); - cudnnSetTensor4dDescriptor(l.state_h_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.state_h_layer->out_c, l.state_h_layer->out_h, l.state_h_layer->out_w); - cudnnSetTensor4dDescriptor(l.state_r_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.state_r_layer->out_c, l.state_r_layer->out_h, l.state_r_layer->out_w); -#endif - - l.batch_normalize = batch_normalize; - - - l.outputs = outputs; - l.output = calloc(outputs*batch*steps, sizeof(float)); - l.delta = calloc(outputs*batch*steps, sizeof(float)); - l.state = calloc(outputs*batch, sizeof(float)); - l.prev_state = calloc(outputs*batch, sizeof(float)); - l.forgot_state = calloc(outputs*batch, sizeof(float)); - l.forgot_delta = calloc(outputs*batch, sizeof(float)); - - l.r_cpu = calloc(outputs*batch, sizeof(float)); - l.z_cpu = calloc(outputs*batch, sizeof(float)); - l.h_cpu = calloc(outputs*batch, sizeof(float)); - - l.forward = forward_gru_layer; - l.backward = backward_gru_layer; - l.update = update_gru_layer; + l.forward = forward_gru_layer; + l.backward = backward_gru_layer; + l.update = update_gru_layer; #ifdef GPU - l.forward_gpu = forward_gru_layer_gpu; - l.backward_gpu = backward_gru_layer_gpu; - l.update_gpu = update_gru_layer_gpu; + l.forward_gpu = forward_gru_layer_gpu; + l.backward_gpu = backward_gru_layer_gpu; + l.update_gpu = update_gru_layer_gpu; - l.forgot_state_gpu = cuda_make_array(l.output, batch*outputs); - l.forgot_delta_gpu = cuda_make_array(l.output, batch*outputs); - l.prev_state_gpu = cuda_make_array(l.output, batch*outputs); - l.state_gpu = cuda_make_array(l.output, batch*outputs); - l.output_gpu = cuda_make_array(l.output, batch*outputs*steps); - l.delta_gpu = cuda_make_array(l.delta, batch*outputs*steps); - l.r_gpu = cuda_make_array(l.output_gpu, batch*outputs); - l.z_gpu = cuda_make_array(l.output_gpu, batch*outputs); - l.h_gpu = cuda_make_array(l.output_gpu, batch*outputs); + l.prev_state_gpu = cuda_make_array(0, batch*outputs); + l.output_gpu = cuda_make_array(0, batch*outputs*steps); + l.delta_gpu = cuda_make_array(0, batch*outputs*steps); + + l.r_gpu = cuda_make_array(l.output, batch*outputs); + l.z_gpu = cuda_make_array(l.output, batch*outputs); + l.hh_gpu = cuda_make_array(l.output, batch*outputs); + l.h_gpu = cuda_make_array(l.output, batch*outputs); + l.temp_gpu = cuda_make_array(l.output, batch*outputs); + l.temp2_gpu = cuda_make_array(l.output, batch*outputs); + l.temp3_gpu = cuda_make_array(l.output, batch*outputs); + l.dh_gpu = cuda_make_array(l.output, batch*outputs); #endif - - return l; + return l; } void update_gru_layer(layer l, int batch, float learning_rate, float momentum, float decay) { - update_connected_layer(*(l.input_layer), batch, learning_rate, momentum, decay); - update_connected_layer(*(l.self_layer), batch, learning_rate, momentum, decay); - update_connected_layer(*(l.output_layer), batch, learning_rate, momentum, decay); } -void forward_gru_layer(layer l, network net) +void forward_gru_layer(layer l, network state) { - network s = net; - s.train = net.train; - int i; - layer input_z_layer = *(l.input_z_layer); - layer input_r_layer = *(l.input_r_layer); - layer input_h_layer = *(l.input_h_layer); - - layer state_z_layer = *(l.state_z_layer); - layer state_r_layer = *(l.state_r_layer); - layer state_h_layer = *(l.state_h_layer); - - fill_cpu(l.outputs * l.batch * l.steps, 0, input_z_layer.delta, 1); - fill_cpu(l.outputs * l.batch * l.steps, 0, input_r_layer.delta, 1); - fill_cpu(l.outputs * l.batch * l.steps, 0, input_h_layer.delta, 1); - - fill_cpu(l.outputs * l.batch * l.steps, 0, state_z_layer.delta, 1); - fill_cpu(l.outputs * l.batch * l.steps, 0, state_r_layer.delta, 1); - fill_cpu(l.outputs * l.batch * l.steps, 0, state_h_layer.delta, 1); - if(net.train) { - fill_cpu(l.outputs * l.batch * l.steps, 0, l.delta, 1); - copy_cpu(l.outputs*l.batch, l.state, 1, l.prev_state, 1); - } - - for (i = 0; i < l.steps; ++i) { - s.input = l.state; - forward_connected_layer(state_z_layer, s); - forward_connected_layer(state_r_layer, s); - - s.input = net.input; - forward_connected_layer(input_z_layer, s); - forward_connected_layer(input_r_layer, s); - forward_connected_layer(input_h_layer, s); - - - copy_cpu(l.outputs*l.batch, input_z_layer.output, 1, l.z_cpu, 1); - axpy_cpu(l.outputs*l.batch, 1, state_z_layer.output, 1, l.z_cpu, 1); - - copy_cpu(l.outputs*l.batch, input_r_layer.output, 1, l.r_cpu, 1); - axpy_cpu(l.outputs*l.batch, 1, state_r_layer.output, 1, l.r_cpu, 1); - - activate_array(l.z_cpu, l.outputs*l.batch, LOGISTIC); - activate_array(l.r_cpu, l.outputs*l.batch, LOGISTIC); - - copy_cpu(l.outputs*l.batch, l.state, 1, l.forgot_state, 1); - mul_cpu(l.outputs*l.batch, l.r_cpu, 1, l.forgot_state, 1); - - s.input = l.forgot_state; - forward_connected_layer(state_h_layer, s); - - copy_cpu(l.outputs*l.batch, input_h_layer.output, 1, l.h_cpu, 1); - axpy_cpu(l.outputs*l.batch, 1, state_h_layer.output, 1, l.h_cpu, 1); - - #ifdef USET - activate_array(l.h_cpu, l.outputs*l.batch, TANH); - #else - activate_array(l.h_cpu, l.outputs*l.batch, LOGISTIC); - #endif - - weighted_sum_cpu(l.state, l.h_cpu, l.z_cpu, l.outputs*l.batch, l.output); - - copy_cpu(l.outputs*l.batch, l.output, 1, l.state, 1); - - net.input += l.inputs*l.batch; - l.output += l.outputs*l.batch; - increment_layer(&input_z_layer, 1); - increment_layer(&input_r_layer, 1); - increment_layer(&input_h_layer, 1); - - increment_layer(&state_z_layer, 1); - increment_layer(&state_r_layer, 1); - increment_layer(&state_h_layer, 1); - } } -void backward_gru_layer(layer l, network net) +void backward_gru_layer(layer l, network state) { } @@ -216,189 +121,202 @@ void push_gru_layer(layer l) void update_gru_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay) { - update_connected_layer_gpu(*(l.input_r_layer), batch, learning_rate, momentum, decay); - update_connected_layer_gpu(*(l.input_z_layer), batch, learning_rate, momentum, decay); - update_connected_layer_gpu(*(l.input_h_layer), batch, learning_rate, momentum, decay); - update_connected_layer_gpu(*(l.state_r_layer), batch, learning_rate, momentum, decay); - update_connected_layer_gpu(*(l.state_z_layer), batch, learning_rate, momentum, decay); - update_connected_layer_gpu(*(l.state_h_layer), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.wr), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.wz), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.wh), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.ur), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.uz), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.uh), batch, learning_rate, momentum, decay); } -void forward_gru_layer_gpu(layer l, network net) +void forward_gru_layer_gpu(layer l, network state) { - network s = net; - s.train = net.train; - int i; - layer input_z_layer = *(l.input_z_layer); - layer input_r_layer = *(l.input_r_layer); - layer input_h_layer = *(l.input_h_layer); + network s = { 0 }; + s.train = state.train; + int i; + layer wz = *(l.wz); + layer wr = *(l.wr); + layer wh = *(l.wh); - layer state_z_layer = *(l.state_z_layer); - layer state_r_layer = *(l.state_r_layer); - layer state_h_layer = *(l.state_h_layer); + layer uz = *(l.uz); + layer ur = *(l.ur); + layer uh = *(l.uh); - fill_ongpu(l.outputs * l.batch * l.steps, 0, input_z_layer.delta_gpu, 1); - fill_ongpu(l.outputs * l.batch * l.steps, 0, input_r_layer.delta_gpu, 1); - fill_ongpu(l.outputs * l.batch * l.steps, 0, input_h_layer.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, wz.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, wr.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, wh.delta_gpu, 1); - fill_ongpu(l.outputs * l.batch * l.steps, 0, state_z_layer.delta_gpu, 1); - fill_ongpu(l.outputs * l.batch * l.steps, 0, state_r_layer.delta_gpu, 1); - fill_ongpu(l.outputs * l.batch * l.steps, 0, state_h_layer.delta_gpu, 1); - if(net.train) { - fill_ongpu(l.outputs * l.batch * l.steps, 0, l.delta_gpu, 1); - copy_ongpu(l.outputs*l.batch, l.state_gpu, 1, l.prev_state_gpu, 1); - } + fill_ongpu(l.outputs * l.batch * l.steps, 0, uz.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, ur.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, uh.delta_gpu, 1); - for (i = 0; i < l.steps; ++i) { - s.input_gpu = l.state_gpu; - forward_connected_layer_gpu(state_z_layer, s); - forward_connected_layer_gpu(state_r_layer, s); + if (state.train) { + fill_ongpu(l.outputs * l.batch * l.steps, 0, l.delta_gpu, 1); + } - s.input_gpu = net.input_gpu; - forward_connected_layer_gpu(input_z_layer, s); - forward_connected_layer_gpu(input_r_layer, s); - forward_connected_layer_gpu(input_h_layer, s); + for (i = 0; i < l.steps; ++i) { + s.input = l.h_gpu; + forward_connected_layer_gpu(uz, s); + forward_connected_layer_gpu(ur, s); + s.input = state.input; + forward_connected_layer_gpu(wz, s); + forward_connected_layer_gpu(wr, s); + forward_connected_layer_gpu(wh, s); - copy_ongpu(l.outputs*l.batch, input_z_layer.output_gpu, 1, l.z_gpu, 1); - axpy_ongpu(l.outputs*l.batch, 1, state_z_layer.output_gpu, 1, l.z_gpu, 1); + copy_ongpu(l.outputs*l.batch, wz.output_gpu, 1, l.z_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, uz.output_gpu, 1, l.z_gpu, 1); - copy_ongpu(l.outputs*l.batch, input_r_layer.output_gpu, 1, l.r_gpu, 1); - axpy_ongpu(l.outputs*l.batch, 1, state_r_layer.output_gpu, 1, l.r_gpu, 1); + copy_ongpu(l.outputs*l.batch, wr.output_gpu, 1, l.r_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, ur.output_gpu, 1, l.r_gpu, 1); - activate_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC); - activate_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC); + activate_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC); + activate_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC); - copy_ongpu(l.outputs*l.batch, l.state_gpu, 1, l.forgot_state_gpu, 1); - mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.forgot_state_gpu, 1); + copy_ongpu(l.outputs*l.batch, l.h_gpu, 1, l.hh_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.hh_gpu, 1); - s.input_gpu = l.forgot_state_gpu; - forward_connected_layer_gpu(state_h_layer, s); + s.input = l.hh_gpu; + forward_connected_layer_gpu(uh, s); - copy_ongpu(l.outputs*l.batch, input_h_layer.output_gpu, 1, l.h_gpu, 1); - axpy_ongpu(l.outputs*l.batch, 1, state_h_layer.output_gpu, 1, l.h_gpu, 1); + copy_ongpu(l.outputs*l.batch, wh.output_gpu, 1, l.hh_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, uh.output_gpu, 1, l.hh_gpu, 1); - #ifdef USET - activate_array_ongpu(l.h_gpu, l.outputs*l.batch, TANH); - #else - activate_array_ongpu(l.h_gpu, l.outputs*l.batch, LOGISTIC); - #endif + activate_array_ongpu(l.hh_gpu, l.outputs*l.batch, TANH); - weighted_sum_gpu(l.state_gpu, l.h_gpu, l.z_gpu, l.outputs*l.batch, l.output_gpu); + weighted_sum_gpu(l.h_gpu, l.hh_gpu, l.z_gpu, l.outputs*l.batch, l.output_gpu); + //ht = z .* ht-1 + (1-z) .* hh + copy_ongpu(l.outputs*l.batch, l.output_gpu, 1, l.h_gpu, 1); - copy_ongpu(l.outputs*l.batch, l.output_gpu, 1, l.state_gpu, 1); + state.input += l.inputs*l.batch; + l.output_gpu += l.outputs*l.batch; - net.input_gpu += l.inputs*l.batch; - l.output_gpu += l.outputs*l.batch; - increment_layer(&input_z_layer, 1); - increment_layer(&input_r_layer, 1); - increment_layer(&input_h_layer, 1); + increment_layer(&wz, 1); + increment_layer(&wr, 1); + increment_layer(&wh, 1); - increment_layer(&state_z_layer, 1); - increment_layer(&state_r_layer, 1); - increment_layer(&state_h_layer, 1); - } + increment_layer(&uz, 1); + increment_layer(&ur, 1); + increment_layer(&uh, 1); + } } -void backward_gru_layer_gpu(layer l, network net) +void backward_gru_layer_gpu(layer l, network state) { - network s = net; - s.train = net.train; - int i; - layer input_z_layer = *(l.input_z_layer); - layer input_r_layer = *(l.input_r_layer); - layer input_h_layer = *(l.input_h_layer); + network s = { 0 }; + s.train = state.train; + int i; + layer wz = *(l.wz); + layer wr = *(l.wr); + layer wh = *(l.wh); - layer state_z_layer = *(l.state_z_layer); - layer state_r_layer = *(l.state_r_layer); - layer state_h_layer = *(l.state_h_layer); + layer uz = *(l.uz); + layer ur = *(l.ur); + layer uh = *(l.uh); - increment_layer(&input_z_layer, l.steps - 1); - increment_layer(&input_r_layer, l.steps - 1); - increment_layer(&input_h_layer, l.steps - 1); + increment_layer(&wz, l.steps - 1); + increment_layer(&wr, l.steps - 1); + increment_layer(&wh, l.steps - 1); - increment_layer(&state_z_layer, l.steps - 1); - increment_layer(&state_r_layer, l.steps - 1); - increment_layer(&state_h_layer, l.steps - 1); + increment_layer(&uz, l.steps - 1); + increment_layer(&ur, l.steps - 1); + increment_layer(&uh, l.steps - 1); - net.input_gpu += l.inputs*l.batch*(l.steps-1); - if(net.delta_gpu) net.delta_gpu += l.inputs*l.batch*(l.steps-1); - l.output_gpu += l.outputs*l.batch*(l.steps-1); - l.delta_gpu += l.outputs*l.batch*(l.steps-1); - for (i = l.steps-1; i >= 0; --i) { - if(i != 0) copy_ongpu(l.outputs*l.batch, l.output_gpu - l.outputs*l.batch, 1, l.prev_state_gpu, 1); - float *prev_delta_gpu = (i == 0) ? 0 : l.delta_gpu - l.outputs*l.batch; + state.input += l.inputs*l.batch*(l.steps - 1); + if (state.delta) state.delta += l.inputs*l.batch*(l.steps - 1); - copy_ongpu(l.outputs*l.batch, input_z_layer.output_gpu, 1, l.z_gpu, 1); - axpy_ongpu(l.outputs*l.batch, 1, state_z_layer.output_gpu, 1, l.z_gpu, 1); + l.output_gpu += l.outputs*l.batch*(l.steps - 1); + l.delta_gpu += l.outputs*l.batch*(l.steps - 1); - copy_ongpu(l.outputs*l.batch, input_r_layer.output_gpu, 1, l.r_gpu, 1); - axpy_ongpu(l.outputs*l.batch, 1, state_r_layer.output_gpu, 1, l.r_gpu, 1); + for (i = l.steps - 1; i >= 0; --i) { + if (i>0) copy_ongpu(l.outputs*l.batch, l.output_gpu - l.outputs*l.batch, 1, l.prev_state_gpu, 1); + l.dh_gpu = (i == 0) ? 0 : l.delta_gpu - l.outputs*l.batch; - activate_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC); - activate_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC); + copy_ongpu(l.outputs*l.batch, wz.output_gpu, 1, l.z_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, uz.output_gpu, 1, l.z_gpu, 1); - copy_ongpu(l.outputs*l.batch, input_h_layer.output_gpu, 1, l.h_gpu, 1); - axpy_ongpu(l.outputs*l.batch, 1, state_h_layer.output_gpu, 1, l.h_gpu, 1); + copy_ongpu(l.outputs*l.batch, wr.output_gpu, 1, l.r_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, ur.output_gpu, 1, l.r_gpu, 1); - #ifdef USET - activate_array_ongpu(l.h_gpu, l.outputs*l.batch, TANH); - #else - activate_array_ongpu(l.h_gpu, l.outputs*l.batch, LOGISTIC); - #endif - - weighted_delta_gpu(l.prev_state_gpu, l.h_gpu, l.z_gpu, prev_delta_gpu, input_h_layer.delta_gpu, input_z_layer.delta_gpu, l.outputs*l.batch, l.delta_gpu); + activate_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC); + activate_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC); - #ifdef USET - gradient_array_ongpu(l.h_gpu, l.outputs*l.batch, TANH, input_h_layer.delta_gpu); - #else - gradient_array_ongpu(l.h_gpu, l.outputs*l.batch, LOGISTIC, input_h_layer.delta_gpu); - #endif + copy_ongpu(l.outputs*l.batch, wh.output_gpu, 1, l.hh_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, uh.output_gpu, 1, l.hh_gpu, 1); - copy_ongpu(l.outputs*l.batch, input_h_layer.delta_gpu, 1, state_h_layer.delta_gpu, 1); - - copy_ongpu(l.outputs*l.batch, l.prev_state_gpu, 1, l.forgot_state_gpu, 1); - mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.forgot_state_gpu, 1); - fill_ongpu(l.outputs*l.batch, 0, l.forgot_delta_gpu, 1); + activate_array_ongpu(l.hh_gpu, l.outputs*l.batch, TANH); - s.input_gpu = l.forgot_state_gpu; - s.delta_gpu = l.forgot_delta_gpu; - - backward_connected_layer_gpu(state_h_layer, s); - if(prev_delta_gpu) mult_add_into_gpu(l.outputs*l.batch, l.forgot_delta_gpu, l.r_gpu, prev_delta_gpu); - mult_add_into_gpu(l.outputs*l.batch, l.forgot_delta_gpu, l.prev_state_gpu, input_r_layer.delta_gpu); + copy_ongpu(l.outputs*l.batch, l.delta_gpu, 1, l.temp3_gpu, 1); - gradient_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC, input_r_layer.delta_gpu); - copy_ongpu(l.outputs*l.batch, input_r_layer.delta_gpu, 1, state_r_layer.delta_gpu, 1); + fill_ongpu(l.outputs*l.batch, 1, l.temp_gpu, 1); + axpy_ongpu(l.outputs*l.batch, -1, l.z_gpu, 1, l.temp_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp_gpu, 1); + gradient_array_ongpu(l.hh_gpu, l.outputs*l.batch, TANH, l.temp_gpu); - gradient_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC, input_z_layer.delta_gpu); - copy_ongpu(l.outputs*l.batch, input_z_layer.delta_gpu, 1, state_z_layer.delta_gpu, 1); - - s.input_gpu = l.prev_state_gpu; - s.delta_gpu = prev_delta_gpu; - - backward_connected_layer_gpu(state_r_layer, s); - backward_connected_layer_gpu(state_z_layer, s); + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wh.delta_gpu, 1); + s.input = state.input; + s.delta = state.delta; + backward_connected_layer_gpu(wh, s); - s.input_gpu = net.input_gpu; - s.delta_gpu = net.delta_gpu; - - backward_connected_layer_gpu(input_h_layer, s); - backward_connected_layer_gpu(input_r_layer, s); - backward_connected_layer_gpu(input_z_layer, s); + copy_ongpu(l.outputs*l.batch, l.prev_state_gpu, 1, l.temp2_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.temp2_gpu, 1); + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, uh.delta_gpu, 1); + fill_ongpu(l.outputs*l.batch, 0, l.temp_gpu, 1); + s.input = l.temp2_gpu; + s.delta = l.temp_gpu; + backward_connected_layer_gpu(uh, s); - net.input_gpu -= l.inputs*l.batch; - if(net.delta_gpu) net.delta_gpu -= l.inputs*l.batch; - l.output_gpu -= l.outputs*l.batch; - l.delta_gpu -= l.outputs*l.batch; - increment_layer(&input_z_layer, -1); - increment_layer(&input_r_layer, -1); - increment_layer(&input_h_layer, -1); + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, l.temp2_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.prev_state_gpu, 1, l.temp2_gpu, 1); + gradient_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC, l.temp2_gpu); - increment_layer(&state_z_layer, -1); - increment_layer(&state_r_layer, -1); - increment_layer(&state_h_layer, -1); - } + copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, wr.delta_gpu, 1); + s.input = state.input; + s.delta = state.delta; + backward_connected_layer_gpu(wr, s); + + copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, ur.delta_gpu, 1); + s.input = l.prev_state_gpu; + s.delta = l.dh_gpu; + backward_connected_layer_gpu(ur, s); + + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, l.temp2_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.temp2_gpu, 1); + if (l.dh_gpu) axpy_ongpu(l.outputs*l.batch, 1, l.temp2_gpu, 1, l.dh_gpu, 1); + + copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp2_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.z_gpu, 1, l.temp2_gpu, 1); + if (l.dh_gpu) axpy_ongpu(l.outputs*l.batch, 1, l.temp2_gpu, 1, l.dh_gpu, 1); + + copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp2_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.prev_state_gpu, 1, l.temp3_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.hh_gpu, 1, l.temp2_gpu, 1); + axpy_ongpu(l.outputs*l.batch, -1, l.temp2_gpu, 1, l.temp3_gpu, 1); + gradient_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC, l.temp3_gpu); + + copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, wz.delta_gpu, 1); + s.input = state.input; + s.delta = state.delta; + backward_connected_layer_gpu(wz, s); + + copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, uz.delta_gpu, 1); + s.input = l.prev_state_gpu; + s.delta = l.dh_gpu; + backward_connected_layer_gpu(uz, s); + + state.input -= l.inputs*l.batch; + if (state.delta) state.delta -= l.inputs*l.batch; + l.output_gpu -= l.outputs*l.batch; + l.delta_gpu -= l.outputs*l.batch; + + increment_layer(&wz, -1); + increment_layer(&wr, -1); + increment_layer(&wh, -1); + + increment_layer(&uz, -1); + increment_layer(&ur, -1); + increment_layer(&uh, -1); + } } -#endif +#endif \ No newline at end of file diff --git a/src/gru_layer.h b/src/gru_layer.h index 9dc456e0..a0e57171 100644 --- a/src/gru_layer.h +++ b/src/gru_layer.h @@ -8,13 +8,13 @@ layer make_gru_layer(int batch, int inputs, int outputs, int steps, int batch_normalize); -void forward_gru_layer(layer l, network net); -void backward_gru_layer(layer l, network net); +void forward_gru_layer(layer l, network state); +void backward_gru_layer(layer l, network state); void update_gru_layer(layer l, int batch, float learning_rate, float momentum, float decay); #ifdef GPU -void forward_gru_layer_gpu(layer l, network net); -void backward_gru_layer_gpu(layer l, network net); +void forward_gru_layer_gpu(layer l, network state); +void backward_gru_layer_gpu(layer l, network state); void update_gru_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay); void push_gru_layer(layer l); void pull_gru_layer(layer l); diff --git a/src/lstm_layer.c b/src/lstm_layer.c new file mode 100644 index 00000000..d806cb50 --- /dev/null +++ b/src/lstm_layer.c @@ -0,0 +1,365 @@ +#include "lstm_layer.h" +#include "connected_layer.h" +#include "utils.h" +#include "cuda.h" +#include "blas.h" +#include "gemm.h" + +#include +#include +#include +#include + +static void increment_layer(layer *l, int steps) +{ + int num = l->outputs*l->batch*steps; + l->output += num; + l->delta += num; + l->x += num; + l->x_norm += num; + +#ifdef GPU + l->output_gpu += num; + l->delta_gpu += num; + l->x_gpu += num; + l->x_norm_gpu += num; +#endif +} + +layer make_lstm_layer(int batch, int inputs, int outputs, int steps, int batch_normalize) +{ + fprintf(stderr, "LSTM Layer: %d inputs, %d outputs\n", inputs, outputs); + batch = batch / steps; + layer l = { 0 }; + l.batch = batch; + l.type = LSTM; + l.steps = steps; + l.inputs = inputs; + + l.uf = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.uf) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); + l.uf->batch = batch; + + l.wf = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.wf) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); + l.wf->batch = batch; + + l.ui = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.ui) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); + l.ui->batch = batch; + + l.wi = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.wi) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); + l.wi->batch = batch; + + l.ug = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.ug) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); + l.ug->batch = batch; + + l.wg = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.wg) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); + l.wg->batch = batch; + + l.uo = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.uo) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize); + l.uo->batch = batch; + + l.wo = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.wo) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize); + l.wo->batch = batch; + + l.batch_normalize = batch_normalize; + l.outputs = outputs; + + l.output = calloc(outputs*batch*steps, sizeof(float)); + l.state = calloc(outputs*batch, sizeof(float)); + + l.forward = forward_lstm_layer; + l.update = update_lstm_layer; + +#ifdef GPU + l.forward_gpu = forward_lstm_layer_gpu; + l.backward_gpu = backward_lstm_layer_gpu; + l.update_gpu = update_lstm_layer_gpu; + + l.prev_state_gpu = cuda_make_array(0, batch*outputs); + l.prev_cell_gpu = cuda_make_array(0, batch*outputs); + + l.output_gpu = cuda_make_array(0, batch*outputs*steps); + l.cell_gpu = cuda_make_array(0, batch*outputs*steps); + l.delta_gpu = cuda_make_array(0, batch*l.outputs*steps); + + l.f_gpu = cuda_make_array(l.output, batch*outputs); + l.i_gpu = cuda_make_array(l.output, batch*outputs); + l.g_gpu = cuda_make_array(l.output, batch*outputs); + l.o_gpu = cuda_make_array(l.output, batch*outputs); + l.c_gpu = cuda_make_array(l.output, batch*outputs); + l.h_gpu = cuda_make_array(l.output, batch*outputs); + l.temp_gpu = cuda_make_array(l.output, batch*outputs); + l.temp2_gpu = cuda_make_array(l.output, batch*outputs); + l.temp3_gpu = cuda_make_array(l.output, batch*outputs); + l.dc_gpu = cuda_make_array(l.output, batch*outputs); + l.dh_gpu = cuda_make_array(l.output, batch*outputs); +#endif + + return l; +} + +void update_lstm_layer(layer l, int batch, float learning_rate, float momentum, float decay) +{ +} + +void forward_lstm_layer(layer l, network state) +{ +} + +#ifdef GPU +void update_lstm_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay) +{ + update_connected_layer_gpu(*(l.wf), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.wi), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.wg), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.wo), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.uf), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.ui), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.ug), batch, learning_rate, momentum, decay); + update_connected_layer_gpu(*(l.uo), batch, learning_rate, momentum, decay); +} + +void forward_lstm_layer_gpu(layer l, network state) +{ + network s = { 0 }; + s.train = state.train; + int i; + layer wf = *(l.wf); + layer wi = *(l.wi); + layer wg = *(l.wg); + layer wo = *(l.wo); + + layer uf = *(l.uf); + layer ui = *(l.ui); + layer ug = *(l.ug); + layer uo = *(l.uo); + + fill_ongpu(l.outputs * l.batch * l.steps, 0, wf.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, wi.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, wg.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, wo.delta_gpu, 1); + + fill_ongpu(l.outputs * l.batch * l.steps, 0, uf.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, ui.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, ug.delta_gpu, 1); + fill_ongpu(l.outputs * l.batch * l.steps, 0, uo.delta_gpu, 1); + if (state.train) { + fill_ongpu(l.outputs * l.batch * l.steps, 0, l.delta_gpu, 1); + } + + for (i = 0; i < l.steps; ++i) { + s.input = l.h_gpu; + forward_connected_layer_gpu(wf, s); + forward_connected_layer_gpu(wi, s); + forward_connected_layer_gpu(wg, s); + forward_connected_layer_gpu(wo, s); + + s.input = state.input; + forward_connected_layer_gpu(uf, s); + forward_connected_layer_gpu(ui, s); + forward_connected_layer_gpu(ug, s); + forward_connected_layer_gpu(uo, s); + + copy_ongpu(l.outputs*l.batch, wf.output_gpu, 1, l.f_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, uf.output_gpu, 1, l.f_gpu, 1); + + copy_ongpu(l.outputs*l.batch, wi.output_gpu, 1, l.i_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, ui.output_gpu, 1, l.i_gpu, 1); + + copy_ongpu(l.outputs*l.batch, wg.output_gpu, 1, l.g_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, ug.output_gpu, 1, l.g_gpu, 1); + + copy_ongpu(l.outputs*l.batch, wo.output_gpu, 1, l.o_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, uo.output_gpu, 1, l.o_gpu, 1); + + activate_array_ongpu(l.f_gpu, l.outputs*l.batch, LOGISTIC); + activate_array_ongpu(l.i_gpu, l.outputs*l.batch, LOGISTIC); + activate_array_ongpu(l.g_gpu, l.outputs*l.batch, TANH); + activate_array_ongpu(l.o_gpu, l.outputs*l.batch, LOGISTIC); + + copy_ongpu(l.outputs*l.batch, l.i_gpu, 1, l.temp_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.g_gpu, 1, l.temp_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.f_gpu, 1, l.c_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, l.temp_gpu, 1, l.c_gpu, 1); + + copy_ongpu(l.outputs*l.batch, l.c_gpu, 1, l.h_gpu, 1); + activate_array_ongpu(l.h_gpu, l.outputs*l.batch, TANH); + mul_ongpu(l.outputs*l.batch, l.o_gpu, 1, l.h_gpu, 1); + + copy_ongpu(l.outputs*l.batch, l.c_gpu, 1, l.cell_gpu, 1); + copy_ongpu(l.outputs*l.batch, l.h_gpu, 1, l.output_gpu, 1); + + state.input += l.inputs*l.batch; + l.output_gpu += l.outputs*l.batch; + l.cell_gpu += l.outputs*l.batch; + + increment_layer(&wf, 1); + increment_layer(&wi, 1); + increment_layer(&wg, 1); + increment_layer(&wo, 1); + + increment_layer(&uf, 1); + increment_layer(&ui, 1); + increment_layer(&ug, 1); + increment_layer(&uo, 1); + } +} + +void backward_lstm_layer_gpu(layer l, network state) +{ + network s = { 0 }; + s.train = state.train; + int i; + layer wf = *(l.wf); + layer wi = *(l.wi); + layer wg = *(l.wg); + layer wo = *(l.wo); + + layer uf = *(l.uf); + layer ui = *(l.ui); + layer ug = *(l.ug); + layer uo = *(l.uo); + + increment_layer(&wf, l.steps - 1); + increment_layer(&wi, l.steps - 1); + increment_layer(&wg, l.steps - 1); + increment_layer(&wo, l.steps - 1); + + increment_layer(&uf, l.steps - 1); + increment_layer(&ui, l.steps - 1); + increment_layer(&ug, l.steps - 1); + increment_layer(&uo, l.steps - 1); + + state.input += l.inputs*l.batch*(l.steps - 1); + if (state.delta) state.delta += l.inputs*l.batch*(l.steps - 1); + + l.output_gpu += l.outputs*l.batch*(l.steps - 1); + l.cell_gpu += l.outputs*l.batch*(l.steps - 1); + l.delta_gpu += l.outputs*l.batch*(l.steps - 1); + + for (i = l.steps - 1; i >= 0; --i) { + if (i != 0) copy_ongpu(l.outputs*l.batch, l.cell_gpu - l.outputs*l.batch, 1, l.prev_cell_gpu, 1); + copy_ongpu(l.outputs*l.batch, l.cell_gpu, 1, l.c_gpu, 1); + if (i != 0) copy_ongpu(l.outputs*l.batch, l.output_gpu - l.outputs*l.batch, 1, l.prev_state_gpu, 1); + copy_ongpu(l.outputs*l.batch, l.output_gpu, 1, l.h_gpu, 1); + + l.dh_gpu = (i == 0) ? 0 : l.delta_gpu - l.outputs*l.batch; + + copy_ongpu(l.outputs*l.batch, wf.output_gpu, 1, l.f_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, uf.output_gpu, 1, l.f_gpu, 1); + + copy_ongpu(l.outputs*l.batch, wi.output_gpu, 1, l.i_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, ui.output_gpu, 1, l.i_gpu, 1); + + copy_ongpu(l.outputs*l.batch, wg.output_gpu, 1, l.g_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, ug.output_gpu, 1, l.g_gpu, 1); + + copy_ongpu(l.outputs*l.batch, wo.output_gpu, 1, l.o_gpu, 1); + axpy_ongpu(l.outputs*l.batch, 1, uo.output_gpu, 1, l.o_gpu, 1); + + activate_array_ongpu(l.f_gpu, l.outputs*l.batch, LOGISTIC); + activate_array_ongpu(l.i_gpu, l.outputs*l.batch, LOGISTIC); + activate_array_ongpu(l.g_gpu, l.outputs*l.batch, TANH); + activate_array_ongpu(l.o_gpu, l.outputs*l.batch, LOGISTIC); + + copy_ongpu(l.outputs*l.batch, l.delta_gpu, 1, l.temp3_gpu, 1); + + copy_ongpu(l.outputs*l.batch, l.c_gpu, 1, l.temp_gpu, 1); + activate_array_ongpu(l.temp_gpu, l.outputs*l.batch, TANH); + + copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp2_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.o_gpu, 1, l.temp2_gpu, 1); + + gradient_array_ongpu(l.temp_gpu, l.outputs*l.batch, TANH, l.temp2_gpu); + axpy_ongpu(l.outputs*l.batch, 1, l.dc_gpu, 1, l.temp2_gpu, 1); + + copy_ongpu(l.outputs*l.batch, l.c_gpu, 1, l.temp_gpu, 1); + activate_array_ongpu(l.temp_gpu, l.outputs*l.batch, TANH); + mul_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp_gpu, 1); + gradient_array_ongpu(l.o_gpu, l.outputs*l.batch, LOGISTIC, l.temp_gpu); + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wo.delta_gpu, 1); + s.input = l.prev_state_gpu; + s.delta = l.dh_gpu; + backward_connected_layer_gpu(wo, s); + + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, uo.delta_gpu, 1); + s.input = state.input; + s.delta = state.delta; + backward_connected_layer_gpu(uo, s); + + copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, l.temp_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.i_gpu, 1, l.temp_gpu, 1); + gradient_array_ongpu(l.g_gpu, l.outputs*l.batch, TANH, l.temp_gpu); + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wg.delta_gpu, 1); + s.input = l.prev_state_gpu; + s.delta = l.dh_gpu; + backward_connected_layer_gpu(wg, s); + + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, ug.delta_gpu, 1); + s.input = state.input; + s.delta = state.delta; + backward_connected_layer_gpu(ug, s); + + copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, l.temp_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.g_gpu, 1, l.temp_gpu, 1); + gradient_array_ongpu(l.i_gpu, l.outputs*l.batch, LOGISTIC, l.temp_gpu); + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wi.delta_gpu, 1); + s.input = l.prev_state_gpu; + s.delta = l.dh_gpu; + backward_connected_layer_gpu(wi, s); + + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, ui.delta_gpu, 1); + s.input = state.input; + s.delta = state.delta; + backward_connected_layer_gpu(ui, s); + + copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, l.temp_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.prev_cell_gpu, 1, l.temp_gpu, 1); + gradient_array_ongpu(l.f_gpu, l.outputs*l.batch, LOGISTIC, l.temp_gpu); + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wf.delta_gpu, 1); + s.input = l.prev_state_gpu; + s.delta = l.dh_gpu; + backward_connected_layer_gpu(wf, s); + + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, uf.delta_gpu, 1); + s.input = state.input; + s.delta = state.delta; + backward_connected_layer_gpu(uf, s); + + copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, l.temp_gpu, 1); + mul_ongpu(l.outputs*l.batch, l.f_gpu, 1, l.temp_gpu, 1); + copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, l.dc_gpu, 1); + + state.input -= l.inputs*l.batch; + if (state.delta) state.delta -= l.inputs*l.batch; + l.output_gpu -= l.outputs*l.batch; + l.cell_gpu -= l.outputs*l.batch; + l.delta_gpu -= l.outputs*l.batch; + + increment_layer(&wf, -1); + increment_layer(&wi, -1); + increment_layer(&wg, -1); + increment_layer(&wo, -1); + + increment_layer(&uf, -1); + increment_layer(&ui, -1); + increment_layer(&ug, -1); + increment_layer(&uo, -1); + } +} +#endif \ No newline at end of file diff --git a/src/lstm_layer.h b/src/lstm_layer.h new file mode 100644 index 00000000..a9ed792d --- /dev/null +++ b/src/lstm_layer.h @@ -0,0 +1,20 @@ +#ifndef LSTM_LAYER_H +#define LSTM_LAYER_H + +#include "activations.h" +#include "layer.h" +#include "network.h" +#define USET + +layer make_lstm_layer(int batch, int inputs, int outputs, int steps, int batch_normalize); + +void forward_lstm_layer(layer l, network state); +void update_lstm_layer(layer l, int batch, float learning, float momentum, float decay); + +#ifdef GPU +void forward_lstm_layer_gpu(layer l, network state); +void backward_lstm_layer_gpu(layer l, network state); +void update_lstm_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay); + +#endif +#endif diff --git a/src/network.c b/src/network.c index 2b21338d..b44e3a8b 100644 --- a/src/network.c +++ b/src/network.c @@ -125,6 +125,8 @@ char *get_layer_string(LAYER_TYPE a) return "rnn"; case GRU: return "gru"; + case LSTM: + return "lstm"; case CRNN: return "crnn"; case MAXPOOL: diff --git a/src/parser.c b/src/parser.c index b31e1cd0..499be075 100644 --- a/src/parser.c +++ b/src/parser.c @@ -29,6 +29,7 @@ #include "route_layer.h" #include "shortcut_layer.h" #include "softmax_layer.h" +#include "lstm_layer.h" #include "utils.h" typedef struct{ @@ -56,6 +57,7 @@ LAYER_TYPE string_to_layer_type(char * type) || strcmp(type, "[network]")==0) return NETWORK; if (strcmp(type, "[crnn]")==0) return CRNN; if (strcmp(type, "[gru]")==0) return GRU; + if (strcmp(type, "[lstm]") == 0) return LSTM; if (strcmp(type, "[rnn]")==0) return RNN; if (strcmp(type, "[conn]")==0 || strcmp(type, "[connected]")==0) return CONNECTED; @@ -239,6 +241,16 @@ layer parse_gru(list *options, size_params params) return l; } +layer parse_lstm(list *options, size_params params) +{ + int output = option_find_int(options, "output", 1); + int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0); + + layer l = make_lstm_layer(params.batch, params.inputs, output, params.time_steps, batch_normalize); + + return l; +} + connected_layer parse_connected(list *options, size_params params) { int output = option_find_int(options, "output",1); @@ -666,6 +678,8 @@ network parse_network_cfg(char *filename) l = parse_rnn(options, params); }else if(lt == GRU){ l = parse_gru(options, params); + }else if (lt == LSTM) { + l = parse_lstm(options, params); }else if(lt == CRNN){ l = parse_crnn(options, params); }else if(lt == CONNECTED){ @@ -906,14 +920,23 @@ void save_weights_upto(network net, char *filename, int cutoff) save_connected_weights(*(l.input_layer), fp); save_connected_weights(*(l.self_layer), fp); save_connected_weights(*(l.output_layer), fp); - } if(l.type == GRU){ - save_connected_weights(*(l.input_z_layer), fp); - save_connected_weights(*(l.input_r_layer), fp); - save_connected_weights(*(l.input_h_layer), fp); - save_connected_weights(*(l.state_z_layer), fp); - save_connected_weights(*(l.state_r_layer), fp); - save_connected_weights(*(l.state_h_layer), fp); - } if(l.type == CRNN){ + } if (l.type == LSTM) { + save_connected_weights(*(l.wi), fp); + save_connected_weights(*(l.wf), fp); + save_connected_weights(*(l.wo), fp); + save_connected_weights(*(l.wg), fp); + save_connected_weights(*(l.ui), fp); + save_connected_weights(*(l.uf), fp); + save_connected_weights(*(l.uo), fp); + save_connected_weights(*(l.ug), fp); + } if (l.type == GRU) { + save_connected_weights(*(l.wz), fp); + save_connected_weights(*(l.wr), fp); + save_connected_weights(*(l.wh), fp); + save_connected_weights(*(l.uz), fp); + save_connected_weights(*(l.ur), fp); + save_connected_weights(*(l.uh), fp); + } if(l.type == CRNN){ save_convolutional_weights(*(l.input_layer), fp); save_convolutional_weights(*(l.self_layer), fp); save_convolutional_weights(*(l.output_layer), fp); @@ -1105,14 +1128,24 @@ void load_weights_upto(network *net, char *filename, int start, int cutoff) load_connected_weights(*(l.self_layer), fp, transpose); load_connected_weights(*(l.output_layer), fp, transpose); } - if(l.type == GRU){ - load_connected_weights(*(l.input_z_layer), fp, transpose); - load_connected_weights(*(l.input_r_layer), fp, transpose); - load_connected_weights(*(l.input_h_layer), fp, transpose); - load_connected_weights(*(l.state_z_layer), fp, transpose); - load_connected_weights(*(l.state_r_layer), fp, transpose); - load_connected_weights(*(l.state_h_layer), fp, transpose); - } + if (l.type == LSTM) { + load_connected_weights(*(l.wi), fp, transpose); + load_connected_weights(*(l.wf), fp, transpose); + load_connected_weights(*(l.wo), fp, transpose); + load_connected_weights(*(l.wg), fp, transpose); + load_connected_weights(*(l.ui), fp, transpose); + load_connected_weights(*(l.uf), fp, transpose); + load_connected_weights(*(l.uo), fp, transpose); + load_connected_weights(*(l.ug), fp, transpose); + } + if (l.type == GRU) { + load_connected_weights(*(l.wz), fp, transpose); + load_connected_weights(*(l.wr), fp, transpose); + load_connected_weights(*(l.wh), fp, transpose); + load_connected_weights(*(l.uz), fp, transpose); + load_connected_weights(*(l.ur), fp, transpose); + load_connected_weights(*(l.uh), fp, transpose); + } if(l.type == LOCAL){ int locations = l.out_w*l.out_h; int size = l.size*l.size*l.c*l.n*locations; From 604a7606372714647ee46b0fc89091073b0cc7c2 Mon Sep 17 00:00:00 2001 From: Yao Lu Date: Tue, 6 Jun 2017 17:16:13 -0700 Subject: [PATCH 3/7] fix GRU, add LSTM --- Makefile | 2 +- include/darknet.h | 57 ++++++++++++++++++------------------ src/gru_layer.c | 1 - src/lstm_layer.h | 6 ++-- src/parser.c | 74 +++++++++++++++++++++++------------------------ 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/Makefile b/Makefile index 9ef36b84..d4a78aad 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -GPU=1 +GPU=0 CUDNN=0 OPENCV=0 DEBUG=0 diff --git a/include/darknet.h b/include/darknet.h index f2ef660a..06d426cb 100644 --- a/include/darknet.h +++ b/include/darknet.h @@ -63,7 +63,7 @@ typedef enum { ACTIVE, RNN, GRU, - LSTM, + LSTM, CRNN, BATCHNORM, NETWORK, @@ -253,20 +253,20 @@ struct layer{ struct layer *input_h_layer; struct layer *state_h_layer; - struct layer *wz; - struct layer *uz; - struct layer *wr; - struct layer *ur; - struct layer *wh; - struct layer *uh; - struct layer *uo; - struct layer *wo; - struct layer *uf; - struct layer *wf; - struct layer *ui; - struct layer *wi; - struct layer *ug; - struct layer *wg; + struct layer *wz; + struct layer *uz; + struct layer *wr; + struct layer *ur; + struct layer *wh; + struct layer *uh; + struct layer *uo; + struct layer *wo; + struct layer *uf; + struct layer *wf; + struct layer *ui; + struct layer *wi; + struct layer *ug; + struct layer *wg; tree *softmax_tree; @@ -279,20 +279,20 @@ struct layer{ float *r_gpu; float *h_gpu; - float *temp_gpu; - float *temp2_gpu; - float *temp3_gpu; + float *temp_gpu; + float *temp2_gpu; + float *temp3_gpu; - float *dh_gpu; - float *hh_gpu; - float *prev_cell_gpu; - float *cell_gpu; - float *f_gpu; - float *i_gpu; - float *g_gpu; - float *o_gpu; - float *c_gpu; - float *dc_gpu; + float *dh_gpu; + float *hh_gpu; + float *prev_cell_gpu; + float *cell_gpu; + float *f_gpu; + float *i_gpu; + float *g_gpu; + float *o_gpu; + float *c_gpu; + float *dc_gpu; float *m_gpu; float *v_gpu; @@ -546,6 +546,7 @@ list *read_cfg(char *filename); #include "dropout_layer.h" #include "gemm.h" #include "gru_layer.h" +#include "lstm_layer.h" #include "im2col.h" #include "image.h" #include "layer.h" diff --git a/src/gru_layer.c b/src/gru_layer.c index 78964817..917c36f9 100644 --- a/src/gru_layer.c +++ b/src/gru_layer.c @@ -185,7 +185,6 @@ void forward_gru_layer_gpu(layer l, network state) activate_array_ongpu(l.hh_gpu, l.outputs*l.batch, TANH); weighted_sum_gpu(l.h_gpu, l.hh_gpu, l.z_gpu, l.outputs*l.batch, l.output_gpu); - //ht = z .* ht-1 + (1-z) .* hh copy_ongpu(l.outputs*l.batch, l.output_gpu, 1, l.h_gpu, 1); state.input += l.inputs*l.batch; diff --git a/src/lstm_layer.h b/src/lstm_layer.h index a9ed792d..8ed387af 100644 --- a/src/lstm_layer.h +++ b/src/lstm_layer.h @@ -8,12 +8,12 @@ layer make_lstm_layer(int batch, int inputs, int outputs, int steps, int batch_normalize); -void forward_lstm_layer(layer l, network state); +void forward_lstm_layer(layer l, network net); void update_lstm_layer(layer l, int batch, float learning, float momentum, float decay); #ifdef GPU -void forward_lstm_layer_gpu(layer l, network state); -void backward_lstm_layer_gpu(layer l, network state); +void forward_lstm_layer_gpu(layer l, network net); +void backward_lstm_layer_gpu(layer l, network net); void update_lstm_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay); #endif diff --git a/src/parser.c b/src/parser.c index 499be075..16012212 100644 --- a/src/parser.c +++ b/src/parser.c @@ -57,7 +57,7 @@ LAYER_TYPE string_to_layer_type(char * type) || strcmp(type, "[network]")==0) return NETWORK; if (strcmp(type, "[crnn]")==0) return CRNN; if (strcmp(type, "[gru]")==0) return GRU; - if (strcmp(type, "[lstm]") == 0) return LSTM; + if (strcmp(type, "[lstm]") == 0) return LSTM; if (strcmp(type, "[rnn]")==0) return RNN; if (strcmp(type, "[conn]")==0 || strcmp(type, "[connected]")==0) return CONNECTED; @@ -678,8 +678,8 @@ network parse_network_cfg(char *filename) l = parse_rnn(options, params); }else if(lt == GRU){ l = parse_gru(options, params); - }else if (lt == LSTM) { - l = parse_lstm(options, params); + }else if (lt == LSTM) { + l = parse_lstm(options, params); }else if(lt == CRNN){ l = parse_crnn(options, params); }else if(lt == CONNECTED){ @@ -921,22 +921,22 @@ void save_weights_upto(network net, char *filename, int cutoff) save_connected_weights(*(l.self_layer), fp); save_connected_weights(*(l.output_layer), fp); } if (l.type == LSTM) { - save_connected_weights(*(l.wi), fp); - save_connected_weights(*(l.wf), fp); - save_connected_weights(*(l.wo), fp); - save_connected_weights(*(l.wg), fp); - save_connected_weights(*(l.ui), fp); - save_connected_weights(*(l.uf), fp); - save_connected_weights(*(l.uo), fp); - save_connected_weights(*(l.ug), fp); - } if (l.type == GRU) { - save_connected_weights(*(l.wz), fp); - save_connected_weights(*(l.wr), fp); - save_connected_weights(*(l.wh), fp); - save_connected_weights(*(l.uz), fp); - save_connected_weights(*(l.ur), fp); - save_connected_weights(*(l.uh), fp); - } if(l.type == CRNN){ + save_connected_weights(*(l.wi), fp); + save_connected_weights(*(l.wf), fp); + save_connected_weights(*(l.wo), fp); + save_connected_weights(*(l.wg), fp); + save_connected_weights(*(l.ui), fp); + save_connected_weights(*(l.uf), fp); + save_connected_weights(*(l.uo), fp); + save_connected_weights(*(l.ug), fp); + } if (l.type == GRU) { + save_connected_weights(*(l.wz), fp); + save_connected_weights(*(l.wr), fp); + save_connected_weights(*(l.wh), fp); + save_connected_weights(*(l.uz), fp); + save_connected_weights(*(l.ur), fp); + save_connected_weights(*(l.uh), fp); + } if(l.type == CRNN){ save_convolutional_weights(*(l.input_layer), fp); save_convolutional_weights(*(l.self_layer), fp); save_convolutional_weights(*(l.output_layer), fp); @@ -1128,24 +1128,24 @@ void load_weights_upto(network *net, char *filename, int start, int cutoff) load_connected_weights(*(l.self_layer), fp, transpose); load_connected_weights(*(l.output_layer), fp, transpose); } - if (l.type == LSTM) { - load_connected_weights(*(l.wi), fp, transpose); - load_connected_weights(*(l.wf), fp, transpose); - load_connected_weights(*(l.wo), fp, transpose); - load_connected_weights(*(l.wg), fp, transpose); - load_connected_weights(*(l.ui), fp, transpose); - load_connected_weights(*(l.uf), fp, transpose); - load_connected_weights(*(l.uo), fp, transpose); - load_connected_weights(*(l.ug), fp, transpose); - } - if (l.type == GRU) { - load_connected_weights(*(l.wz), fp, transpose); - load_connected_weights(*(l.wr), fp, transpose); - load_connected_weights(*(l.wh), fp, transpose); - load_connected_weights(*(l.uz), fp, transpose); - load_connected_weights(*(l.ur), fp, transpose); - load_connected_weights(*(l.uh), fp, transpose); - } + if (l.type == LSTM) { + load_connected_weights(*(l.wi), fp, transpose); + load_connected_weights(*(l.wf), fp, transpose); + load_connected_weights(*(l.wo), fp, transpose); + load_connected_weights(*(l.wg), fp, transpose); + load_connected_weights(*(l.ui), fp, transpose); + load_connected_weights(*(l.uf), fp, transpose); + load_connected_weights(*(l.uo), fp, transpose); + load_connected_weights(*(l.ug), fp, transpose); + } + if (l.type == GRU) { + load_connected_weights(*(l.wz), fp, transpose); + load_connected_weights(*(l.wr), fp, transpose); + load_connected_weights(*(l.wh), fp, transpose); + load_connected_weights(*(l.uz), fp, transpose); + load_connected_weights(*(l.ur), fp, transpose); + load_connected_weights(*(l.uh), fp, transpose); + } if(l.type == LOCAL){ int locations = l.out_w*l.out_h; int size = l.size*l.size*l.c*l.n*locations; From a4f5e71c6326ef991ea0a7669c1956a7fbd2f800 Mon Sep 17 00:00:00 2001 From: Yao Lu Date: Tue, 6 Jun 2017 17:20:26 -0700 Subject: [PATCH 4/7] fix GRU, add LSTM --- src/network.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/network.c b/src/network.c index b44e3a8b..86a98730 100644 --- a/src/network.c +++ b/src/network.c @@ -125,8 +125,8 @@ char *get_layer_string(LAYER_TYPE a) return "rnn"; case GRU: return "gru"; - case LSTM: - return "lstm"; + case LSTM: + return "lstm"; case CRNN: return "crnn"; case MAXPOOL: From 59262f4c773b5185a05722d055acfbc2ca816e51 Mon Sep 17 00:00:00 2001 From: Yao Lu Date: Tue, 6 Jun 2017 17:21:12 -0700 Subject: [PATCH 5/7] Revert "fix GRU, add LSTM" This reverts commit a4f5e71c6326ef991ea0a7669c1956a7fbd2f800. --- src/network.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/network.c b/src/network.c index 86a98730..b44e3a8b 100644 --- a/src/network.c +++ b/src/network.c @@ -125,8 +125,8 @@ char *get_layer_string(LAYER_TYPE a) return "rnn"; case GRU: return "gru"; - case LSTM: - return "lstm"; + case LSTM: + return "lstm"; case CRNN: return "crnn"; case MAXPOOL: From c04744d15e060e2ebedbfa9870289e8b1e8264ce Mon Sep 17 00:00:00 2001 From: Yao Lu Date: Tue, 6 Jun 2017 17:22:23 -0700 Subject: [PATCH 6/7] fix GRU, add LSTM --- src/network.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/network.c b/src/network.c index b44e3a8b..82eade31 100644 --- a/src/network.c +++ b/src/network.c @@ -125,8 +125,8 @@ char *get_layer_string(LAYER_TYPE a) return "rnn"; case GRU: return "gru"; - case LSTM: - return "lstm"; + case LSTM: + return "lstm"; case CRNN: return "crnn"; case MAXPOOL: From d286762c7aaccab7854b8433315b6360c2e82fb6 Mon Sep 17 00:00:00 2001 From: Yao Lu Date: Tue, 6 Jun 2017 17:23:35 -0700 Subject: [PATCH 7/7] fix GRU, add LSTM --- include/darknet.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/darknet.h b/include/darknet.h index 06d426cb..c8bbd1d9 100644 --- a/include/darknet.h +++ b/include/darknet.h @@ -186,7 +186,7 @@ struct layer{ float * forgot_state; float * forgot_delta; float * state_delta; - + float * concat; float * concat_delta;