mirror of
https://github.com/pjreddie/darknet.git
synced 2023-08-10 21:13:14 +03:00
Fix GRU, Add LSTM
This commit is contained in:
Yao Lu
parent
d528cbdb7b
commit
e9f3b79776
4
Makefile
4
Makefile
@ -1,4 +1,4 @@
|
||||
GPU=0
|
||||
GPU=1
|
||||
CUDNN=0
|
||||
OPENCV=0
|
||||
DEBUG=0
|
||||
@ -51,7 +51,7 @@ CFLAGS+= -DCUDNN
|
||||
LDFLAGS+= -lcudnn
|
||||
endif
|
||||
|
||||
OBJ=gemm.o utils.o cuda.o deconvolutional_layer.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o detection_layer.o route_layer.o box.o normalization_layer.o avgpool_layer.o layer.o local_layer.o shortcut_layer.o activation_layer.o rnn_layer.o gru_layer.o crnn_layer.o demo.o batchnorm_layer.o region_layer.o reorg_layer.o tree.o
|
||||
OBJ=gemm.o utils.o cuda.o deconvolutional_layer.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o detection_layer.o route_layer.o box.o normalization_layer.o avgpool_layer.o layer.o local_layer.o shortcut_layer.o activation_layer.o rnn_layer.o gru_layer.o crnn_layer.o demo.o batchnorm_layer.o region_layer.o reorg_layer.o tree.o lstm_layer.o
|
||||
EXECOBJA=captcha.o lsd.o super.o voxel.o art.o tag.o cifar.o go.o rnn.o rnn_vid.o compare.o segmenter.o regressor.o classifier.o coco.o dice.o yolo.o detector.o writing.o nightmare.o swag.o darknet.o
|
||||
ifeq ($(GPU), 1)
|
||||
LDFLAGS+= -lstdc++
|
||||
|
||||
@ -63,6 +63,7 @@ typedef enum {
|
||||
ACTIVE,
|
||||
RNN,
|
||||
GRU,
|
||||
LSTM,
|
||||
CRNN,
|
||||
BATCHNORM,
|
||||
NETWORK,
|
||||
@ -185,7 +186,7 @@ struct layer{
|
||||
float * forgot_state;
|
||||
float * forgot_delta;
|
||||
float * state_delta;
|
||||
|
||||
|
||||
float * concat;
|
||||
float * concat_delta;
|
||||
|
||||
@ -251,6 +252,21 @@ struct layer{
|
||||
|
||||
struct layer *input_h_layer;
|
||||
struct layer *state_h_layer;
|
||||
|
||||
struct layer *wz;
|
||||
struct layer *uz;
|
||||
struct layer *wr;
|
||||
struct layer *ur;
|
||||
struct layer *wh;
|
||||
struct layer *uh;
|
||||
struct layer *uo;
|
||||
struct layer *wo;
|
||||
struct layer *uf;
|
||||
struct layer *wf;
|
||||
struct layer *ui;
|
||||
struct layer *wi;
|
||||
struct layer *ug;
|
||||
struct layer *wg;
|
||||
|
||||
tree *softmax_tree;
|
||||
|
||||
@ -263,6 +279,21 @@ struct layer{
|
||||
float *r_gpu;
|
||||
float *h_gpu;
|
||||
|
||||
float *temp_gpu;
|
||||
float *temp2_gpu;
|
||||
float *temp3_gpu;
|
||||
|
||||
float *dh_gpu;
|
||||
float *hh_gpu;
|
||||
float *prev_cell_gpu;
|
||||
float *cell_gpu;
|
||||
float *f_gpu;
|
||||
float *i_gpu;
|
||||
float *g_gpu;
|
||||
float *o_gpu;
|
||||
float *c_gpu;
|
||||
float *dc_gpu;
|
||||
|
||||
float *m_gpu;
|
||||
float *v_gpu;
|
||||
float *bias_m_gpu;
|
||||
|
||||
526
src/gru_layer.c
526
src/gru_layer.c
@ -12,195 +12,100 @@
|
||||
|
||||
static void increment_layer(layer *l, int steps)
|
||||
{
|
||||
int num = l->outputs*l->batch*steps;
|
||||
l->output += num;
|
||||
l->delta += num;
|
||||
l->x += num;
|
||||
l->x_norm += num;
|
||||
int num = l->outputs*l->batch*steps;
|
||||
l->output += num;
|
||||
l->delta += num;
|
||||
l->x += num;
|
||||
l->x_norm += num;
|
||||
|
||||
#ifdef GPU
|
||||
l->output_gpu += num;
|
||||
l->delta_gpu += num;
|
||||
l->x_gpu += num;
|
||||
l->x_norm_gpu += num;
|
||||
l->output_gpu += num;
|
||||
l->delta_gpu += num;
|
||||
l->x_gpu += num;
|
||||
l->x_norm_gpu += num;
|
||||
#endif
|
||||
}
|
||||
|
||||
layer make_gru_layer(int batch, int inputs, int outputs, int steps, int batch_normalize)
|
||||
{
|
||||
fprintf(stderr, "GRU Layer: %d inputs, %d outputs\n", inputs, outputs);
|
||||
batch = batch / steps;
|
||||
layer l = {0};
|
||||
l.batch = batch;
|
||||
l.type = GRU;
|
||||
l.steps = steps;
|
||||
l.inputs = inputs;
|
||||
fprintf(stderr, "GRU Layer: %d inputs, %d outputs\n", inputs, outputs);
|
||||
batch = batch / steps;
|
||||
layer l = { 0 };
|
||||
l.batch = batch;
|
||||
l.type = GRU;
|
||||
l.steps = steps;
|
||||
l.inputs = inputs;
|
||||
|
||||
l.input_z_layer = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.input_z_layer) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.input_z_layer->batch = batch;
|
||||
l.wz = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.wz) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.wz->batch = batch;
|
||||
|
||||
l.state_z_layer = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.state_z_layer) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.state_z_layer->batch = batch;
|
||||
l.uz = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.uz) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.uz->batch = batch;
|
||||
|
||||
l.wr = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.wr) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.wr->batch = batch;
|
||||
|
||||
l.ur = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.ur) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.ur->batch = batch;
|
||||
|
||||
l.input_r_layer = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.input_r_layer) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.input_r_layer->batch = batch;
|
||||
l.wh = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.wh) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.wh->batch = batch;
|
||||
|
||||
l.state_r_layer = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.state_r_layer) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.state_r_layer->batch = batch;
|
||||
l.uh = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.uh) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.uh->batch = batch;
|
||||
|
||||
l.batch_normalize = batch_normalize;
|
||||
l.outputs = outputs;
|
||||
|
||||
l.output = calloc(outputs*batch*steps, sizeof(float));
|
||||
l.state = calloc(outputs*batch, sizeof(float));
|
||||
|
||||
l.input_h_layer = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.input_h_layer) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.input_h_layer->batch = batch;
|
||||
|
||||
l.state_h_layer = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.state_h_layer) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.state_h_layer->batch = batch;
|
||||
|
||||
#ifdef CUDNN
|
||||
cudnnSetTensor4dDescriptor(l.input_z_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.input_z_layer->out_c, l.input_z_layer->out_h, l.input_z_layer->out_w);
|
||||
cudnnSetTensor4dDescriptor(l.input_h_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.input_h_layer->out_c, l.input_h_layer->out_h, l.input_h_layer->out_w);
|
||||
cudnnSetTensor4dDescriptor(l.input_r_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.input_r_layer->out_c, l.input_r_layer->out_h, l.input_r_layer->out_w);
|
||||
cudnnSetTensor4dDescriptor(l.state_z_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.state_z_layer->out_c, l.state_z_layer->out_h, l.state_z_layer->out_w);
|
||||
cudnnSetTensor4dDescriptor(l.state_h_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.state_h_layer->out_c, l.state_h_layer->out_h, l.state_h_layer->out_w);
|
||||
cudnnSetTensor4dDescriptor(l.state_r_layer->dstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, batch, l.state_r_layer->out_c, l.state_r_layer->out_h, l.state_r_layer->out_w);
|
||||
#endif
|
||||
|
||||
l.batch_normalize = batch_normalize;
|
||||
|
||||
|
||||
l.outputs = outputs;
|
||||
l.output = calloc(outputs*batch*steps, sizeof(float));
|
||||
l.delta = calloc(outputs*batch*steps, sizeof(float));
|
||||
l.state = calloc(outputs*batch, sizeof(float));
|
||||
l.prev_state = calloc(outputs*batch, sizeof(float));
|
||||
l.forgot_state = calloc(outputs*batch, sizeof(float));
|
||||
l.forgot_delta = calloc(outputs*batch, sizeof(float));
|
||||
|
||||
l.r_cpu = calloc(outputs*batch, sizeof(float));
|
||||
l.z_cpu = calloc(outputs*batch, sizeof(float));
|
||||
l.h_cpu = calloc(outputs*batch, sizeof(float));
|
||||
|
||||
l.forward = forward_gru_layer;
|
||||
l.backward = backward_gru_layer;
|
||||
l.update = update_gru_layer;
|
||||
l.forward = forward_gru_layer;
|
||||
l.backward = backward_gru_layer;
|
||||
l.update = update_gru_layer;
|
||||
|
||||
#ifdef GPU
|
||||
l.forward_gpu = forward_gru_layer_gpu;
|
||||
l.backward_gpu = backward_gru_layer_gpu;
|
||||
l.update_gpu = update_gru_layer_gpu;
|
||||
l.forward_gpu = forward_gru_layer_gpu;
|
||||
l.backward_gpu = backward_gru_layer_gpu;
|
||||
l.update_gpu = update_gru_layer_gpu;
|
||||
|
||||
l.forgot_state_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.forgot_delta_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.prev_state_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.state_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.output_gpu = cuda_make_array(l.output, batch*outputs*steps);
|
||||
l.delta_gpu = cuda_make_array(l.delta, batch*outputs*steps);
|
||||
l.r_gpu = cuda_make_array(l.output_gpu, batch*outputs);
|
||||
l.z_gpu = cuda_make_array(l.output_gpu, batch*outputs);
|
||||
l.h_gpu = cuda_make_array(l.output_gpu, batch*outputs);
|
||||
l.prev_state_gpu = cuda_make_array(0, batch*outputs);
|
||||
l.output_gpu = cuda_make_array(0, batch*outputs*steps);
|
||||
l.delta_gpu = cuda_make_array(0, batch*outputs*steps);
|
||||
|
||||
l.r_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.z_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.hh_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.h_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.temp_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.temp2_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.temp3_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.dh_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
#endif
|
||||
|
||||
return l;
|
||||
return l;
|
||||
}
|
||||
|
||||
void update_gru_layer(layer l, int batch, float learning_rate, float momentum, float decay)
|
||||
{
|
||||
update_connected_layer(*(l.input_layer), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer(*(l.self_layer), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer(*(l.output_layer), batch, learning_rate, momentum, decay);
|
||||
}
|
||||
|
||||
void forward_gru_layer(layer l, network net)
|
||||
void forward_gru_layer(layer l, network state)
|
||||
{
|
||||
network s = net;
|
||||
s.train = net.train;
|
||||
int i;
|
||||
layer input_z_layer = *(l.input_z_layer);
|
||||
layer input_r_layer = *(l.input_r_layer);
|
||||
layer input_h_layer = *(l.input_h_layer);
|
||||
|
||||
layer state_z_layer = *(l.state_z_layer);
|
||||
layer state_r_layer = *(l.state_r_layer);
|
||||
layer state_h_layer = *(l.state_h_layer);
|
||||
|
||||
fill_cpu(l.outputs * l.batch * l.steps, 0, input_z_layer.delta, 1);
|
||||
fill_cpu(l.outputs * l.batch * l.steps, 0, input_r_layer.delta, 1);
|
||||
fill_cpu(l.outputs * l.batch * l.steps, 0, input_h_layer.delta, 1);
|
||||
|
||||
fill_cpu(l.outputs * l.batch * l.steps, 0, state_z_layer.delta, 1);
|
||||
fill_cpu(l.outputs * l.batch * l.steps, 0, state_r_layer.delta, 1);
|
||||
fill_cpu(l.outputs * l.batch * l.steps, 0, state_h_layer.delta, 1);
|
||||
if(net.train) {
|
||||
fill_cpu(l.outputs * l.batch * l.steps, 0, l.delta, 1);
|
||||
copy_cpu(l.outputs*l.batch, l.state, 1, l.prev_state, 1);
|
||||
}
|
||||
|
||||
for (i = 0; i < l.steps; ++i) {
|
||||
s.input = l.state;
|
||||
forward_connected_layer(state_z_layer, s);
|
||||
forward_connected_layer(state_r_layer, s);
|
||||
|
||||
s.input = net.input;
|
||||
forward_connected_layer(input_z_layer, s);
|
||||
forward_connected_layer(input_r_layer, s);
|
||||
forward_connected_layer(input_h_layer, s);
|
||||
|
||||
|
||||
copy_cpu(l.outputs*l.batch, input_z_layer.output, 1, l.z_cpu, 1);
|
||||
axpy_cpu(l.outputs*l.batch, 1, state_z_layer.output, 1, l.z_cpu, 1);
|
||||
|
||||
copy_cpu(l.outputs*l.batch, input_r_layer.output, 1, l.r_cpu, 1);
|
||||
axpy_cpu(l.outputs*l.batch, 1, state_r_layer.output, 1, l.r_cpu, 1);
|
||||
|
||||
activate_array(l.z_cpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array(l.r_cpu, l.outputs*l.batch, LOGISTIC);
|
||||
|
||||
copy_cpu(l.outputs*l.batch, l.state, 1, l.forgot_state, 1);
|
||||
mul_cpu(l.outputs*l.batch, l.r_cpu, 1, l.forgot_state, 1);
|
||||
|
||||
s.input = l.forgot_state;
|
||||
forward_connected_layer(state_h_layer, s);
|
||||
|
||||
copy_cpu(l.outputs*l.batch, input_h_layer.output, 1, l.h_cpu, 1);
|
||||
axpy_cpu(l.outputs*l.batch, 1, state_h_layer.output, 1, l.h_cpu, 1);
|
||||
|
||||
#ifdef USET
|
||||
activate_array(l.h_cpu, l.outputs*l.batch, TANH);
|
||||
#else
|
||||
activate_array(l.h_cpu, l.outputs*l.batch, LOGISTIC);
|
||||
#endif
|
||||
|
||||
weighted_sum_cpu(l.state, l.h_cpu, l.z_cpu, l.outputs*l.batch, l.output);
|
||||
|
||||
copy_cpu(l.outputs*l.batch, l.output, 1, l.state, 1);
|
||||
|
||||
net.input += l.inputs*l.batch;
|
||||
l.output += l.outputs*l.batch;
|
||||
increment_layer(&input_z_layer, 1);
|
||||
increment_layer(&input_r_layer, 1);
|
||||
increment_layer(&input_h_layer, 1);
|
||||
|
||||
increment_layer(&state_z_layer, 1);
|
||||
increment_layer(&state_r_layer, 1);
|
||||
increment_layer(&state_h_layer, 1);
|
||||
}
|
||||
}
|
||||
|
||||
void backward_gru_layer(layer l, network net)
|
||||
void backward_gru_layer(layer l, network state)
|
||||
{
|
||||
}
|
||||
|
||||
@ -216,189 +121,202 @@ void push_gru_layer(layer l)
|
||||
|
||||
void update_gru_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay)
|
||||
{
|
||||
update_connected_layer_gpu(*(l.input_r_layer), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.input_z_layer), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.input_h_layer), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.state_r_layer), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.state_z_layer), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.state_h_layer), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.wr), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.wz), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.wh), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.ur), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.uz), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.uh), batch, learning_rate, momentum, decay);
|
||||
}
|
||||
|
||||
void forward_gru_layer_gpu(layer l, network net)
|
||||
void forward_gru_layer_gpu(layer l, network state)
|
||||
{
|
||||
network s = net;
|
||||
s.train = net.train;
|
||||
int i;
|
||||
layer input_z_layer = *(l.input_z_layer);
|
||||
layer input_r_layer = *(l.input_r_layer);
|
||||
layer input_h_layer = *(l.input_h_layer);
|
||||
network s = { 0 };
|
||||
s.train = state.train;
|
||||
int i;
|
||||
layer wz = *(l.wz);
|
||||
layer wr = *(l.wr);
|
||||
layer wh = *(l.wh);
|
||||
|
||||
layer state_z_layer = *(l.state_z_layer);
|
||||
layer state_r_layer = *(l.state_r_layer);
|
||||
layer state_h_layer = *(l.state_h_layer);
|
||||
layer uz = *(l.uz);
|
||||
layer ur = *(l.ur);
|
||||
layer uh = *(l.uh);
|
||||
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, input_z_layer.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, input_r_layer.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, input_h_layer.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, wz.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, wr.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, wh.delta_gpu, 1);
|
||||
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, state_z_layer.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, state_r_layer.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, state_h_layer.delta_gpu, 1);
|
||||
if(net.train) {
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, l.delta_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, l.state_gpu, 1, l.prev_state_gpu, 1);
|
||||
}
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, uz.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, ur.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, uh.delta_gpu, 1);
|
||||
|
||||
for (i = 0; i < l.steps; ++i) {
|
||||
s.input_gpu = l.state_gpu;
|
||||
forward_connected_layer_gpu(state_z_layer, s);
|
||||
forward_connected_layer_gpu(state_r_layer, s);
|
||||
if (state.train) {
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, l.delta_gpu, 1);
|
||||
}
|
||||
|
||||
s.input_gpu = net.input_gpu;
|
||||
forward_connected_layer_gpu(input_z_layer, s);
|
||||
forward_connected_layer_gpu(input_r_layer, s);
|
||||
forward_connected_layer_gpu(input_h_layer, s);
|
||||
for (i = 0; i < l.steps; ++i) {
|
||||
s.input = l.h_gpu;
|
||||
forward_connected_layer_gpu(uz, s);
|
||||
forward_connected_layer_gpu(ur, s);
|
||||
|
||||
s.input = state.input;
|
||||
forward_connected_layer_gpu(wz, s);
|
||||
forward_connected_layer_gpu(wr, s);
|
||||
forward_connected_layer_gpu(wh, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, input_z_layer.output_gpu, 1, l.z_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, state_z_layer.output_gpu, 1, l.z_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, wz.output_gpu, 1, l.z_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, uz.output_gpu, 1, l.z_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, input_r_layer.output_gpu, 1, l.r_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, state_r_layer.output_gpu, 1, l.r_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, wr.output_gpu, 1, l.r_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, ur.output_gpu, 1, l.r_gpu, 1);
|
||||
|
||||
activate_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.state_gpu, 1, l.forgot_state_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.forgot_state_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, l.h_gpu, 1, l.hh_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.hh_gpu, 1);
|
||||
|
||||
s.input_gpu = l.forgot_state_gpu;
|
||||
forward_connected_layer_gpu(state_h_layer, s);
|
||||
s.input = l.hh_gpu;
|
||||
forward_connected_layer_gpu(uh, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, input_h_layer.output_gpu, 1, l.h_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, state_h_layer.output_gpu, 1, l.h_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, wh.output_gpu, 1, l.hh_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, uh.output_gpu, 1, l.hh_gpu, 1);
|
||||
|
||||
#ifdef USET
|
||||
activate_array_ongpu(l.h_gpu, l.outputs*l.batch, TANH);
|
||||
#else
|
||||
activate_array_ongpu(l.h_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
#endif
|
||||
activate_array_ongpu(l.hh_gpu, l.outputs*l.batch, TANH);
|
||||
|
||||
weighted_sum_gpu(l.state_gpu, l.h_gpu, l.z_gpu, l.outputs*l.batch, l.output_gpu);
|
||||
weighted_sum_gpu(l.h_gpu, l.hh_gpu, l.z_gpu, l.outputs*l.batch, l.output_gpu);
|
||||
//ht = z .* ht-1 + (1-z) .* hh
|
||||
copy_ongpu(l.outputs*l.batch, l.output_gpu, 1, l.h_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.output_gpu, 1, l.state_gpu, 1);
|
||||
state.input += l.inputs*l.batch;
|
||||
l.output_gpu += l.outputs*l.batch;
|
||||
|
||||
net.input_gpu += l.inputs*l.batch;
|
||||
l.output_gpu += l.outputs*l.batch;
|
||||
increment_layer(&input_z_layer, 1);
|
||||
increment_layer(&input_r_layer, 1);
|
||||
increment_layer(&input_h_layer, 1);
|
||||
increment_layer(&wz, 1);
|
||||
increment_layer(&wr, 1);
|
||||
increment_layer(&wh, 1);
|
||||
|
||||
increment_layer(&state_z_layer, 1);
|
||||
increment_layer(&state_r_layer, 1);
|
||||
increment_layer(&state_h_layer, 1);
|
||||
}
|
||||
increment_layer(&uz, 1);
|
||||
increment_layer(&ur, 1);
|
||||
increment_layer(&uh, 1);
|
||||
}
|
||||
}
|
||||
|
||||
void backward_gru_layer_gpu(layer l, network net)
|
||||
void backward_gru_layer_gpu(layer l, network state)
|
||||
{
|
||||
network s = net;
|
||||
s.train = net.train;
|
||||
int i;
|
||||
layer input_z_layer = *(l.input_z_layer);
|
||||
layer input_r_layer = *(l.input_r_layer);
|
||||
layer input_h_layer = *(l.input_h_layer);
|
||||
network s = { 0 };
|
||||
s.train = state.train;
|
||||
int i;
|
||||
layer wz = *(l.wz);
|
||||
layer wr = *(l.wr);
|
||||
layer wh = *(l.wh);
|
||||
|
||||
layer state_z_layer = *(l.state_z_layer);
|
||||
layer state_r_layer = *(l.state_r_layer);
|
||||
layer state_h_layer = *(l.state_h_layer);
|
||||
layer uz = *(l.uz);
|
||||
layer ur = *(l.ur);
|
||||
layer uh = *(l.uh);
|
||||
|
||||
increment_layer(&input_z_layer, l.steps - 1);
|
||||
increment_layer(&input_r_layer, l.steps - 1);
|
||||
increment_layer(&input_h_layer, l.steps - 1);
|
||||
increment_layer(&wz, l.steps - 1);
|
||||
increment_layer(&wr, l.steps - 1);
|
||||
increment_layer(&wh, l.steps - 1);
|
||||
|
||||
increment_layer(&state_z_layer, l.steps - 1);
|
||||
increment_layer(&state_r_layer, l.steps - 1);
|
||||
increment_layer(&state_h_layer, l.steps - 1);
|
||||
increment_layer(&uz, l.steps - 1);
|
||||
increment_layer(&ur, l.steps - 1);
|
||||
increment_layer(&uh, l.steps - 1);
|
||||
|
||||
net.input_gpu += l.inputs*l.batch*(l.steps-1);
|
||||
if(net.delta_gpu) net.delta_gpu += l.inputs*l.batch*(l.steps-1);
|
||||
l.output_gpu += l.outputs*l.batch*(l.steps-1);
|
||||
l.delta_gpu += l.outputs*l.batch*(l.steps-1);
|
||||
for (i = l.steps-1; i >= 0; --i) {
|
||||
if(i != 0) copy_ongpu(l.outputs*l.batch, l.output_gpu - l.outputs*l.batch, 1, l.prev_state_gpu, 1);
|
||||
float *prev_delta_gpu = (i == 0) ? 0 : l.delta_gpu - l.outputs*l.batch;
|
||||
state.input += l.inputs*l.batch*(l.steps - 1);
|
||||
if (state.delta) state.delta += l.inputs*l.batch*(l.steps - 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, input_z_layer.output_gpu, 1, l.z_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, state_z_layer.output_gpu, 1, l.z_gpu, 1);
|
||||
l.output_gpu += l.outputs*l.batch*(l.steps - 1);
|
||||
l.delta_gpu += l.outputs*l.batch*(l.steps - 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, input_r_layer.output_gpu, 1, l.r_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, state_r_layer.output_gpu, 1, l.r_gpu, 1);
|
||||
for (i = l.steps - 1; i >= 0; --i) {
|
||||
if (i>0) copy_ongpu(l.outputs*l.batch, l.output_gpu - l.outputs*l.batch, 1, l.prev_state_gpu, 1);
|
||||
l.dh_gpu = (i == 0) ? 0 : l.delta_gpu - l.outputs*l.batch;
|
||||
|
||||
activate_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
copy_ongpu(l.outputs*l.batch, wz.output_gpu, 1, l.z_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, uz.output_gpu, 1, l.z_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, input_h_layer.output_gpu, 1, l.h_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, state_h_layer.output_gpu, 1, l.h_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, wr.output_gpu, 1, l.r_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, ur.output_gpu, 1, l.r_gpu, 1);
|
||||
|
||||
#ifdef USET
|
||||
activate_array_ongpu(l.h_gpu, l.outputs*l.batch, TANH);
|
||||
#else
|
||||
activate_array_ongpu(l.h_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
#endif
|
||||
|
||||
weighted_delta_gpu(l.prev_state_gpu, l.h_gpu, l.z_gpu, prev_delta_gpu, input_h_layer.delta_gpu, input_z_layer.delta_gpu, l.outputs*l.batch, l.delta_gpu);
|
||||
activate_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
|
||||
#ifdef USET
|
||||
gradient_array_ongpu(l.h_gpu, l.outputs*l.batch, TANH, input_h_layer.delta_gpu);
|
||||
#else
|
||||
gradient_array_ongpu(l.h_gpu, l.outputs*l.batch, LOGISTIC, input_h_layer.delta_gpu);
|
||||
#endif
|
||||
copy_ongpu(l.outputs*l.batch, wh.output_gpu, 1, l.hh_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, uh.output_gpu, 1, l.hh_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, input_h_layer.delta_gpu, 1, state_h_layer.delta_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.prev_state_gpu, 1, l.forgot_state_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.forgot_state_gpu, 1);
|
||||
fill_ongpu(l.outputs*l.batch, 0, l.forgot_delta_gpu, 1);
|
||||
activate_array_ongpu(l.hh_gpu, l.outputs*l.batch, TANH);
|
||||
|
||||
s.input_gpu = l.forgot_state_gpu;
|
||||
s.delta_gpu = l.forgot_delta_gpu;
|
||||
|
||||
backward_connected_layer_gpu(state_h_layer, s);
|
||||
if(prev_delta_gpu) mult_add_into_gpu(l.outputs*l.batch, l.forgot_delta_gpu, l.r_gpu, prev_delta_gpu);
|
||||
mult_add_into_gpu(l.outputs*l.batch, l.forgot_delta_gpu, l.prev_state_gpu, input_r_layer.delta_gpu);
|
||||
copy_ongpu(l.outputs*l.batch, l.delta_gpu, 1, l.temp3_gpu, 1);
|
||||
|
||||
gradient_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC, input_r_layer.delta_gpu);
|
||||
copy_ongpu(l.outputs*l.batch, input_r_layer.delta_gpu, 1, state_r_layer.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs*l.batch, 1, l.temp_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, -1, l.z_gpu, 1, l.temp_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp_gpu, 1);
|
||||
gradient_array_ongpu(l.hh_gpu, l.outputs*l.batch, TANH, l.temp_gpu);
|
||||
|
||||
gradient_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC, input_z_layer.delta_gpu);
|
||||
copy_ongpu(l.outputs*l.batch, input_z_layer.delta_gpu, 1, state_z_layer.delta_gpu, 1);
|
||||
|
||||
s.input_gpu = l.prev_state_gpu;
|
||||
s.delta_gpu = prev_delta_gpu;
|
||||
|
||||
backward_connected_layer_gpu(state_r_layer, s);
|
||||
backward_connected_layer_gpu(state_z_layer, s);
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wh.delta_gpu, 1);
|
||||
s.input = state.input;
|
||||
s.delta = state.delta;
|
||||
backward_connected_layer_gpu(wh, s);
|
||||
|
||||
s.input_gpu = net.input_gpu;
|
||||
s.delta_gpu = net.delta_gpu;
|
||||
|
||||
backward_connected_layer_gpu(input_h_layer, s);
|
||||
backward_connected_layer_gpu(input_r_layer, s);
|
||||
backward_connected_layer_gpu(input_z_layer, s);
|
||||
copy_ongpu(l.outputs*l.batch, l.prev_state_gpu, 1, l.temp2_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.temp2_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, uh.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs*l.batch, 0, l.temp_gpu, 1);
|
||||
s.input = l.temp2_gpu;
|
||||
s.delta = l.temp_gpu;
|
||||
backward_connected_layer_gpu(uh, s);
|
||||
|
||||
net.input_gpu -= l.inputs*l.batch;
|
||||
if(net.delta_gpu) net.delta_gpu -= l.inputs*l.batch;
|
||||
l.output_gpu -= l.outputs*l.batch;
|
||||
l.delta_gpu -= l.outputs*l.batch;
|
||||
increment_layer(&input_z_layer, -1);
|
||||
increment_layer(&input_r_layer, -1);
|
||||
increment_layer(&input_h_layer, -1);
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, l.temp2_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.prev_state_gpu, 1, l.temp2_gpu, 1);
|
||||
gradient_array_ongpu(l.r_gpu, l.outputs*l.batch, LOGISTIC, l.temp2_gpu);
|
||||
|
||||
increment_layer(&state_z_layer, -1);
|
||||
increment_layer(&state_r_layer, -1);
|
||||
increment_layer(&state_h_layer, -1);
|
||||
}
|
||||
copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, wr.delta_gpu, 1);
|
||||
s.input = state.input;
|
||||
s.delta = state.delta;
|
||||
backward_connected_layer_gpu(wr, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, ur.delta_gpu, 1);
|
||||
s.input = l.prev_state_gpu;
|
||||
s.delta = l.dh_gpu;
|
||||
backward_connected_layer_gpu(ur, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, l.temp2_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.r_gpu, 1, l.temp2_gpu, 1);
|
||||
if (l.dh_gpu) axpy_ongpu(l.outputs*l.batch, 1, l.temp2_gpu, 1, l.dh_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp2_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.z_gpu, 1, l.temp2_gpu, 1);
|
||||
if (l.dh_gpu) axpy_ongpu(l.outputs*l.batch, 1, l.temp2_gpu, 1, l.dh_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp2_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.prev_state_gpu, 1, l.temp3_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.hh_gpu, 1, l.temp2_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, -1, l.temp2_gpu, 1, l.temp3_gpu, 1);
|
||||
gradient_array_ongpu(l.z_gpu, l.outputs*l.batch, LOGISTIC, l.temp3_gpu);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, wz.delta_gpu, 1);
|
||||
s.input = state.input;
|
||||
s.delta = state.delta;
|
||||
backward_connected_layer_gpu(wz, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, uz.delta_gpu, 1);
|
||||
s.input = l.prev_state_gpu;
|
||||
s.delta = l.dh_gpu;
|
||||
backward_connected_layer_gpu(uz, s);
|
||||
|
||||
state.input -= l.inputs*l.batch;
|
||||
if (state.delta) state.delta -= l.inputs*l.batch;
|
||||
l.output_gpu -= l.outputs*l.batch;
|
||||
l.delta_gpu -= l.outputs*l.batch;
|
||||
|
||||
increment_layer(&wz, -1);
|
||||
increment_layer(&wr, -1);
|
||||
increment_layer(&wh, -1);
|
||||
|
||||
increment_layer(&uz, -1);
|
||||
increment_layer(&ur, -1);
|
||||
increment_layer(&uh, -1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@ -8,13 +8,13 @@
|
||||
|
||||
layer make_gru_layer(int batch, int inputs, int outputs, int steps, int batch_normalize);
|
||||
|
||||
void forward_gru_layer(layer l, network net);
|
||||
void backward_gru_layer(layer l, network net);
|
||||
void forward_gru_layer(layer l, network state);
|
||||
void backward_gru_layer(layer l, network state);
|
||||
void update_gru_layer(layer l, int batch, float learning_rate, float momentum, float decay);
|
||||
|
||||
#ifdef GPU
|
||||
void forward_gru_layer_gpu(layer l, network net);
|
||||
void backward_gru_layer_gpu(layer l, network net);
|
||||
void forward_gru_layer_gpu(layer l, network state);
|
||||
void backward_gru_layer_gpu(layer l, network state);
|
||||
void update_gru_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay);
|
||||
void push_gru_layer(layer l);
|
||||
void pull_gru_layer(layer l);
|
||||
|
||||
365
src/lstm_layer.c
Normal file
365
src/lstm_layer.c
Normal file
@ -0,0 +1,365 @@
|
||||
#include "lstm_layer.h"
|
||||
#include "connected_layer.h"
|
||||
#include "utils.h"
|
||||
#include "cuda.h"
|
||||
#include "blas.h"
|
||||
#include "gemm.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
static void increment_layer(layer *l, int steps)
|
||||
{
|
||||
int num = l->outputs*l->batch*steps;
|
||||
l->output += num;
|
||||
l->delta += num;
|
||||
l->x += num;
|
||||
l->x_norm += num;
|
||||
|
||||
#ifdef GPU
|
||||
l->output_gpu += num;
|
||||
l->delta_gpu += num;
|
||||
l->x_gpu += num;
|
||||
l->x_norm_gpu += num;
|
||||
#endif
|
||||
}
|
||||
|
||||
layer make_lstm_layer(int batch, int inputs, int outputs, int steps, int batch_normalize)
|
||||
{
|
||||
fprintf(stderr, "LSTM Layer: %d inputs, %d outputs\n", inputs, outputs);
|
||||
batch = batch / steps;
|
||||
layer l = { 0 };
|
||||
l.batch = batch;
|
||||
l.type = LSTM;
|
||||
l.steps = steps;
|
||||
l.inputs = inputs;
|
||||
|
||||
l.uf = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.uf) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.uf->batch = batch;
|
||||
|
||||
l.wf = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.wf) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.wf->batch = batch;
|
||||
|
||||
l.ui = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.ui) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.ui->batch = batch;
|
||||
|
||||
l.wi = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.wi) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.wi->batch = batch;
|
||||
|
||||
l.ug = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.ug) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.ug->batch = batch;
|
||||
|
||||
l.wg = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.wg) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.wg->batch = batch;
|
||||
|
||||
l.uo = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.uo) = make_connected_layer(batch*steps, inputs, outputs, LINEAR, batch_normalize);
|
||||
l.uo->batch = batch;
|
||||
|
||||
l.wo = malloc(sizeof(layer));
|
||||
fprintf(stderr, "\t\t");
|
||||
*(l.wo) = make_connected_layer(batch*steps, outputs, outputs, LINEAR, batch_normalize);
|
||||
l.wo->batch = batch;
|
||||
|
||||
l.batch_normalize = batch_normalize;
|
||||
l.outputs = outputs;
|
||||
|
||||
l.output = calloc(outputs*batch*steps, sizeof(float));
|
||||
l.state = calloc(outputs*batch, sizeof(float));
|
||||
|
||||
l.forward = forward_lstm_layer;
|
||||
l.update = update_lstm_layer;
|
||||
|
||||
#ifdef GPU
|
||||
l.forward_gpu = forward_lstm_layer_gpu;
|
||||
l.backward_gpu = backward_lstm_layer_gpu;
|
||||
l.update_gpu = update_lstm_layer_gpu;
|
||||
|
||||
l.prev_state_gpu = cuda_make_array(0, batch*outputs);
|
||||
l.prev_cell_gpu = cuda_make_array(0, batch*outputs);
|
||||
|
||||
l.output_gpu = cuda_make_array(0, batch*outputs*steps);
|
||||
l.cell_gpu = cuda_make_array(0, batch*outputs*steps);
|
||||
l.delta_gpu = cuda_make_array(0, batch*l.outputs*steps);
|
||||
|
||||
l.f_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.i_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.g_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.o_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.c_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.h_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.temp_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.temp2_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.temp3_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.dc_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
l.dh_gpu = cuda_make_array(l.output, batch*outputs);
|
||||
#endif
|
||||
|
||||
return l;
|
||||
}
|
||||
|
||||
void update_lstm_layer(layer l, int batch, float learning_rate, float momentum, float decay)
|
||||
{
|
||||
}
|
||||
|
||||
void forward_lstm_layer(layer l, network state)
|
||||
{
|
||||
}
|
||||
|
||||
#ifdef GPU
|
||||
void update_lstm_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay)
|
||||
{
|
||||
update_connected_layer_gpu(*(l.wf), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.wi), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.wg), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.wo), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.uf), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.ui), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.ug), batch, learning_rate, momentum, decay);
|
||||
update_connected_layer_gpu(*(l.uo), batch, learning_rate, momentum, decay);
|
||||
}
|
||||
|
||||
void forward_lstm_layer_gpu(layer l, network state)
|
||||
{
|
||||
network s = { 0 };
|
||||
s.train = state.train;
|
||||
int i;
|
||||
layer wf = *(l.wf);
|
||||
layer wi = *(l.wi);
|
||||
layer wg = *(l.wg);
|
||||
layer wo = *(l.wo);
|
||||
|
||||
layer uf = *(l.uf);
|
||||
layer ui = *(l.ui);
|
||||
layer ug = *(l.ug);
|
||||
layer uo = *(l.uo);
|
||||
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, wf.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, wi.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, wg.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, wo.delta_gpu, 1);
|
||||
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, uf.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, ui.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, ug.delta_gpu, 1);
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, uo.delta_gpu, 1);
|
||||
if (state.train) {
|
||||
fill_ongpu(l.outputs * l.batch * l.steps, 0, l.delta_gpu, 1);
|
||||
}
|
||||
|
||||
for (i = 0; i < l.steps; ++i) {
|
||||
s.input = l.h_gpu;
|
||||
forward_connected_layer_gpu(wf, s);
|
||||
forward_connected_layer_gpu(wi, s);
|
||||
forward_connected_layer_gpu(wg, s);
|
||||
forward_connected_layer_gpu(wo, s);
|
||||
|
||||
s.input = state.input;
|
||||
forward_connected_layer_gpu(uf, s);
|
||||
forward_connected_layer_gpu(ui, s);
|
||||
forward_connected_layer_gpu(ug, s);
|
||||
forward_connected_layer_gpu(uo, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, wf.output_gpu, 1, l.f_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, uf.output_gpu, 1, l.f_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, wi.output_gpu, 1, l.i_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, ui.output_gpu, 1, l.i_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, wg.output_gpu, 1, l.g_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, ug.output_gpu, 1, l.g_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, wo.output_gpu, 1, l.o_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, uo.output_gpu, 1, l.o_gpu, 1);
|
||||
|
||||
activate_array_ongpu(l.f_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.i_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.g_gpu, l.outputs*l.batch, TANH);
|
||||
activate_array_ongpu(l.o_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.i_gpu, 1, l.temp_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.g_gpu, 1, l.temp_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.f_gpu, 1, l.c_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, l.temp_gpu, 1, l.c_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.c_gpu, 1, l.h_gpu, 1);
|
||||
activate_array_ongpu(l.h_gpu, l.outputs*l.batch, TANH);
|
||||
mul_ongpu(l.outputs*l.batch, l.o_gpu, 1, l.h_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.c_gpu, 1, l.cell_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, l.h_gpu, 1, l.output_gpu, 1);
|
||||
|
||||
state.input += l.inputs*l.batch;
|
||||
l.output_gpu += l.outputs*l.batch;
|
||||
l.cell_gpu += l.outputs*l.batch;
|
||||
|
||||
increment_layer(&wf, 1);
|
||||
increment_layer(&wi, 1);
|
||||
increment_layer(&wg, 1);
|
||||
increment_layer(&wo, 1);
|
||||
|
||||
increment_layer(&uf, 1);
|
||||
increment_layer(&ui, 1);
|
||||
increment_layer(&ug, 1);
|
||||
increment_layer(&uo, 1);
|
||||
}
|
||||
}
|
||||
|
||||
void backward_lstm_layer_gpu(layer l, network state)
|
||||
{
|
||||
network s = { 0 };
|
||||
s.train = state.train;
|
||||
int i;
|
||||
layer wf = *(l.wf);
|
||||
layer wi = *(l.wi);
|
||||
layer wg = *(l.wg);
|
||||
layer wo = *(l.wo);
|
||||
|
||||
layer uf = *(l.uf);
|
||||
layer ui = *(l.ui);
|
||||
layer ug = *(l.ug);
|
||||
layer uo = *(l.uo);
|
||||
|
||||
increment_layer(&wf, l.steps - 1);
|
||||
increment_layer(&wi, l.steps - 1);
|
||||
increment_layer(&wg, l.steps - 1);
|
||||
increment_layer(&wo, l.steps - 1);
|
||||
|
||||
increment_layer(&uf, l.steps - 1);
|
||||
increment_layer(&ui, l.steps - 1);
|
||||
increment_layer(&ug, l.steps - 1);
|
||||
increment_layer(&uo, l.steps - 1);
|
||||
|
||||
state.input += l.inputs*l.batch*(l.steps - 1);
|
||||
if (state.delta) state.delta += l.inputs*l.batch*(l.steps - 1);
|
||||
|
||||
l.output_gpu += l.outputs*l.batch*(l.steps - 1);
|
||||
l.cell_gpu += l.outputs*l.batch*(l.steps - 1);
|
||||
l.delta_gpu += l.outputs*l.batch*(l.steps - 1);
|
||||
|
||||
for (i = l.steps - 1; i >= 0; --i) {
|
||||
if (i != 0) copy_ongpu(l.outputs*l.batch, l.cell_gpu - l.outputs*l.batch, 1, l.prev_cell_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, l.cell_gpu, 1, l.c_gpu, 1);
|
||||
if (i != 0) copy_ongpu(l.outputs*l.batch, l.output_gpu - l.outputs*l.batch, 1, l.prev_state_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, l.output_gpu, 1, l.h_gpu, 1);
|
||||
|
||||
l.dh_gpu = (i == 0) ? 0 : l.delta_gpu - l.outputs*l.batch;
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, wf.output_gpu, 1, l.f_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, uf.output_gpu, 1, l.f_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, wi.output_gpu, 1, l.i_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, ui.output_gpu, 1, l.i_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, wg.output_gpu, 1, l.g_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, ug.output_gpu, 1, l.g_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, wo.output_gpu, 1, l.o_gpu, 1);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, uo.output_gpu, 1, l.o_gpu, 1);
|
||||
|
||||
activate_array_ongpu(l.f_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.i_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
activate_array_ongpu(l.g_gpu, l.outputs*l.batch, TANH);
|
||||
activate_array_ongpu(l.o_gpu, l.outputs*l.batch, LOGISTIC);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.delta_gpu, 1, l.temp3_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.c_gpu, 1, l.temp_gpu, 1);
|
||||
activate_array_ongpu(l.temp_gpu, l.outputs*l.batch, TANH);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp2_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.o_gpu, 1, l.temp2_gpu, 1);
|
||||
|
||||
gradient_array_ongpu(l.temp_gpu, l.outputs*l.batch, TANH, l.temp2_gpu);
|
||||
axpy_ongpu(l.outputs*l.batch, 1, l.dc_gpu, 1, l.temp2_gpu, 1);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.c_gpu, 1, l.temp_gpu, 1);
|
||||
activate_array_ongpu(l.temp_gpu, l.outputs*l.batch, TANH);
|
||||
mul_ongpu(l.outputs*l.batch, l.temp3_gpu, 1, l.temp_gpu, 1);
|
||||
gradient_array_ongpu(l.o_gpu, l.outputs*l.batch, LOGISTIC, l.temp_gpu);
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wo.delta_gpu, 1);
|
||||
s.input = l.prev_state_gpu;
|
||||
s.delta = l.dh_gpu;
|
||||
backward_connected_layer_gpu(wo, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, uo.delta_gpu, 1);
|
||||
s.input = state.input;
|
||||
s.delta = state.delta;
|
||||
backward_connected_layer_gpu(uo, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, l.temp_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.i_gpu, 1, l.temp_gpu, 1);
|
||||
gradient_array_ongpu(l.g_gpu, l.outputs*l.batch, TANH, l.temp_gpu);
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wg.delta_gpu, 1);
|
||||
s.input = l.prev_state_gpu;
|
||||
s.delta = l.dh_gpu;
|
||||
backward_connected_layer_gpu(wg, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, ug.delta_gpu, 1);
|
||||
s.input = state.input;
|
||||
s.delta = state.delta;
|
||||
backward_connected_layer_gpu(ug, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, l.temp_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.g_gpu, 1, l.temp_gpu, 1);
|
||||
gradient_array_ongpu(l.i_gpu, l.outputs*l.batch, LOGISTIC, l.temp_gpu);
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wi.delta_gpu, 1);
|
||||
s.input = l.prev_state_gpu;
|
||||
s.delta = l.dh_gpu;
|
||||
backward_connected_layer_gpu(wi, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, ui.delta_gpu, 1);
|
||||
s.input = state.input;
|
||||
s.delta = state.delta;
|
||||
backward_connected_layer_gpu(ui, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, l.temp_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.prev_cell_gpu, 1, l.temp_gpu, 1);
|
||||
gradient_array_ongpu(l.f_gpu, l.outputs*l.batch, LOGISTIC, l.temp_gpu);
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, wf.delta_gpu, 1);
|
||||
s.input = l.prev_state_gpu;
|
||||
s.delta = l.dh_gpu;
|
||||
backward_connected_layer_gpu(wf, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, uf.delta_gpu, 1);
|
||||
s.input = state.input;
|
||||
s.delta = state.delta;
|
||||
backward_connected_layer_gpu(uf, s);
|
||||
|
||||
copy_ongpu(l.outputs*l.batch, l.temp2_gpu, 1, l.temp_gpu, 1);
|
||||
mul_ongpu(l.outputs*l.batch, l.f_gpu, 1, l.temp_gpu, 1);
|
||||
copy_ongpu(l.outputs*l.batch, l.temp_gpu, 1, l.dc_gpu, 1);
|
||||
|
||||
state.input -= l.inputs*l.batch;
|
||||
if (state.delta) state.delta -= l.inputs*l.batch;
|
||||
l.output_gpu -= l.outputs*l.batch;
|
||||
l.cell_gpu -= l.outputs*l.batch;
|
||||
l.delta_gpu -= l.outputs*l.batch;
|
||||
|
||||
increment_layer(&wf, -1);
|
||||
increment_layer(&wi, -1);
|
||||
increment_layer(&wg, -1);
|
||||
increment_layer(&wo, -1);
|
||||
|
||||
increment_layer(&uf, -1);
|
||||
increment_layer(&ui, -1);
|
||||
increment_layer(&ug, -1);
|
||||
increment_layer(&uo, -1);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
20
src/lstm_layer.h
Normal file
20
src/lstm_layer.h
Normal file
@ -0,0 +1,20 @@
|
||||
#ifndef LSTM_LAYER_H
|
||||
#define LSTM_LAYER_H
|
||||
|
||||
#include "activations.h"
|
||||
#include "layer.h"
|
||||
#include "network.h"
|
||||
#define USET
|
||||
|
||||
layer make_lstm_layer(int batch, int inputs, int outputs, int steps, int batch_normalize);
|
||||
|
||||
void forward_lstm_layer(layer l, network state);
|
||||
void update_lstm_layer(layer l, int batch, float learning, float momentum, float decay);
|
||||
|
||||
#ifdef GPU
|
||||
void forward_lstm_layer_gpu(layer l, network state);
|
||||
void backward_lstm_layer_gpu(layer l, network state);
|
||||
void update_lstm_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay);
|
||||
|
||||
#endif
|
||||
#endif
|
||||
@ -125,6 +125,8 @@ char *get_layer_string(LAYER_TYPE a)
|
||||
return "rnn";
|
||||
case GRU:
|
||||
return "gru";
|
||||
case LSTM:
|
||||
return "lstm";
|
||||
case CRNN:
|
||||
return "crnn";
|
||||
case MAXPOOL:
|
||||
|
||||
65
src/parser.c
65
src/parser.c
@ -29,6 +29,7 @@
|
||||
#include "route_layer.h"
|
||||
#include "shortcut_layer.h"
|
||||
#include "softmax_layer.h"
|
||||
#include "lstm_layer.h"
|
||||
#include "utils.h"
|
||||
|
||||
typedef struct{
|
||||
@ -56,6 +57,7 @@ LAYER_TYPE string_to_layer_type(char * type)
|
||||
|| strcmp(type, "[network]")==0) return NETWORK;
|
||||
if (strcmp(type, "[crnn]")==0) return CRNN;
|
||||
if (strcmp(type, "[gru]")==0) return GRU;
|
||||
if (strcmp(type, "[lstm]") == 0) return LSTM;
|
||||
if (strcmp(type, "[rnn]")==0) return RNN;
|
||||
if (strcmp(type, "[conn]")==0
|
||||
|| strcmp(type, "[connected]")==0) return CONNECTED;
|
||||
@ -239,6 +241,16 @@ layer parse_gru(list *options, size_params params)
|
||||
return l;
|
||||
}
|
||||
|
||||
layer parse_lstm(list *options, size_params params)
|
||||
{
|
||||
int output = option_find_int(options, "output", 1);
|
||||
int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0);
|
||||
|
||||
layer l = make_lstm_layer(params.batch, params.inputs, output, params.time_steps, batch_normalize);
|
||||
|
||||
return l;
|
||||
}
|
||||
|
||||
connected_layer parse_connected(list *options, size_params params)
|
||||
{
|
||||
int output = option_find_int(options, "output",1);
|
||||
@ -666,6 +678,8 @@ network parse_network_cfg(char *filename)
|
||||
l = parse_rnn(options, params);
|
||||
}else if(lt == GRU){
|
||||
l = parse_gru(options, params);
|
||||
}else if (lt == LSTM) {
|
||||
l = parse_lstm(options, params);
|
||||
}else if(lt == CRNN){
|
||||
l = parse_crnn(options, params);
|
||||
}else if(lt == CONNECTED){
|
||||
@ -906,14 +920,23 @@ void save_weights_upto(network net, char *filename, int cutoff)
|
||||
save_connected_weights(*(l.input_layer), fp);
|
||||
save_connected_weights(*(l.self_layer), fp);
|
||||
save_connected_weights(*(l.output_layer), fp);
|
||||
} if(l.type == GRU){
|
||||
save_connected_weights(*(l.input_z_layer), fp);
|
||||
save_connected_weights(*(l.input_r_layer), fp);
|
||||
save_connected_weights(*(l.input_h_layer), fp);
|
||||
save_connected_weights(*(l.state_z_layer), fp);
|
||||
save_connected_weights(*(l.state_r_layer), fp);
|
||||
save_connected_weights(*(l.state_h_layer), fp);
|
||||
} if(l.type == CRNN){
|
||||
} if (l.type == LSTM) {
|
||||
save_connected_weights(*(l.wi), fp);
|
||||
save_connected_weights(*(l.wf), fp);
|
||||
save_connected_weights(*(l.wo), fp);
|
||||
save_connected_weights(*(l.wg), fp);
|
||||
save_connected_weights(*(l.ui), fp);
|
||||
save_connected_weights(*(l.uf), fp);
|
||||
save_connected_weights(*(l.uo), fp);
|
||||
save_connected_weights(*(l.ug), fp);
|
||||
} if (l.type == GRU) {
|
||||
save_connected_weights(*(l.wz), fp);
|
||||
save_connected_weights(*(l.wr), fp);
|
||||
save_connected_weights(*(l.wh), fp);
|
||||
save_connected_weights(*(l.uz), fp);
|
||||
save_connected_weights(*(l.ur), fp);
|
||||
save_connected_weights(*(l.uh), fp);
|
||||
} if(l.type == CRNN){
|
||||
save_convolutional_weights(*(l.input_layer), fp);
|
||||
save_convolutional_weights(*(l.self_layer), fp);
|
||||
save_convolutional_weights(*(l.output_layer), fp);
|
||||
@ -1105,14 +1128,24 @@ void load_weights_upto(network *net, char *filename, int start, int cutoff)
|
||||
load_connected_weights(*(l.self_layer), fp, transpose);
|
||||
load_connected_weights(*(l.output_layer), fp, transpose);
|
||||
}
|
||||
if(l.type == GRU){
|
||||
load_connected_weights(*(l.input_z_layer), fp, transpose);
|
||||
load_connected_weights(*(l.input_r_layer), fp, transpose);
|
||||
load_connected_weights(*(l.input_h_layer), fp, transpose);
|
||||
load_connected_weights(*(l.state_z_layer), fp, transpose);
|
||||
load_connected_weights(*(l.state_r_layer), fp, transpose);
|
||||
load_connected_weights(*(l.state_h_layer), fp, transpose);
|
||||
}
|
||||
if (l.type == LSTM) {
|
||||
load_connected_weights(*(l.wi), fp, transpose);
|
||||
load_connected_weights(*(l.wf), fp, transpose);
|
||||
load_connected_weights(*(l.wo), fp, transpose);
|
||||
load_connected_weights(*(l.wg), fp, transpose);
|
||||
load_connected_weights(*(l.ui), fp, transpose);
|
||||
load_connected_weights(*(l.uf), fp, transpose);
|
||||
load_connected_weights(*(l.uo), fp, transpose);
|
||||
load_connected_weights(*(l.ug), fp, transpose);
|
||||
}
|
||||
if (l.type == GRU) {
|
||||
load_connected_weights(*(l.wz), fp, transpose);
|
||||
load_connected_weights(*(l.wr), fp, transpose);
|
||||
load_connected_weights(*(l.wh), fp, transpose);
|
||||
load_connected_weights(*(l.uz), fp, transpose);
|
||||
load_connected_weights(*(l.ur), fp, transpose);
|
||||
load_connected_weights(*(l.uh), fp, transpose);
|
||||
}
|
||||
if(l.type == LOCAL){
|
||||
int locations = l.out_w*l.out_h;
|
||||
int size = l.size*l.size*l.c*l.n*locations;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user