Some fixes to momentum

src/cnn.c

@@ -374,7 +374,7 @@ void train_detection_net()
 void train_imagenet_distributed(char *address)
 {
     float avg_loss = 1;
-    srand(0);
+    srand(time(0));
     network net = parse_network_cfg("cfg/alexnet.client");
     printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
     int imgs = 1000/net.batch+1;
@@ -412,11 +412,11 @@ void train_imagenet()
 {
     float avg_loss = 1;
     //network net = parse_network_cfg("/home/pjreddie/imagenet_backup/alexnet_1270.cfg");
-    srand(0);
+    srand(time(0));
     network net = parse_network_cfg("cfg/alexnet.cfg");
     printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
     int imgs = 1000/net.batch+1;
-    imgs=1;
+    //imgs=1;
     int i = 0;
     char **labels = get_labels("/home/pjreddie/data/imagenet/cls.labels.list");
     list *plist = get_paths("/data/imagenet/cls.train.list");
@@ -872,7 +872,7 @@ void test_correct_alexnet()
 
 void run_server()
 {
-    srand(0);
+    srand(time(0));
     network net = parse_network_cfg("cfg/alexnet.server");
     server_update(net);
 }

src/connected_layer.c

@@ -24,22 +24,20 @@ connected_layer *make_connected_layer(int batch, int inputs, int outputs, ACTIVA
     layer->delta = calloc(batch*outputs, sizeof(float*));
 
     layer->weight_updates = calloc(inputs*outputs, sizeof(float));
-    //layer->weight_adapt = calloc(inputs*outputs, sizeof(float));
     layer->weights = calloc(inputs*outputs, sizeof(float));
     float scale = 1./inputs;
     scale = .01;
-    for(i = 0; i < inputs*outputs; ++i)
-        layer->weights[i] = scale*2*(rand_uniform()-.5);
-
-    layer->bias_updates = calloc(outputs, sizeof(float));
-    //layer->bias_adapt = calloc(outputs, sizeof(float));
-    layer->biases = calloc(outputs, sizeof(float));
-    for(i = 0; i < outputs; ++i){
-        //layer->biases[i] = rand_normal()*scale + scale;
-        layer->biases[i] = 1;
+    for(i = 0; i < inputs*outputs; ++i){
+        layer->weights[i] = scale*rand_normal();
     }
 
-    #ifdef GPU
+    layer->bias_updates = calloc(outputs, sizeof(float));
+    layer->biases = calloc(outputs, sizeof(float));
+    for(i = 0; i < outputs; ++i){
+        layer->biases[i] = .01;
+    }
+
+#ifdef GPU
     layer->weights_cl = cl_make_array(layer->weights, inputs*outputs);
     layer->biases_cl = cl_make_array(layer->biases, outputs);
 
@@ -48,7 +46,7 @@ connected_layer *make_connected_layer(int batch, int inputs, int outputs, ACTIVA
 
     layer->output_cl = cl_make_array(layer->output, outputs*batch);
     layer->delta_cl = cl_make_array(layer->delta, outputs*batch);
-    #endif
+#endif
     layer->activation = activation;
     fprintf(stderr, "Connected Layer: %d inputs, %d outputs\n", inputs, outputs);
     return layer;
@@ -59,7 +57,7 @@ void update_connected_layer(connected_layer layer)
     axpy_cpu(layer.outputs, layer.learning_rate, layer.bias_updates, 1, layer.biases, 1);
     scal_cpu(layer.outputs, layer.momentum, layer.bias_updates, 1);
 
-    scal_cpu(layer.inputs*layer.outputs, 1.-layer.learning_rate*layer.decay, layer.weights, 1);
+    axpy_cpu(layer.inputs*layer.outputs, -layer.decay, layer.weights, 1, layer.weight_updates, 1);
     axpy_cpu(layer.inputs*layer.outputs, layer.learning_rate, layer.weight_updates, 1, layer.weights, 1);
     scal_cpu(layer.inputs*layer.outputs, layer.momentum, layer.weight_updates, 1);
 }
@@ -129,7 +127,7 @@ void update_connected_layer_gpu(connected_layer layer)
     axpy_ongpu(layer.outputs, layer.learning_rate, layer.bias_updates_cl, 1, layer.biases_cl, 1);
     scal_ongpu(layer.outputs, layer.momentum, layer.bias_updates_cl, 1);
 
-    scal_ongpu(layer.inputs*layer.outputs, 1.-layer.learning_rate*layer.decay, layer.weights_cl, 1);
+    axpy_ongpu(layer.inputs*layer.outputs, -layer.decay, layer.weights_cl, 1, layer.weight_updates_cl, 1);
     axpy_ongpu(layer.inputs*layer.outputs, layer.learning_rate, layer.weight_updates_cl, 1, layer.weights_cl, 1);
     scal_ongpu(layer.inputs*layer.outputs, layer.momentum, layer.weight_updates_cl, 1);
     pull_connected_layer(layer);
@@ -176,4 +174,4 @@ void backward_connected_layer_gpu(connected_layer layer, cl_mem input, cl_mem de
 
     if(c) gemm_ongpu(0,1,m,n,k,1,a,k,b,k,0,c,n);
 }
-    #endif
+#endif

src/convolutional_layer.c

@@ -64,10 +64,10 @@ convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, in
     layer->bias_updates = calloc(n, sizeof(float));
     float scale = 1./(size*size*c);
     scale = .01;
-    for(i = 0; i < c*n*size*size; ++i) layer->filters[i] = scale*2*(rand_uniform()-.5);
+    for(i = 0; i < c*n*size*size; ++i) layer->filters[i] = scale*rand_normal();
     for(i = 0; i < n; ++i){
         //layer->biases[i] = rand_normal()*scale + scale;
-        layer->biases[i] = .5;
+        layer->biases[i] = .01;
     }
     int out_h = convolutional_out_height(*layer);
     int out_w = convolutional_out_width(*layer);
@@ -204,7 +204,7 @@ void update_convolutional_layer(convolutional_layer layer)
     axpy_cpu(layer.n, layer.learning_rate, layer.bias_updates, 1, layer.biases, 1);
     scal_cpu(layer.n, layer.momentum, layer.bias_updates, 1);
 
-    scal_cpu(size, 1.-layer.learning_rate*layer.decay, layer.filters, 1);
+    axpy_cpu(size, -layer.decay, layer.filters, 1, layer.filter_updates, 1);
     axpy_cpu(size, layer.learning_rate, layer.filter_updates, 1, layer.filters, 1);
     scal_cpu(size, layer.momentum, layer.filter_updates, 1);
 }
@@ -409,7 +409,7 @@ void update_convolutional_layer_gpu(convolutional_layer layer)
     axpy_ongpu(layer.n, layer.learning_rate, layer.bias_updates_cl, 1, layer.biases_cl, 1);
     scal_ongpu(layer.n,layer.momentum, layer.bias_updates_cl, 1);
 
-    scal_ongpu(size, 1.-layer.learning_rate*layer.decay, layer.filters_cl, 1);
+    axpy_ongpu(size, -layer.decay, layer.filters_cl, 1, layer.filter_updates_cl, 1);
     axpy_ongpu(size, layer.learning_rate, layer.filter_updates_cl, 1, layer.filters_cl, 1);
     scal_ongpu(size, layer.momentum, layer.filter_updates_cl, 1);
     pull_convolutional_layer(layer);

src/server.c

@@ -9,6 +9,7 @@
 
 #include "mini_blas.h"
 #include "utils.h"
+#include "parser.h"
 #include "server.h"
 #include "connected_layer.h"
 #include "convolutional_layer.h"
@@ -82,7 +83,6 @@ void handle_connection(void *pointer)
     connection_info info = *(connection_info *) pointer;
     int fd = info.fd;
     network net = info.net;
-    ++*(info.counter);
     int i;
     for(i = 0; i < net.n; ++i){
         if(net.types[i] == CONVOLUTIONAL){
@@ -117,6 +117,8 @@ void handle_connection(void *pointer)
     }
     printf("Received updates\n");
     close(fd);
+    ++*(info.counter);
+    if(*(info.counter)%10==0) save_network(net, "/home/pjreddie/imagenet_backup/alexnet.part");
 }
 
 void server_update(network net)

src/utils.c

@@ -262,10 +262,11 @@ int max_index(float *a, int n)
 
 float rand_normal()
 {
+    int n = 12;
     int i;
     float sum= 0;
-    for(i = 0; i < 12; ++i) sum += (float)rand()/RAND_MAX;
-    return sum-6.;
+    for(i = 0; i < n; ++i) sum += (float)rand()/RAND_MAX;
+    return sum-n/2.;
 }
 float rand_uniform()
 {