From d7286c273211ffeb1f56594f863d1ee9922be6d4 Mon Sep 17 00:00:00 2001 From: Joseph Redmon Date: Wed, 6 Nov 2013 16:09:41 -0800 Subject: [PATCH] Loading may or may not work. But probably. --- .gitignore | 1 + Makefile | 2 +- src/activations.c | 11 +++++++ src/activations.h | 9 +++++- src/connected_layer.c | 34 +++++++++++----------- src/connected_layer.h | 2 +- src/convolutional_layer.c | 20 ++++++------- src/convolutional_layer.h | 5 +++- src/image.h | 1 + src/maxpool_layer.c | 8 ++--- src/maxpool_layer.h | 2 +- src/network.c | 50 ++++++++++++++++++++++++++++++-- src/network.h | 6 +++- src/tests.c | 61 +++++++++++++++++++++++++++------------ 14 files changed, 155 insertions(+), 57 deletions(-) diff --git a/.gitignore b/.gitignore index d15b2e84..9153fbb0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ images/ opencv/ convnet/ +decaf/ cnn # OS Generated # diff --git a/Makefile b/Makefile index 4cddfd58..bf7cfd34 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CFLAGS=-Wall `pkg-config --cflags opencv` -O0 -g LDFLAGS=`pkg-config --libs opencv` -lm VPATH=./src/ -OBJ=network.o image.o tests.o convolutional_layer.o connected_layer.o maxpool_layer.o activations.o +OBJ=network.o image.o tests.o convolutional_layer.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o all: cnn diff --git a/src/activations.c b/src/activations.c index aef21cb0..a1280296 100644 --- a/src/activations.c +++ b/src/activations.c @@ -1,6 +1,17 @@ #include "activations.h" #include +#include +#include + +ACTIVATION get_activation(char *s) +{ + if (strcmp(s, "sigmoid")==0) return SIGMOID; + if (strcmp(s, "relu")==0) return RELU; + if (strcmp(s, "identity")==0) return IDENTITY; + fprintf(stderr, "Couldn't find activation function %s, going with ReLU\n", s); + return RELU; +} double identity_activation(double x) { diff --git a/src/activations.h b/src/activations.h index 294cf287..09584cc5 100644 --- a/src/activations.h +++ b/src/activations.h @@ -1,10 +1,17 @@ +#ifndef ACTIVATIONS_H +#define ACTIVATIONS_H + typedef enum{ SIGMOID, RELU, IDENTITY -}ACTIVATOR_TYPE; +}ACTIVATION; +ACTIVATION get_activation(char *s); double relu_activation(double x); double relu_gradient(double x); double sigmoid_activation(double x); double sigmoid_gradient(double x); double identity_activation(double x); double identity_gradient(double x); + +#endif + diff --git a/src/connected_layer.c b/src/connected_layer.c index 11143b9c..9fafc38c 100644 --- a/src/connected_layer.c +++ b/src/connected_layer.c @@ -4,34 +4,34 @@ #include #include -connected_layer make_connected_layer(int inputs, int outputs, ACTIVATOR_TYPE activator) +connected_layer *make_connected_layer(int inputs, int outputs, ACTIVATION activator) { int i; - connected_layer layer; - layer.inputs = inputs; - layer.outputs = outputs; + connected_layer *layer = calloc(1, sizeof(connected_layer)); + layer->inputs = inputs; + layer->outputs = outputs; - layer.output = calloc(outputs, sizeof(double*)); + layer->output = calloc(outputs, sizeof(double*)); - layer.weight_updates = calloc(inputs*outputs, sizeof(double)); - layer.weights = calloc(inputs*outputs, sizeof(double)); + layer->weight_updates = calloc(inputs*outputs, sizeof(double)); + layer->weights = calloc(inputs*outputs, sizeof(double)); for(i = 0; i < inputs*outputs; ++i) - layer.weights[i] = .5 - (double)rand()/RAND_MAX; + layer->weights[i] = .5 - (double)rand()/RAND_MAX; - layer.bias_updates = calloc(outputs, sizeof(double)); - layer.biases = calloc(outputs, sizeof(double)); + layer->bias_updates = calloc(outputs, sizeof(double)); + layer->biases = calloc(outputs, sizeof(double)); for(i = 0; i < outputs; ++i) - layer.biases[i] = (double)rand()/RAND_MAX; + layer->biases[i] = (double)rand()/RAND_MAX; if(activator == SIGMOID){ - layer.activation = sigmoid_activation; - layer.gradient = sigmoid_gradient; + layer->activation = sigmoid_activation; + layer->gradient = sigmoid_gradient; }else if(activator == RELU){ - layer.activation = relu_activation; - layer.gradient = relu_gradient; + layer->activation = relu_activation; + layer->gradient = relu_gradient; }else if(activator == IDENTITY){ - layer.activation = identity_activation; - layer.gradient = identity_gradient; + layer->activation = identity_activation; + layer->gradient = identity_gradient; } return layer; diff --git a/src/connected_layer.h b/src/connected_layer.h index 4f0e42c6..eaea3069 100644 --- a/src/connected_layer.h +++ b/src/connected_layer.h @@ -16,7 +16,7 @@ typedef struct{ double (* gradient)(); } connected_layer; -connected_layer make_connected_layer(int inputs, int outputs, ACTIVATOR_TYPE activator); +connected_layer *make_connected_layer(int inputs, int outputs, ACTIVATION activator); void run_connected_layer(double *input, connected_layer layer); void learn_connected_layer(double *input, connected_layer layer); diff --git a/src/convolutional_layer.c b/src/convolutional_layer.c index 80531333..7478158f 100644 --- a/src/convolutional_layer.c +++ b/src/convolutional_layer.c @@ -10,20 +10,20 @@ double convolution_gradient(double x) return (x>=0); } -convolutional_layer make_convolutional_layer(int h, int w, int c, int n, int size, int stride) +convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int size, int stride) { int i; - convolutional_layer layer; - layer.n = n; - layer.stride = stride; - layer.kernels = calloc(n, sizeof(image)); - layer.kernel_updates = calloc(n, sizeof(image)); + convolutional_layer *layer = calloc(1, sizeof(convolutional_layer)); + layer->n = n; + layer->stride = stride; + layer->kernels = calloc(n, sizeof(image)); + layer->kernel_updates = calloc(n, sizeof(image)); for(i = 0; i < n; ++i){ - layer.kernels[i] = make_random_kernel(size, c); - layer.kernel_updates[i] = make_random_kernel(size, c); + layer->kernels[i] = make_random_kernel(size, c); + layer->kernel_updates[i] = make_random_kernel(size, c); } - layer.output = make_image((h-1)/stride+1, (w-1)/stride+1, n); - layer.upsampled = make_image(h,w,n); + layer->output = make_image((h-1)/stride+1, (w-1)/stride+1, n); + layer->upsampled = make_image(h,w,n); return layer; } diff --git a/src/convolutional_layer.h b/src/convolutional_layer.h index 24287157..75be04b7 100644 --- a/src/convolutional_layer.h +++ b/src/convolutional_layer.h @@ -12,9 +12,12 @@ typedef struct { image output; } convolutional_layer; -convolutional_layer make_convolutional_layer(int w, int h, int c, int n, int size, int stride); +convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int size, int stride); void run_convolutional_layer(const image input, const convolutional_layer layer); void learn_convolutional_layer(image input, convolutional_layer layer); +void update_convolutional_layer(convolutional_layer layer, double step); +void backpropagate_convolutional_layer(image input, convolutional_layer layer); +void backpropagate_convolutional_layer_convolve(image input, convolutional_layer layer); #endif diff --git a/src/image.h b/src/image.h index 85306f05..e2fe8c05 100644 --- a/src/image.h +++ b/src/image.h @@ -14,6 +14,7 @@ void normalize_image(image p); void threshold_image(image p, double t); void zero_image(image m); void rotate_image(image m); +void subtract_image(image a, image b); void show_image(image p, char *name); void show_image_layers(image p, char *name); diff --git a/src/maxpool_layer.c b/src/maxpool_layer.c index 38ac582a..6f7d2a2d 100644 --- a/src/maxpool_layer.c +++ b/src/maxpool_layer.c @@ -1,10 +1,10 @@ #include "maxpool_layer.h" -maxpool_layer make_maxpool_layer(int h, int w, int c, int stride) +maxpool_layer *make_maxpool_layer(int h, int w, int c, int stride) { - maxpool_layer layer; - layer.stride = stride; - layer.output = make_image((h-1)/stride+1, (w-1)/stride+1, c); + maxpool_layer *layer = calloc(1, sizeof(maxpool_layer)); + layer->stride = stride; + layer->output = make_image((h-1)/stride+1, (w-1)/stride+1, c); return layer; } diff --git a/src/maxpool_layer.h b/src/maxpool_layer.h index 077bcfa6..4d7726df 100644 --- a/src/maxpool_layer.h +++ b/src/maxpool_layer.h @@ -8,7 +8,7 @@ typedef struct { image output; } maxpool_layer; -maxpool_layer make_maxpool_layer(int h, int w, int c, int stride); +maxpool_layer *make_maxpool_layer(int h, int w, int c, int stride); void run_maxpool_layer(const image input, const maxpool_layer layer); #endif diff --git a/src/network.c b/src/network.c index 0a74b632..53184d90 100644 --- a/src/network.c +++ b/src/network.c @@ -5,6 +5,15 @@ #include "convolutional_layer.h" #include "maxpool_layer.h" +network make_network(int n) +{ + network net; + net.n = n; + net.layers = calloc(net.n, sizeof(void *)); + net.types = calloc(net.n, sizeof(LAYER_TYPE)); + return net; +} + void run_network(image input, network net) { int i; @@ -84,9 +93,9 @@ void learn_network(image input, network net) } } -double *get_network_output(network net) + +double *get_network_output_layer(network net, int i) { - int i = net.n-1; if(net.types[i] == CONVOLUTIONAL){ convolutional_layer layer = *(convolutional_layer *)net.layers[i]; return layer.output.data; @@ -101,6 +110,43 @@ double *get_network_output(network net) } return 0; } + +int get_network_output_size_layer(network net, int i) +{ + if(net.types[i] == CONVOLUTIONAL){ + convolutional_layer layer = *(convolutional_layer *)net.layers[i]; + return layer.output.h*layer.output.w*layer.output.c; + } + else if(net.types[i] == MAXPOOL){ + maxpool_layer layer = *(maxpool_layer *)net.layers[i]; + return layer.output.h*layer.output.w*layer.output.c; + } + else if(net.types[i] == CONNECTED){ + connected_layer layer = *(connected_layer *)net.layers[i]; + return layer.outputs; + } + return 0; +} + +double *get_network_output(network net) +{ + int i = net.n-1; + return get_network_output_layer(net, i); +} + +image get_network_image_layer(network net, int i) +{ + if(net.types[i] == CONVOLUTIONAL){ + convolutional_layer layer = *(convolutional_layer *)net.layers[i]; + return layer.output; + } + else if(net.types[i] == MAXPOOL){ + maxpool_layer layer = *(maxpool_layer *)net.layers[i]; + return layer.output; + } + return make_image(0,0,0); +} + image get_network_image(network net) { int i; diff --git a/src/network.h b/src/network.h index 2fb9225f..ad2b1dcc 100644 --- a/src/network.h +++ b/src/network.h @@ -16,11 +16,15 @@ typedef struct { LAYER_TYPE *types; } network; +network make_network(int n); void run_network(image input, network net); -double *get_network_output(network net); void learn_network(image input, network net); void update_network(network net, double step); +double *get_network_output(network net); +double *get_network_output_layer(network net, int i); +int get_network_output_size_layer(network net, int i); image get_network_image(network net); +image get_network_image_layer(network net, int i); #endif diff --git a/src/tests.c b/src/tests.c index f2b50dc9..0e639be8 100644 --- a/src/tests.c +++ b/src/tests.c @@ -3,6 +3,7 @@ #include "maxpool_layer.h" #include "network.h" #include "image.h" +#include "parser.h" #include #include @@ -39,7 +40,7 @@ void test_convolutional_layer() int n = 3; int stride = 1; int size = 3; - convolutional_layer layer = make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride); + convolutional_layer layer = *make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride); char buff[256]; for(i = 0; i < n; ++i) { sprintf(buff, "Kernel %d", i); @@ -47,7 +48,7 @@ void test_convolutional_layer() } run_convolutional_layer(dog, layer); - maxpool_layer mlayer = make_maxpool_layer(layer.output.h, layer.output.w, layer.output.c, 2); + maxpool_layer mlayer = *make_maxpool_layer(layer.output.h, layer.output.w, layer.output.c, 2); run_maxpool_layer(layer.output,mlayer); show_image_layers(mlayer.output, "Test Maxpool Layer"); @@ -112,25 +113,25 @@ void test_network() int n = 48; int stride = 4; int size = 11; - convolutional_layer cl = make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride); - maxpool_layer ml = make_maxpool_layer(cl.output.h, cl.output.w, cl.output.c, 2); + convolutional_layer cl = *make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride); + maxpool_layer ml = *make_maxpool_layer(cl.output.h, cl.output.w, cl.output.c, 2); n = 128; size = 5; stride = 1; - convolutional_layer cl2 = make_convolutional_layer(ml.output.h, ml.output.w, ml.output.c, n, size, stride); - maxpool_layer ml2 = make_maxpool_layer(cl2.output.h, cl2.output.w, cl2.output.c, 2); + convolutional_layer cl2 = *make_convolutional_layer(ml.output.h, ml.output.w, ml.output.c, n, size, stride); + maxpool_layer ml2 = *make_maxpool_layer(cl2.output.h, cl2.output.w, cl2.output.c, 2); n = 192; size = 3; - convolutional_layer cl3 = make_convolutional_layer(ml2.output.h, ml2.output.w, ml2.output.c, n, size, stride); - convolutional_layer cl4 = make_convolutional_layer(cl3.output.h, cl3.output.w, cl3.output.c, n, size, stride); + convolutional_layer cl3 = *make_convolutional_layer(ml2.output.h, ml2.output.w, ml2.output.c, n, size, stride); + convolutional_layer cl4 = *make_convolutional_layer(cl3.output.h, cl3.output.w, cl3.output.c, n, size, stride); n = 128; - convolutional_layer cl5 = make_convolutional_layer(cl4.output.h, cl4.output.w, cl4.output.c, n, size, stride); - maxpool_layer ml3 = make_maxpool_layer(cl5.output.h, cl5.output.w, cl5.output.c, 4); - connected_layer nl = make_connected_layer(ml3.output.h*ml3.output.w*ml3.output.c, 4096, RELU); - connected_layer nl2 = make_connected_layer(4096, 4096, RELU); - connected_layer nl3 = make_connected_layer(4096, 1000, RELU); + convolutional_layer cl5 = *make_convolutional_layer(cl4.output.h, cl4.output.w, cl4.output.c, n, size, stride); + maxpool_layer ml3 = *make_maxpool_layer(cl5.output.h, cl5.output.w, cl5.output.c, 4); + connected_layer nl = *make_connected_layer(ml3.output.h*ml3.output.w*ml3.output.c, 4096, RELU); + connected_layer nl2 = *make_connected_layer(4096, 4096, RELU); + connected_layer nl3 = *make_connected_layer(4096, 1000, RELU); net.layers[0] = &cl; net.layers[1] = &ml; @@ -164,7 +165,7 @@ void test_backpropagate() image dog = load_image("dog.jpg"); show_image(dog, "Test Backpropagate Input"); image dog_copy = copy_image(dog); - convolutional_layer cl = make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride); + convolutional_layer cl = *make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride); run_convolutional_layer(dog, cl); show_image(cl.output, "Test Backpropagate Output"); int i; @@ -196,9 +197,9 @@ void test_ann() net.types[1] = CONNECTED; net.types[2] = CONNECTED; - connected_layer nl = make_connected_layer(1, 20, RELU); - connected_layer nl2 = make_connected_layer(20, 20, RELU); - connected_layer nl3 = make_connected_layer(20, 1, RELU); + connected_layer nl = *make_connected_layer(1, 20, RELU); + connected_layer nl2 = *make_connected_layer(20, 20, RELU); + connected_layer nl3 = *make_connected_layer(20, 1, RELU); net.layers[0] = &nl; net.layers[1] = &nl2; @@ -225,10 +226,34 @@ void test_ann() } +void test_parser() +{ + network net = parse_network_cfg("test.cfg"); + image t = make_image(1,1,1); + int count = 0; + + double avgerr = 0; + while(1){ + double v = ((double)rand()/RAND_MAX); + double truth = v*v; + set_pixel(t,0,0,0,v); + run_network(t, net); + double *out = get_network_output(net); + double err = pow((out[0]-truth),2.); + avgerr = .99 * avgerr + .01 * err; + //if(++count % 100000 == 0) printf("%f\n", avgerr); + if(++count % 100000 == 0) printf("%f %f :%f AVG %f \n", truth, out[0], err, avgerr); + out[0] = truth - out[0]; + learn_network(t, net); + update_network(net, .001); + } +} + int main() { + test_parser(); //test_backpropagate(); - test_ann(); + //test_ann(); //test_convolve(); //test_upsample(); //test_rotate();