First Commit!

.gitignore

@@ -0,0 +1,13 @@
+*.o
+*.dSYM
+*.csv
+images/
+opencv/
+cnn
+
+# OS Generated #
+.DS_Store*
+ehthumbs.db
+Icon?
+Thumbs.db
+*.swp

Makefile

@@ -0,0 +1,21 @@
+CC=gcc
+CFLAGS=-Wall `pkg-config --cflags opencv` -O3 -flto -ffast-math
+#CFLAGS=-Wall `pkg-config --cflags opencv` -O0 -g
+LDFLAGS=`pkg-config --libs opencv` -lm
+VPATH=./src/
+
+OBJ=network.o image.o tests.o convolutional_layer.o connected_layer.o maxpool_layer.o
+
+all: cnn
+
+cnn: $(OBJ)
+	$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@
+
+%.o: %.c 
+	$(CC) $(CFLAGS) -c $< -o $@
+
+.PHONY: clean
+
+clean:
+	rm -rf $(OBJ) cnn
+

src/connected_layer.c

@@ -0,0 +1,92 @@
+#include "connected_layer.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+double activation(double x)
+{
+    return x*(x>0);
+}
+
+double gradient(double x)
+{
+    return (x>=0);
+}
+
+connected_layer make_connected_layer(int inputs, int outputs)
+{
+    int i;
+    connected_layer layer;
+    layer.inputs = inputs;
+    layer.outputs = outputs;
+
+    layer.output = calloc(outputs, sizeof(double*));
+
+    layer.weight_updates = calloc(inputs*outputs, sizeof(double));
+    layer.weights = calloc(inputs*outputs, sizeof(double));
+    for(i = 0; i < inputs*outputs; ++i)
+        layer.weights[i] = .5 - (double)rand()/RAND_MAX;
+
+    layer.bias_updates = calloc(outputs, sizeof(double));
+    layer.biases = calloc(outputs, sizeof(double));
+    for(i = 0; i < outputs; ++i)
+        layer.biases[i] = (double)rand()/RAND_MAX;
+
+    return layer;
+}
+
+void run_connected_layer(double *input, connected_layer layer)
+{
+    int i, j;
+    for(i = 0; i < layer.outputs; ++i){
+        layer.output[i] = layer.biases[i];
+        for(j = 0; j < layer.inputs; ++j){
+            layer.output[i] += input[j]*layer.weights[i*layer.outputs + j];
+        }
+        layer.output[i] = activation(layer.output[i]);
+    }
+}
+
+void backpropagate_connected_layer(double *input, connected_layer layer)
+{
+    int i, j;
+    double *old_input = calloc(layer.inputs, sizeof(double));
+    memcpy(old_input, input, layer.inputs*sizeof(double));
+    memset(input, 0, layer.inputs*sizeof(double));
+
+    for(i = 0; i < layer.outputs; ++i){
+        for(j = 0; j < layer.inputs; ++j){
+            input[j] += layer.output[i]*layer.weights[i*layer.outputs + j];
+        }
+    }
+    for(j = 0; j < layer.inputs; ++j){
+        input[j] = input[j]*gradient(old_input[j]);
+    }
+    free(old_input);
+}
+
+void calculate_updates_connected_layer(double *input, connected_layer layer)
+{
+    int i, j;
+    for(i = 0; i < layer.outputs; ++i){
+        layer.bias_updates[i] += layer.output[i];
+        for(j = 0; j < layer.inputs; ++j){
+            layer.weight_updates[i*layer.outputs + j] += layer.output[i]*input[j];
+        }
+    }
+}
+
+void update_connected_layer(connected_layer layer, double step)
+{
+    int i,j;
+    for(i = 0; i < layer.outputs; ++i){
+        layer.biases[i] += step*layer.bias_updates[i];
+        for(j = 0; j < layer.inputs; ++j){
+            int index = i*layer.outputs+j;
+            layer.weights[index] = layer.weight_updates[index];
+        }
+    }
+    memset(layer.bias_updates, 0, layer.outputs*sizeof(double));
+    memset(layer.weight_updates, 0, layer.outputs*layer.inputs*sizeof(double));
+}
+

src/connected_layer.h

@@ -0,0 +1,21 @@
+#ifndef CONNECTED_LAYER_H
+#define CONNECTED_LAYER_H
+
+typedef struct{
+    int inputs;
+    int outputs;
+    double *weights;
+    double *biases;
+    double *weight_updates;
+    double *bias_updates;
+    double *output;
+} connected_layer;
+
+connected_layer make_connected_layer(int inputs, int outputs);
+void run_connected_layer(double *input, connected_layer layer);
+void backpropagate_connected_layer(double *input, connected_layer layer);
+void calculate_updates_connected_layer(double *input, connected_layer layer);
+void update_connected_layer(connected_layer layer, double step);
+
+#endif
+

src/convolutional_layer.c

@@ -0,0 +1,86 @@
+#include "convolutional_layer.h"
+
+double convolution_activation(double x)
+{
+    return x*(x>0);
+}
+
+double convolution_gradient(double x)
+{
+    return (x>=0);
+}
+
+convolutional_layer make_convolutional_layer(int h, int w, int c, int n, int size, int stride)
+{
+    int i;
+    convolutional_layer layer;
+    layer.n = n;
+    layer.stride = stride;
+    layer.kernels = calloc(n, sizeof(image));
+    layer.kernel_updates = calloc(n, sizeof(image));
+    for(i = 0; i < n; ++i){
+        layer.kernels[i] = make_random_kernel(size, c);
+        layer.kernel_updates[i] = make_random_kernel(size, c);
+    }
+    layer.output = make_image((h-1)/stride+1, (w-1)/stride+1, n);
+    layer.upsampled = make_image(h,w,n);
+    return layer;
+}
+
+void run_convolutional_layer(const image input, const convolutional_layer layer)
+{
+    int i;
+    for(i = 0; i < layer.n; ++i){
+        convolve(input, layer.kernels[i], layer.stride, i, layer.output);
+    }
+    for(i = 0; i < input.h*input.w*input.c; ++i){
+        input.data[i] = convolution_activation(input.data[i]);
+    }
+}
+
+void backpropagate_layer(image input, convolutional_layer layer)
+{
+    int i;
+    zero_image(input);
+    for(i = 0; i < layer.n; ++i){
+        back_convolve(input, layer.kernels[i], layer.stride, i, layer.output);
+    }
+}
+
+void backpropagate_layer_convolve(image input, convolutional_layer layer)
+{
+    int i,j;
+    for(i = 0; i < layer.n; ++i){
+        rotate_image(layer.kernels[i]);
+    }
+
+    zero_image(input);
+    upsample_image(layer.output, layer.stride, layer.upsampled);
+    for(j = 0; j < input.c; ++j){
+        for(i = 0; i < layer.n; ++i){
+            two_d_convolve(layer.upsampled, i, layer.kernels[i], j, 1, input, j);
+        }
+    }
+
+    for(i = 0; i < layer.n; ++i){
+        rotate_image(layer.kernels[i]);
+    }
+}
+
+void error_convolutional_layer(image input, convolutional_layer layer)
+{
+    int i;
+    for(i = 0; i < layer.n; ++i){
+        kernel_update(input, layer.kernel_updates[i], layer.stride, i, layer.output);
+    }
+    image old_input = copy_image(input);
+    zero_image(input);
+    for(i = 0; i < layer.n; ++i){
+        back_convolve(input, layer.kernels[i], layer.stride, i, layer.output);
+    }
+    for(i = 0; i < input.h*input.w*input.c; ++i){
+        input.data[i] = input.data[i]*convolution_gradient(input.data[i]);
+    }
+    free_image(old_input);
+}
+

src/convolutional_layer.h

@@ -0,0 +1,21 @@
+#ifndef CONVOLUTIONAL_LAYER_H
+#define CONVOLUTIONAL_LAYER_H
+
+#include "image.h"
+
+typedef struct {
+    int n;
+    int stride;
+    image *kernels;
+    image *kernel_updates;
+    image upsampled;
+    image output;
+} convolutional_layer;
+
+convolutional_layer make_convolutional_layer(int w, int h, int c, int n, int size, int stride);
+void run_convolutional_layer(const image input, const convolutional_layer layer);
+void backpropagate_layer(image input, convolutional_layer layer);
+void backpropagate_layer_convolve(image input, convolutional_layer layer);
+
+#endif
+

src/image.c

@@ -0,0 +1,348 @@
+#include "image.h"
+#include <stdio.h>
+
+int windows = 0;
+
+void subtract_image(image a, image b)
+{
+    int i;
+    for(i = 0; i < a.h*a.w*a.c; ++i) a.data[i] -= b.data[i];
+}
+
+void normalize_image(image p)
+{
+    double *min = calloc(p.c, sizeof(double));
+    double *max = calloc(p.c, sizeof(double));
+    int i,j;
+    for(i = 0; i < p.c; ++i) min[i] = max[i] = p.data[i*p.h*p.w];
+
+    for(j = 0; j < p.c; ++j){
+        for(i = 0; i < p.h*p.w; ++i){
+            double v = p.data[i+j*p.h*p.w];
+            if(v < min[j]) min[j] = v;
+            if(v > max[j]) max[j] = v;
+        }
+    }
+    for(i = 0; i < p.c; ++i){
+        if(max[i] - min[i] < .00001){
+            min[i] = 0;
+            max[i] = 1;
+        }
+    }
+    for(j = 0; j < p.c; ++j){
+        for(i = 0; i < p.w*p.h; ++i){
+            p.data[i+j*p.h*p.w] = (p.data[i+j*p.h*p.w] - min[j])/(max[j]-min[j]);
+        }
+    }
+}
+
+void threshold_image(image p, double t)
+{
+    int i;
+    for(i = 0; i < p.w*p.h*p.c; ++i){
+        if(p.data[i] < t) p.data[i] = 0;
+    }
+}
+
+image copy_image(image p)
+{
+    image copy = p;
+    copy.data = calloc(p.h*p.w*p.c, sizeof(double));
+    memcpy(copy.data, p.data, p.h*p.w*p.c*sizeof(double));
+    return copy;
+}
+
+void show_image(image p, char *name)
+{
+    int i,j,k;
+    image copy = copy_image(p);
+    normalize_image(copy);
+
+    char buff[256];
+    sprintf(buff, "%s (%d)", name, windows);
+
+    IplImage *disp = cvCreateImage(cvSize(p.w,p.h), IPL_DEPTH_8U, p.c);
+    int step = disp->widthStep;
+    cvNamedWindow(buff, CV_WINDOW_AUTOSIZE); 
+    cvMoveWindow(buff, 100*(windows%10) + 200*(windows/10), 100*(windows%10));
+    ++windows;
+    for(i = 0; i < p.h; ++i){
+        for(j = 0; j < p.w; ++j){
+            for(k= 0; k < p.c; ++k){
+                disp->imageData[i*step + j*p.c + k] = (unsigned char)(get_pixel(copy,i,j,k)*255);
+            }
+        }
+    }
+    if(disp->height < 100 || disp->width < 100){
+        IplImage *buffer = disp;
+        disp = cvCreateImage(cvSize(100,100*p.h/p.w), buffer->depth, buffer->nChannels);
+        cvResize(buffer, disp, CV_INTER_NN);
+        cvReleaseImage(&buffer);
+    }
+    cvShowImage(buff, disp);
+    cvReleaseImage(&disp);
+}
+
+void show_image_layers(image p, char *name)
+{
+    int i;
+    char buff[256];
+    for(i = 0; i < p.c; ++i){
+        sprintf(buff, "%s - Layer %d", name, i);
+        image layer = get_image_layer(p, i);
+        show_image(layer, buff);
+        free_image(layer);
+    }
+}
+
+image make_image(int h, int w, int c)
+{
+    image out;
+    out.h = h;
+    out.w = w;
+    out.c = c;
+    out.data = calloc(h*w*c, sizeof(double));
+    return out;
+}
+
+void zero_image(image m)
+{
+    memset(m.data, 0, m.h*m.w*m.c*sizeof(double));
+}
+
+void zero_channel(image m, int c)
+{
+    memset(&(m.data[c*m.h*m.w]), 0, m.h*m.w*sizeof(double));
+}
+
+void rotate_image(image m)
+{
+    int i,j;
+    for(j = 0; j < m.c; ++j){
+        for(i = 0; i < m.h*m.w/2; ++i){
+            double swap = m.data[j*m.h*m.w + i];
+            m.data[j*m.h*m.w + i] = m.data[j*m.h*m.w + (m.h*m.w-1 - i)];
+            m.data[j*m.h*m.w + (m.h*m.w-1 - i)] = swap;
+        }
+    }
+}
+
+image make_random_image(int h, int w, int c)
+{
+    image out = make_image(h,w,c);
+    int i;
+    for(i = 0; i < h*w*c; ++i){
+        out.data[i] = (double)rand()/RAND_MAX;
+    }
+    return out;
+}
+
+image make_random_kernel(int size, int c)
+{
+    int pad;
+    if((pad=(size%2==0))) ++size;
+    image out = make_random_image(size,size,c);
+    int i,k;
+    if(pad){
+        for(k = 0; k < out.c; ++k){
+            for(i = 0; i < size; ++i) {
+                set_pixel(out, i, 0, k, 0);
+                set_pixel(out, 0, i, k, 0);
+            }
+        }
+    }
+    return out;
+}
+
+
+image load_image(char *filename)
+{
+    IplImage* src = 0;
+    if( (src = cvLoadImage(filename,-1)) == 0 )
+    {
+        printf("Cannot load file image %s\n", filename);
+        exit(0);
+    }
+    unsigned char *data = (unsigned char *)src->imageData;
+    int c = src->nChannels;
+    int h = src->height;
+    int w = src->width;
+    int step = src->widthStep;
+    image out = make_image(h,w,c);
+    int i, j, k, count=0;;
+
+    for(k= 0; k < c; ++k){
+        for(i = 0; i < h; ++i){
+            for(j = 0; j < w; ++j){
+                out.data[count++] = data[i*step + j*c + k];
+            }
+        }
+    }
+    cvReleaseImage(&src);
+    return out;
+}
+
+image get_image_layer(image m, int l)
+{
+    image out = make_image(m.h, m.w, 1);
+    int i;
+    for(i = 0; i < m.h*m.w; ++i){
+        out.data[i] = m.data[i+l*m.h*m.w];
+    }
+    return out;
+}
+
+double get_pixel(image m, int x, int y, int c)
+{
+    assert(x < m.h && y < m.w && c < m.c);
+    return m.data[c*m.h*m.w + x*m.w + y];
+}
+double get_pixel_extend(image m, int x, int y, int c)
+{
+    if(x < 0 || x >= m.h || y < 0 || y >= m.w || c < 0 || c >= m.c) return 0;
+    return get_pixel(m, x, y, c);
+}
+void set_pixel(image m, int x, int y, int c, double val)
+{
+    assert(x < m.h && y < m.w && c < m.c);
+    m.data[c*m.h*m.w + x*m.w + y] = val;
+}
+void set_pixel_extend(image m, int x, int y, int c, double val)
+{
+    if(x < 0 || x >= m.h || y < 0 || y >= m.w || c < 0 || c >= m.c) return;
+    set_pixel(m, x, y, c, val);
+}
+
+void add_pixel(image m, int x, int y, int c, double val)
+{
+    assert(x < m.h && y < m.w && c < m.c);
+    m.data[c*m.h*m.w + x*m.w + y] += val;
+}
+
+void add_pixel_extend(image m, int x, int y, int c, double val)
+{
+    if(x < 0 || x >= m.h || y < 0 || y >= m.w || c < 0 || c >= m.c) return;
+    add_pixel(m, x, y, c, val);
+}
+
+void two_d_convolve(image m, int mc, image kernel, int kc, int stride, image out, int oc)
+{
+    int x,y,i,j;
+    for(x = 0; x < m.h; x += stride){
+        for(y = 0; y < m.w; y += stride){
+            double sum = 0;
+            for(i = 0; i < kernel.h; ++i){
+                for(j = 0; j < kernel.w; ++j){
+                    sum += get_pixel(kernel, i, j, kc)*get_pixel_extend(m, x+i-kernel.h/2, y+j-kernel.w/2, mc);
+                }
+            }
+            add_pixel(out, x/stride, y/stride, oc, sum);
+        }
+    }
+}
+
+double single_convolve(image m, image kernel, int x, int y)
+{
+    double sum = 0;
+    int i, j, k;
+    for(i = 0; i < kernel.h; ++i){
+        for(j = 0; j < kernel.w; ++j){
+            for(k = 0; k < kernel.c; ++k){
+                sum += get_pixel(kernel, i, j, k)*get_pixel_extend(m, x+i-kernel.h/2, y+j-kernel.w/2, k);
+            }
+        }
+    }
+    return sum;
+}
+
+void convolve(image m, image kernel, int stride, int channel, image out)
+{
+    assert(m.c == kernel.c);
+    int i;
+    zero_channel(out, channel);
+    for(i = 0; i < m.c; ++i){
+        two_d_convolve(m, i, kernel, i, stride, out, channel);
+    }
+    /*
+    int j;
+    for(i = 0; i < m.h; i += stride){
+        for(j = 0; j < m.w; j += stride){
+            double val = single_convolve(m, kernel, i, j);
+            set_pixel(out, i/stride, j/stride, channel, val);
+        }
+    }
+    */
+}
+
+void upsample_image(image m, int stride, image out)
+{
+    int i,j,k;
+    zero_image(out);
+    for(k = 0; k < m.c; ++k){
+        for(i = 0; i < m.h; ++i){
+            for(j = 0; j< m.w; ++j){
+                double val = get_pixel(m, i, j, k);
+                set_pixel(out, i*stride, j*stride, k, val);
+            }
+        }
+    }
+}
+
+void single_update(image m, image update, int x, int y, double error)
+{
+    int i, j, k;
+    for(i = 0; i < update.h; ++i){
+        for(j = 0; j < update.w; ++j){
+            for(k = 0; k < update.c; ++k){
+                double val = get_pixel_extend(m, x+i-update.h/2, y+j-update.w/2, k);
+                add_pixel(update, i, j, k, val*error);
+            }
+        }
+    }
+}
+
+void kernel_update(image m, image update, int stride, int channel, image out)
+{
+    assert(m.c == update.c);
+    zero_image(update);
+    int i, j;
+    for(i = 0; i < m.h; i += stride){
+        for(j = 0; j < m.w; j += stride){
+            double error = get_pixel(out, i/stride, j/stride, channel);
+            single_update(m, update, i, j, error);
+        }
+    }
+    for(i = 0; i < update.h*update.w*update.c; ++i){
+        update.data[i] /= (m.h/stride)*(m.w/stride);
+    }
+}
+
+void single_back_convolve(image m, image kernel, int x, int y, double val)
+{
+    int i, j, k;
+    for(i = 0; i < kernel.h; ++i){
+        for(j = 0; j < kernel.w; ++j){
+            for(k = 0; k < kernel.c; ++k){
+                double pval = get_pixel(kernel, i, j, k) * val;
+                add_pixel_extend(m, x+i-kernel.h/2, y+j-kernel.w/2, k, pval);
+            }
+        }
+    }
+}
+
+void back_convolve(image m, image kernel, int stride, int channel, image out)
+{
+    assert(m.c == kernel.c);
+    int i, j;
+    for(i = 0; i < m.h; i += stride){
+        for(j = 0; j < m.w; j += stride){
+            double val = get_pixel(out, i/stride, j/stride, channel);
+            single_back_convolve(m, kernel, i, j, val);
+        }
+    }
+}
+
+void free_image(image m)
+{
+    free(m.data);
+}

src/image.h

@@ -0,0 +1,42 @@
+#ifndef IMAGE_H
+#define IMAGE_H
+
+#include "opencv2/highgui/highgui_c.h"
+#include "opencv2/imgproc/imgproc_c.h"
+typedef struct {
+    int h;
+    int w;
+    int c;
+    double *data;
+} image;
+
+void normalize_image(image p);
+void threshold_image(image p, double t);
+void zero_image(image m);
+void rotate_image(image m);
+
+void show_image(image p, char *name);
+void show_image_layers(image p, char *name);
+
+image make_image(int h, int w, int c);
+image make_random_image(int h, int w, int c);
+image make_random_kernel(int size, int c);
+image copy_image(image p);
+image load_image(char *filename);
+
+double get_pixel(image m, int x, int y, int c);
+double get_pixel_extend(image m, int x, int y, int c);
+void set_pixel(image m, int x, int y, int c, double val);
+
+
+image get_image_layer(image m, int l);
+
+void two_d_convolve(image m, int mc, image kernel, int kc, int stride, image out, int oc);
+void upsample_image(image m, int stride, image out);
+void convolve(image m, image kernel, int stride, int channel, image out);
+void back_convolve(image m, image kernel, int stride, int channel, image out);
+void kernel_update(image m, image update, int stride, int channel, image out);
+
+void free_image(image m);
+#endif
+

src/maxpool_layer.c

@@ -0,0 +1,24 @@
+#include "maxpool_layer.h"
+
+maxpool_layer make_maxpool_layer(int h, int w, int c, int stride)
+{
+    maxpool_layer layer;
+    layer.stride = stride;
+    layer.output = make_image((h-1)/stride+1, (w-1)/stride+1, c);
+    return layer;
+}
+
+void run_maxpool_layer(const image input, const maxpool_layer layer)
+{
+    int i,j,k;
+    for(i = 0; i < layer.output.h*layer.output.w*layer.output.c; ++i) layer.output.data[i] = -DBL_MAX;
+    for(i = 0; i < input.h; ++i){
+        for(j = 0; j < input.w; ++j){
+            for(k = 0; k < input.c; ++k){
+                double val = get_pixel(input, i, j, k);
+                double cur = get_pixel(layer.output, i/layer.stride, j/layer.stride, k);
+                if(val > cur) set_pixel(layer.output, i/layer.stride, j/layer.stride, k, val);
+            }
+        }
+    }
+}

src/maxpool_layer.h

@@ -0,0 +1,15 @@
+#ifndef MAXPOOL_LAYER_H
+#define MAXPOOL_LAYER_H
+
+#include "image.h"
+
+typedef struct {
+    int stride;
+    image output;
+} maxpool_layer;
+
+maxpool_layer make_maxpool_layer(int h, int w, int c, int stride);
+void run_maxpool_layer(const image input, const maxpool_layer layer);
+
+#endif
+

src/network.c

@@ -0,0 +1,48 @@
+#include "network.h"
+#include "image.h"
+
+#include "connected_layer.h"
+#include "convolutional_layer.h"
+#include "maxpool_layer.h"
+
+void run_network(image input, network net)
+{
+    int i;
+    double *input_d = 0;
+    for(i = 0; i < net.n; ++i){
+        if(net.types[i] == CONVOLUTIONAL){
+            convolutional_layer layer = *(convolutional_layer *)net.layers[i];
+            run_convolutional_layer(input, layer);
+            input = layer.output;
+            input_d = layer.output.data;
+        }
+        else if(net.types[i] == CONNECTED){
+            connected_layer layer = *(connected_layer *)net.layers[i];
+            run_connected_layer(input_d, layer);
+            input_d = layer.output;
+        }
+        else if(net.types[i] == MAXPOOL){
+            maxpool_layer layer = *(maxpool_layer *)net.layers[i];
+            run_maxpool_layer(input, layer);
+            input = layer.output;
+            input_d = layer.output.data;
+        }
+    }
+}
+
+image get_network_image(network net)
+{
+    int i;
+    for(i = net.n-1; i >= 0; --i){
+        if(net.types[i] == CONVOLUTIONAL){
+            convolutional_layer layer = *(convolutional_layer *)net.layers[i];
+            return layer.output;
+        }
+        else if(net.types[i] == MAXPOOL){
+            maxpool_layer layer = *(maxpool_layer *)net.layers[i];
+            return layer.output;
+        }
+    }
+    return make_image(1,1,1);
+}
+

src/network.h

@@ -0,0 +1,23 @@
+// Oh boy, why am I about to do this....
+#ifndef NETWORK_H
+#define NETWORK_H
+
+#include "image.h"
+
+typedef enum {
+    CONVOLUTIONAL,
+    CONNECTED,
+    MAXPOOL
+} LAYER_TYPE;
+
+typedef struct {
+    int n;
+    void **layers;
+    LAYER_TYPE *types;
+} network;
+
+void run_network(image input, network net);
+image get_network_image(network net);
+
+#endif
+

src/tests.c

@@ -0,0 +1,200 @@
+#include "connected_layer.h"
+#include "convolutional_layer.h"
+#include "maxpool_layer.h"
+#include "network.h"
+#include "image.h"
+
+#include <time.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+void test_convolve()
+{
+    image dog = load_image("dog.jpg");
+    //show_image_layers(dog, "Dog");
+    printf("dog channels %d\n", dog.c);
+    image kernel = make_random_image(3,3,dog.c);
+    image edge = make_image(dog.h, dog.w, 1);
+    int i;
+    clock_t start = clock(), end;
+    for(i = 0; i < 1000; ++i){
+        convolve(dog, kernel, 1, 0, edge);
+    }
+    end = clock();
+    printf("Convolutions: %lf seconds\n", (double)(end-start)/CLOCKS_PER_SEC);
+    show_image_layers(edge, "Test Convolve");
+}
+
+void test_color()
+{
+    image dog = load_image("test_color.png");
+    show_image_layers(dog, "Test Color");
+}
+
+void test_convolutional_layer()
+{
+    srand(0);
+    image dog = load_image("test_dog.jpg");
+    int i;
+    int n = 5;
+    int stride = 1;
+    int size = 8;
+    convolutional_layer layer = make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride);
+    char buff[256];
+    for(i = 0; i < n; ++i) {
+        sprintf(buff, "Kernel %d", i);
+        show_image(layer.kernels[i], buff);
+    }
+    run_convolutional_layer(dog, layer);
+    
+    maxpool_layer mlayer = make_maxpool_layer(layer.output.h, layer.output.w, layer.output.c, 3);
+    run_maxpool_layer(layer.output,mlayer);
+
+    show_image_layers(mlayer.output, "Test Maxpool Layer");
+}
+
+void test_load()
+{
+    image dog = load_image("dog.jpg");
+    show_image(dog, "Test Load");
+    show_image_layers(dog, "Test Load");
+}
+void test_upsample()
+{
+    image dog = load_image("dog.jpg");
+    int n = 3;
+    image up = make_image(n*dog.h, n*dog.w, dog.c);
+    upsample_image(dog, n, up);
+    show_image(up, "Test Upsample");
+    show_image_layers(up, "Test Upsample");
+}
+
+void test_rotate()
+{
+    int i;
+    image dog = load_image("dog.jpg");
+    clock_t start = clock(), end;
+    for(i = 0; i < 1001; ++i){
+        rotate_image(dog);
+    }
+    end = clock();
+    printf("Rotations: %lf seconds\n", (double)(end-start)/CLOCKS_PER_SEC);
+    show_image(dog, "Test Rotate");
+
+    image random = make_random_image(3,3,3);
+    show_image(random, "Test Rotate Random");
+    rotate_image(random);
+    show_image(random, "Test Rotate Random");
+    rotate_image(random);
+    show_image(random, "Test Rotate Random");
+}
+
+void test_network()
+{
+    network net;
+    net.n = 11;
+    net.layers = calloc(net.n, sizeof(void *));
+    net.types = calloc(net.n, sizeof(LAYER_TYPE));
+    net.types[0] = CONVOLUTIONAL;
+    net.types[1] = MAXPOOL;
+    net.types[2] = CONVOLUTIONAL;
+    net.types[3] = MAXPOOL;
+    net.types[4] = CONVOLUTIONAL;
+    net.types[5] = CONVOLUTIONAL;
+    net.types[6] = CONVOLUTIONAL;
+    net.types[7] = MAXPOOL;
+    net.types[8] = CONNECTED;
+    net.types[9] = CONNECTED;
+    net.types[10] = CONNECTED;
+
+    image dog = load_image("test_hinton.jpg");
+
+    int n = 48;
+    int stride = 4;
+    int size = 11;
+    convolutional_layer cl = make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride);
+    maxpool_layer ml = make_maxpool_layer(cl.output.h, cl.output.w, cl.output.c, 2);
+
+    n = 128;
+    size = 5;
+    stride = 1;
+    convolutional_layer cl2 = make_convolutional_layer(ml.output.h, ml.output.w, ml.output.c, n, size, stride);
+    maxpool_layer ml2 = make_maxpool_layer(cl2.output.h, cl2.output.w, cl2.output.c, 2);
+
+    n = 192;
+    size = 3;
+    convolutional_layer cl3 = make_convolutional_layer(ml2.output.h, ml2.output.w, ml2.output.c, n, size, stride);
+    convolutional_layer cl4 = make_convolutional_layer(cl3.output.h, cl3.output.w, cl3.output.c, n, size, stride);
+    n = 128;
+    convolutional_layer cl5 = make_convolutional_layer(cl4.output.h, cl4.output.w, cl4.output.c, n, size, stride);
+    maxpool_layer ml3 = make_maxpool_layer(cl5.output.h, cl5.output.w, cl5.output.c, 4);
+    connected_layer nl = make_connected_layer(ml3.output.h*ml3.output.w*ml3.output.c, 4096);
+    connected_layer nl2 = make_connected_layer(4096, 4096);
+    connected_layer nl3 = make_connected_layer(4096, 1000);
+
+    net.layers[0] = &cl;
+    net.layers[1] = &ml;
+    net.layers[2] = &cl2;
+    net.layers[3] = &ml2;
+    net.layers[4] = &cl3;
+    net.layers[5] = &cl4;
+    net.layers[6] = &cl5;
+    net.layers[7] = &ml3;
+    net.layers[8] = &nl;
+    net.layers[9] = &nl2;
+    net.layers[10] = &nl3;
+
+    int i;
+    clock_t start = clock(), end;
+    for(i = 0; i < 10; ++i){
+        run_network(dog, net);
+        rotate_image(dog);
+    }
+    end = clock();
+    printf("Ran %lf second per iteration\n", (double)(end-start)/CLOCKS_PER_SEC/10);
+
+    show_image_layers(get_network_image(net), "Test Network Layer");
+}
+void test_backpropagate()
+{
+    int n = 3;
+    int size = 4;
+    int stride = 10;
+    image dog = load_image("dog.jpg");
+    show_image(dog, "Test Backpropagate Input");
+    image dog_copy = copy_image(dog);
+    convolutional_layer cl = make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride);
+    run_convolutional_layer(dog, cl);
+    show_image(cl.output, "Test Backpropagate Output");
+    int i;
+    clock_t start = clock(), end;
+    for(i = 0; i < 100; ++i){
+        backpropagate_layer(dog_copy, cl);
+    }
+    end = clock();
+    printf("Backpropagate: %lf seconds\n", (double)(end-start)/CLOCKS_PER_SEC);
+    start = clock();
+    for(i = 0; i < 100; ++i){
+        backpropagate_layer_convolve(dog, cl);
+    }
+    end = clock();
+    printf("Backpropagate Using Convolutions: %lf seconds\n", (double)(end-start)/CLOCKS_PER_SEC);
+    show_image(dog_copy, "Test Backpropagate 1");
+    show_image(dog, "Test Backpropagate 2");
+    subtract_image(dog, dog_copy);
+    show_image(dog, "Test Backpropagate Difference");
+}
+
+int main()
+{
+    //test_backpropagate();
+    //test_convolve();
+    //test_upsample();
+    //test_rotate();
+    //test_load();
+    test_network();
+    //test_convolutional_layer();
+    //test_color();
+    cvWaitKey(0);
+    return 0;
+}