From d1965bdb969920c85f72785ec6e1f3d7bda957de Mon Sep 17 00:00:00 2001 From: Joseph Redmon Date: Sun, 13 Mar 2016 23:18:42 -0700 Subject: [PATCH] Go --- Makefile | 10 +- cfg/go.test.cfg | 67 ++ src/blas.c | 4 +- src/blas_kernels.cu | 16 +- src/cifar.c | 163 +++- src/classifier.c | 93 +- src/coco_demo.c | 10 +- src/convolutional_kernels.cu | 51 +- src/darknet.c | 9 +- src/data.c | 85 ++ src/data.h | 2 + src/go.c | 249 +++++ src/image.c | 1348 +++++++++++++------------- src/image.h | 2 + src/layer.h | 1 + src/matrix.c | 42 + src/matrix.h | 3 + src/parser.c | 9 + src/tag.c | 11 +- src/yolo.c | 10 +- src/{coco_kernels.cu => yolo_demo.c} | 76 +- 21 files changed, 1503 insertions(+), 758 deletions(-) create mode 100644 cfg/go.test.cfg create mode 100644 src/go.c rename src/{coco_kernels.cu => yolo_demo.c} (50%) diff --git a/Makefile b/Makefile index 528437d9..2ecf6cc1 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ -GPU=1 -OPENCV=1 +GPU=0 +OPENCV=0 DEBUG=0 -ARCH= --gpu-architecture=compute_20 --gpu-code=compute_20 +ARCH= --gpu-architecture=compute_20 --gpu-code=compute_20 VPATH=./src/ EXEC=darknet @@ -34,9 +34,9 @@ CFLAGS+= -DGPU LDFLAGS+= -L/usr/local/cuda/lib64 -lcuda -lcudart -lcublas -lcurand endif -OBJ=gemm.o utils.o cuda.o deconvolutional_layer.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o darknet.o detection_layer.o imagenet.o captcha.o route_layer.o writing.o box.o nightmare.o normalization_layer.o avgpool_layer.o coco.o dice.o yolo.o layer.o compare.o classifier.o local_layer.o swag.o shortcut_layer.o activation_layer.o rnn_layer.o rnn.o rnn_vid.o crnn_layer.o coco_demo.o tag.o cifar.o +OBJ=gemm.o utils.o cuda.o deconvolutional_layer.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o darknet.o detection_layer.o imagenet.o captcha.o route_layer.o writing.o box.o nightmare.o normalization_layer.o avgpool_layer.o coco.o dice.o yolo.o layer.o compare.o classifier.o local_layer.o swag.o shortcut_layer.o activation_layer.o rnn_layer.o rnn.o rnn_vid.o crnn_layer.o coco_demo.o tag.o cifar.o yolo_demo.o go.o ifeq ($(GPU), 1) -OBJ+=convolutional_kernels.o deconvolutional_kernels.o activation_kernels.o im2col_kernels.o col2im_kernels.o blas_kernels.o crop_layer_kernels.o dropout_layer_kernels.o maxpool_layer_kernels.o softmax_layer_kernels.o network_kernels.o avgpool_layer_kernels.o yolo_kernels.o +OBJ+=convolutional_kernels.o deconvolutional_kernels.o activation_kernels.o im2col_kernels.o col2im_kernels.o blas_kernels.o crop_layer_kernels.o dropout_layer_kernels.o maxpool_layer_kernels.o softmax_layer_kernels.o network_kernels.o avgpool_layer_kernels.o endif OBJS = $(addprefix $(OBJDIR), $(OBJ)) diff --git a/cfg/go.test.cfg b/cfg/go.test.cfg new file mode 100644 index 00000000..700d4f1e --- /dev/null +++ b/cfg/go.test.cfg @@ -0,0 +1,67 @@ +[net] +batch=1 +subdivisions=1 +height=19 +width=19 +channels=1 +momentum=0.9 +decay=0.0005 + +learning_rate=0.1 +max_batches = 0 +policy=steps +steps=50000, 90000 +scales=.1, .1 + +[convolutional] +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky +batch_normalize=1 + +[convolutional] +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky +batch_normalize=1 + +[convolutional] +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky +batch_normalize=1 + +[convolutional] +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky +batch_normalize=1 + +[convolutional] +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky +batch_normalize=1 + +[convolutional] +filters=1 +size=1 +stride=1 +pad=1 +activation=leaky + +[softmax] + +[cost] +type=sse + diff --git a/src/blas.c b/src/blas.c index 978f1ed7..35a4c40a 100644 --- a/src/blas.c +++ b/src/blas.c @@ -46,7 +46,7 @@ void mean_cpu(float *x, int batch, int filters, int spatial, float *mean) void variance_cpu(float *x, float *mean, int batch, int filters, int spatial, float *variance) { - float scale = 1./(batch * spatial); + float scale = 1./(batch * spatial - 1); int i,j,k; for(i = 0; i < filters; ++i){ variance[i] = 0; @@ -67,7 +67,7 @@ void normalize_cpu(float *x, float *mean, float *variance, int batch, int filter for(f = 0; f < filters; ++f){ for(i = 0; i < spatial; ++i){ int index = b*filters*spatial + f*spatial + i; - x[index] = (x[index] - mean[f])/(sqrt(variance[f]) + .00001f); + x[index] = (x[index] - mean[f])/(sqrt(variance[f]) + .000001f); } } } diff --git a/src/blas_kernels.cu b/src/blas_kernels.cu index be0e553b..98366f82 100644 --- a/src/blas_kernels.cu +++ b/src/blas_kernels.cu @@ -15,7 +15,7 @@ __global__ void normalize_kernel(int N, float *x, float *mean, float *variance, if (index >= N) return; int f = (index/spatial)%filters; - x[index] = (x[index] - mean[f])/(sqrt(variance[f]) + .00001f); + x[index] = (x[index] - mean[f])/(sqrt(variance[f]) + .000001f); } __global__ void normalize_delta_kernel(int N, float *x, float *mean, float *variance, float *mean_delta, float *variance_delta, int batch, int filters, int spatial, float *delta) @@ -24,7 +24,7 @@ __global__ void normalize_delta_kernel(int N, float *x, float *mean, float *vari if (index >= N) return; int f = (index/spatial)%filters; - delta[index] = delta[index] * 1./(sqrt(variance[f]) + .00001f) + variance_delta[f] * 2. * (x[index] - mean[f]) / (spatial * batch) + mean_delta[f]/(spatial*batch); + delta[index] = delta[index] * 1./(sqrt(variance[f]) + .000001f) + variance_delta[f] * 2. * (x[index] - mean[f]) / (spatial * batch) + mean_delta[f]/(spatial*batch); } extern "C" void normalize_delta_gpu(float *x, float *mean, float *variance, float *mean_delta, float *variance_delta, int batch, int filters, int spatial, float *delta) @@ -46,7 +46,7 @@ __global__ void variance_delta_kernel(float *x, float *delta, float *mean, floa variance_delta[i] += delta[index]*(x[index] - mean[i]); } } - variance_delta[i] *= -.5 * pow(variance[i] + .00001f, (float)(-3./2.)); + variance_delta[i] *= -.5 * pow(variance[i] + .000001f, (float)(-3./2.)); } __global__ void accumulate_kernel(float *x, int n, int groups, float *sum) @@ -83,7 +83,7 @@ __global__ void fast_mean_delta_kernel(float *delta, float *variance, int batch, for(i = 0; i < threads; ++i){ mean_delta[filter] += local[i]; } - mean_delta[filter] *= (-1./sqrt(variance[filter] + .00001f)); + mean_delta[filter] *= (-1./sqrt(variance[filter] + .000001f)); } } @@ -111,7 +111,7 @@ __global__ void fast_variance_delta_kernel(float *x, float *delta, float *mean, for(i = 0; i < threads; ++i){ variance_delta[filter] += local[i]; } - variance_delta[filter] *= -.5 * pow(variance[filter] + .00001f, (float)(-3./2.)); + variance_delta[filter] *= -.5 * pow(variance[filter] + .000001f, (float)(-3./2.)); } } @@ -128,7 +128,7 @@ __global__ void mean_delta_kernel(float *delta, float *variance, int batch, int mean_delta[i] += delta[index]; } } - mean_delta[i] *= (-1./sqrt(variance[i] + .00001f)); + mean_delta[i] *= (-1./sqrt(variance[i] + .000001f)); } extern "C" void mean_delta_gpu(float *delta, float *variance, int batch, int filters, int spatial, float *mean_delta) @@ -167,7 +167,7 @@ __global__ void mean_kernel(float *x, int batch, int filters, int spatial, floa __global__ void variance_kernel(float *x, float *mean, int batch, int filters, int spatial, float *variance) { - float scale = 1./(batch * spatial); + float scale = 1./(batch * spatial - 1); int j,k; int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; if (i >= filters) return; @@ -288,7 +288,7 @@ __global__ void fast_variance_kernel(float *x, float *mean, int batch, int filt for(i = 0; i < threads; ++i){ variance[filter] += local[i]; } - variance[filter] /= spatial * batch; + variance[filter] /= (spatial * batch - 1); } } diff --git a/src/cifar.c b/src/cifar.c index f8878770..de52bb81 100644 --- a/src/cifar.c +++ b/src/cifar.c @@ -33,7 +33,7 @@ void train_cifar(char *cfgfile, char *weightfile) float loss = train_network_sgd(net, train, 1); if(avg_loss == -1) avg_loss = loss; - avg_loss = avg_loss*.9 + loss*.1; + avg_loss = avg_loss*.95 + loss*.05; printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen); if(*net.seen/N > epoch){ epoch = *net.seen/N; @@ -57,6 +57,95 @@ void train_cifar(char *cfgfile, char *weightfile) free_data(train); } +void train_cifar_distill(char *cfgfile, char *weightfile) +{ + data_seed = time(0); + srand(time(0)); + float avg_loss = -1; + char *base = basecfg(cfgfile); + printf("%s\n", base); + network net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); + + char *backup_directory = "/home/pjreddie/backup/"; + int classes = 10; + int N = 50000; + + char **labels = get_labels("data/cifar/labels.txt"); + int epoch = (*net.seen)/N; + + data train = load_all_cifar10(); + matrix soft = csv_to_matrix("results/ensemble.csv"); + + float weight = .9; + scale_matrix(soft, weight); + scale_matrix(train.y, 1. - weight); + matrix_add_matrix(soft, train.y); + + while(get_current_batch(net) < net.max_batches || net.max_batches == 0){ + clock_t time=clock(); + + float loss = train_network_sgd(net, train, 1); + if(avg_loss == -1) avg_loss = loss; + avg_loss = avg_loss*.95 + loss*.05; + printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen); + if(*net.seen/N > epoch){ + epoch = *net.seen/N; + char buff[256]; + sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch); + save_weights(net, buff); + } + if(get_current_batch(net)%100 == 0){ + char buff[256]; + sprintf(buff, "%s/%s.backup",backup_directory,base); + save_weights(net, buff); + } + } + char buff[256]; + sprintf(buff, "%s/%s.weights", backup_directory, base); + save_weights(net, buff); + + free_network(net); + free_ptrs((void**)labels, classes); + free(base); + free_data(train); +} + +void test_cifar_multi(char *filename, char *weightfile) +{ + network net = parse_network_cfg(filename); + if(weightfile){ + load_weights(&net, weightfile); + } + set_batch_network(&net, 1); + srand(time(0)); + + float avg_acc = 0; + data test = load_cifar10_data("data/cifar/cifar-10-batches-bin/test_batch.bin"); + + int i; + for(i = 0; i < test.X.rows; ++i){ + image im = float_to_image(32, 32, 3, test.X.vals[i]); + + float pred[10] = {0}; + + float *p = network_predict(net, im.data); + axpy_cpu(10, 1, p, 1, pred, 1); + flip_image(im); + p = network_predict(net, im.data); + axpy_cpu(10, 1, p, 1, pred, 1); + + int index = max_index(pred, 10); + int class = max_index(test.y.vals[i], 10); + if(index == class) avg_acc += 1; + free_image(im); + printf("%4d: %.2f%%\n", i, 100.*avg_acc/(i+1)); + } +} + void test_cifar(char *filename, char *weightfile) { network net = parse_network_cfg(filename); @@ -79,6 +168,73 @@ void test_cifar(char *filename, char *weightfile) free_data(test); } +void test_cifar_csv(char *filename, char *weightfile) +{ + network net = parse_network_cfg(filename); + if(weightfile){ + load_weights(&net, weightfile); + } + srand(time(0)); + + data test = load_cifar10_data("data/cifar/cifar-10-batches-bin/test_batch.bin"); + + matrix pred = network_predict_data(net, test); + + int i; + for(i = 0; i < test.X.rows; ++i){ + image im = float_to_image(32, 32, 3, test.X.vals[i]); + flip_image(im); + } + matrix pred2 = network_predict_data(net, test); + scale_matrix(pred, .5); + scale_matrix(pred2, .5); + matrix_add_matrix(pred2, pred); + + matrix_to_csv(pred); + fprintf(stderr, "Accuracy: %f\n", matrix_topk_accuracy(test.y, pred, 1)); + free_data(test); +} + +void test_cifar_csvtrain(char *filename, char *weightfile) +{ + network net = parse_network_cfg(filename); + if(weightfile){ + load_weights(&net, weightfile); + } + srand(time(0)); + + data test = load_all_cifar10(); + + matrix pred = network_predict_data(net, test); + + int i; + for(i = 0; i < test.X.rows; ++i){ + image im = float_to_image(32, 32, 3, test.X.vals[i]); + flip_image(im); + } + matrix pred2 = network_predict_data(net, test); + scale_matrix(pred, .5); + scale_matrix(pred2, .5); + matrix_add_matrix(pred2, pred); + + matrix_to_csv(pred); + fprintf(stderr, "Accuracy: %f\n", matrix_topk_accuracy(test.y, pred, 1)); + free_data(test); +} + +void eval_cifar_csv() +{ + data test = load_cifar10_data("data/cifar/cifar-10-batches-bin/test_batch.bin"); + + matrix pred = csv_to_matrix("results/combined.csv"); + fprintf(stderr, "%d %d\n", pred.rows, pred.cols); + + fprintf(stderr, "Accuracy: %f\n", matrix_topk_accuracy(test.y, pred, 1)); + free_data(test); + free_matrix(pred); +} + + void run_cifar(int argc, char **argv) { if(argc < 4){ @@ -89,7 +245,12 @@ void run_cifar(int argc, char **argv) char *cfg = argv[3]; char *weights = (argc > 4) ? argv[4] : 0; if(0==strcmp(argv[2], "train")) train_cifar(cfg, weights); + else if(0==strcmp(argv[2], "distill")) train_cifar_distill(cfg, weights); else if(0==strcmp(argv[2], "test")) test_cifar(cfg, weights); + else if(0==strcmp(argv[2], "multi")) test_cifar_multi(cfg, weights); + else if(0==strcmp(argv[2], "csv")) test_cifar_csv(cfg, weights); + else if(0==strcmp(argv[2], "csvtrain")) test_cifar_csvtrain(cfg, weights); + else if(0==strcmp(argv[2], "eval")) eval_cifar_csv(); } diff --git a/src/classifier.c b/src/classifier.c index fdbe5344..2e974a52 100644 --- a/src/classifier.c +++ b/src/classifier.c @@ -3,6 +3,7 @@ #include "parser.h" #include "option_list.h" #include "blas.h" +#include #ifdef OPENCV #include "opencv2/highgui/highgui_c.h" @@ -239,8 +240,8 @@ void validate_classifier_10(char *datacfg, char *filename, char *weightfile) } int w = net.w; int h = net.h; - image im = load_image_color(paths[i], w, h); int shift = 32; + image im = load_image_color(paths[i], w+shift, h+shift); image images[10]; images[0] = crop_image(im, -shift, -shift, w, h); images[1] = crop_image(im, shift, -shift, w, h); @@ -299,6 +300,7 @@ void validate_classifier_full(char *datacfg, char *filename, char *weightfile) float avg_topk = 0; int *indexes = calloc(topk, sizeof(int)); + int size = net.w; for(i = 0; i < m; ++i){ int class = -1; char *path = paths[i]; @@ -309,13 +311,15 @@ void validate_classifier_full(char *datacfg, char *filename, char *weightfile) } } image im = load_image_color(paths[i], 0, 0); - resize_network(&net, im.w, im.h); + image resized = resize_min(im, size); + resize_network(&net, resized.w, resized.h); //show_image(im, "orig"); //show_image(crop, "cropped"); //cvWaitKey(0); - float *pred = network_predict(net, im.data); + float *pred = network_predict(net, resized.data); free_image(im); + free_image(resized); top_k(pred, classes, topk, indexes); if(indexes[0] == class) avg_acc += 1; @@ -406,7 +410,7 @@ void validate_classifier_multi(char *datacfg, char *filename, char *weightfile) char **labels = get_labels(label_list); list *plist = get_paths(valid_list); - int scales[] = {224, 256, 384, 480, 512}; + int scales[] = {192, 224, 288, 320, 352}; int nscales = sizeof(scales)/sizeof(scales[0]); char **paths = (char **)list_to_array(plist); @@ -429,16 +433,8 @@ void validate_classifier_multi(char *datacfg, char *filename, char *weightfile) float *pred = calloc(classes, sizeof(float)); image im = load_image_color(paths[i], 0, 0); for(j = 0; j < nscales; ++j){ - int w, h; - if(im.w < im.h){ - w = scales[j]; - h = (im.h*w)/im.w; - } else { - h = scales[j]; - w = (im.w * h) / im.h; - } - resize_network(&net, w, h); - image r = resize_image(im, w, h); + image r = resize_min(im, scales[j]); + resize_network(&net, r.w, r.h); float *p = network_predict(net, r.data); axpy_cpu(classes, 1, p, 1, pred, 1); flip_image(r); @@ -577,6 +573,73 @@ void test_classifier(char *datacfg, char *cfgfile, char *weightfile, int target_ } +void demo_classifier(char *datacfg, char *cfgfile, char *weightfile, int cam_index, const char *filename) +{ +#ifdef OPENCV + printf("Classifier Demo\n"); + network net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + set_batch_network(&net, 1); + list *options = read_data_cfg(datacfg); + + srand(2222222); + CvCapture * cap; + + if(filename){ + cap = cvCaptureFromFile(filename); + }else{ + cap = cvCaptureFromCAM(cam_index); + } + + int top = option_find_int(options, "top", 1); + + char *name_list = option_find_str(options, "names", 0); + char **names = get_labels(name_list); + + int *indexes = calloc(top, sizeof(int)); + + if(!cap) error("Couldn't connect to webcam.\n"); + cvNamedWindow("Classifier", CV_WINDOW_NORMAL); + cvResizeWindow("Classifier", 512, 512); + float fps = 0; + int i; + + while(1){ + struct timeval tval_before, tval_after, tval_result; + gettimeofday(&tval_before, NULL); + + image in = get_image_from_stream(cap); + image in_s = resize_image(in, net.w, net.h); + show_image(in, "Classifier"); + + float *predictions = network_predict(net, in_s.data); + top_predictions(net, top, indexes); + + printf("\033[2J"); + printf("\033[1;1H"); + printf("\nFPS:%.0f\n",fps); + + for(i = 0; i < top; ++i){ + int index = indexes[i]; + printf("%.1f%%: %s\n", predictions[index]*100, names[index]); + } + + free_image(in_s); + free_image(in); + + cvWaitKey(10); + + gettimeofday(&tval_after, NULL); + timersub(&tval_after, &tval_before, &tval_result); + float curr = 1000000.f/((long int)tval_result.tv_usec); + fps = .9*fps + .1*curr; + } +#endif +} + + void run_classifier(int argc, char **argv) { if(argc < 4){ @@ -584,6 +647,7 @@ void run_classifier(int argc, char **argv) return; } + int cam_index = find_int_arg(argc, argv, "-c", 0); char *data = argv[3]; char *cfg = argv[4]; char *weights = (argc > 5) ? argv[5] : 0; @@ -592,6 +656,7 @@ void run_classifier(int argc, char **argv) int layer = layer_s ? atoi(layer_s) : -1; if(0==strcmp(argv[2], "predict")) predict_classifier(data, cfg, weights, filename); else if(0==strcmp(argv[2], "train")) train_classifier(data, cfg, weights); + else if(0==strcmp(argv[2], "demo")) demo_classifier(data, cfg, weights, cam_index, filename); else if(0==strcmp(argv[2], "test")) test_classifier(data, cfg, weights, layer); else if(0==strcmp(argv[2], "valid")) validate_classifier(data, cfg, weights); else if(0==strcmp(argv[2], "valid10")) validate_classifier_10(data, cfg, weights); diff --git a/src/coco_demo.c b/src/coco_demo.c index 4ba8eef2..6f4d5014 100644 --- a/src/coco_demo.c +++ b/src/coco_demo.c @@ -71,7 +71,7 @@ void *detect_in_thread_coco(void *ptr) void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam_index, const char *filename) { demo_thresh = thresh; - printf("YOLO demo\n"); + printf("COCO demo\n"); net = parse_network_cfg(cfgfile); if(weightfile){ load_weights(&net, weightfile); @@ -87,8 +87,8 @@ void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam_index, con } if(!cap) error("Couldn't connect to webcam.\n"); - cvNamedWindow("YOLO", CV_WINDOW_NORMAL); - cvResizeWindow("YOLO", 512, 512); + cvNamedWindow("COCO", CV_WINDOW_NORMAL); + cvResizeWindow("COCO", 512, 512); detection_layer l = net.layers[net.n-1]; int j; @@ -127,8 +127,8 @@ void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam_index, con gettimeofday(&tval_before, NULL); if(pthread_create(&fetch_thread, 0, fetch_in_thread_coco, 0)) error("Thread creation failed"); if(pthread_create(&detect_thread, 0, detect_in_thread_coco, 0)) error("Thread creation failed"); - show_image(disp, "YOLO"); - save_image(disp, "YOLO"); + show_image(disp, "COCO"); + //save_image(disp, "COCO"); free_image(disp); cvWaitKey(10); pthread_join(fetch_thread, 0); diff --git a/src/convolutional_kernels.cu b/src/convolutional_kernels.cu index 4f474d68..85b92df6 100644 --- a/src/convolutional_kernels.cu +++ b/src/convolutional_kernels.cu @@ -115,6 +115,46 @@ __global__ void backward_bias_kernel(float *bias_updates, float *delta, int batc } } +__global__ void dot_kernel(float *output, float scale, int batch, int n, int size, float *delta) +{ + int index = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; + int f1 = index / n; + int f2 = index % n; + if (f2 <= f1) return; + + float sum = 0; + float norm1 = 0; + float norm2 = 0; + int b, i; + for(b = 0; b < batch; ++b){ + for(i = 0; i < size; ++i){ + int i1 = b * size * n + f1 * size + i; + int i2 = b * size * n + f2 * size + i; + sum += output[i1] * output[i2]; + norm1 += output[i1] * output[i1]; + norm2 += output[i2] * output[i2]; + } + } + norm1 = sqrt(norm1); + norm2 = sqrt(norm2); + float norm = norm1 * norm2; + sum = sum / norm; + for(b = 0; b < batch; ++b){ + for(i = 0; i < size; ++i){ + int i1 = b * size * n + f1 * size + i; + int i2 = b * size * n + f2 * size + i; + delta[i1] += - scale * sum * output[i2] / norm; + delta[i2] += - scale * sum * output[i1] / norm; + } + } +} + +void dot_error_gpu(layer l) +{ + dot_kernel<<>>(l.output_gpu, l.dot, l.batch, l.n, l.out_w * l.out_h, l.delta_gpu); + check_error(cudaPeekAtLastError()); +} + void backward_bias_gpu(float *bias_updates, float *delta, int batch, int n, int size) { backward_bias_kernel<<>>(bias_updates, delta, batch, n, size); @@ -123,9 +163,9 @@ void backward_bias_gpu(float *bias_updates, float *delta, int batch, int n, int void swap_binary(convolutional_layer *l) { - float *swap = l->filters_gpu; - l->filters_gpu = l->binary_filters_gpu; - l->binary_filters_gpu = swap; + float *swap = l->filters_gpu; + l->filters_gpu = l->binary_filters_gpu; + l->binary_filters_gpu = swap; } void forward_convolutional_layer_gpu(convolutional_layer l, network_state state) @@ -150,8 +190,8 @@ void forward_convolutional_layer_gpu(convolutional_layer l, network_state state) gemm_ongpu(0,0,m,n,k,1.,a,k,b,n,1.,c+i*m*n,n); } - if(l.batch_normalize){ - if(state.train){ + if (l.batch_normalize) { + if (state.train) { fast_mean_gpu(l.output_gpu, l.batch, l.n, l.out_h*l.out_w, l.mean_gpu); fast_variance_gpu(l.output_gpu, l.mean_gpu, l.batch, l.n, l.out_h*l.out_w, l.variance_gpu); @@ -172,6 +212,7 @@ void forward_convolutional_layer_gpu(convolutional_layer l, network_state state) add_bias_gpu(l.output_gpu, l.biases_gpu, l.batch, l.n, n); activate_array_ongpu(l.output_gpu, m*n*l.batch, l.activation); + if(l.dot > 0) dot_error_gpu(l); if(l.binary) swap_binary(&l); } diff --git a/src/darknet.c b/src/darknet.c index 5722729a..0865c61a 100644 --- a/src/darknet.c +++ b/src/darknet.c @@ -24,6 +24,7 @@ extern void run_char_rnn(int argc, char **argv); extern void run_vid_rnn(int argc, char **argv); extern void run_tag(int argc, char **argv); extern void run_cifar(int argc, char **argv); +extern void run_go(int argc, char **argv); void change_rate(char *filename, float scale, float add) { @@ -171,13 +172,13 @@ void denormalize_net(char *cfgfile, char *weightfile, char *outfile) { gpu_index = -1; network net = parse_network_cfg(cfgfile); - if(weightfile){ + if (weightfile) { load_weights(&net, weightfile); } int i; - for(i = 0; i < net.n; ++i){ + for (i = 0; i < net.n; ++i) { layer l = net.layers[i]; - if(l.type == CONVOLUTIONAL){ + if (l.type == CONVOLUTIONAL && l.batch_normalize) { denormalize_convolutional_layer(l); net.layers[i].batch_normalize=0; } @@ -228,6 +229,8 @@ int main(int argc, char **argv) run_yolo(argc, argv); } else if (0 == strcmp(argv[1], "cifar")){ run_cifar(argc, argv); + } else if (0 == strcmp(argv[1], "go")){ + run_go(argc, argv); } else if (0 == strcmp(argv[1], "rnn")){ run_char_rnn(argc, argv); } else if (0 == strcmp(argv[1], "vid")){ diff --git a/src/data.c b/src/data.c index c429a73a..88e3eabc 100644 --- a/src/data.c +++ b/src/data.c @@ -95,6 +95,11 @@ matrix load_image_cropped_paths(char **paths, int n, int min, int max, int size) image crop = random_crop_image(im, min, max, size); int flip = rand_r(&data_seed)%2; if (flip) flip_image(crop); + /* + show_image(im, "orig"); + show_image(crop, "crop"); + cvWaitKey(0); + */ free_image(im); X.vals[i] = crop.data; X.cols = crop.h*crop.w*crop.c; @@ -863,6 +868,17 @@ void get_next_batch(data d, int n, int offset, float *X, float *y) } } +void smooth_data(data d) +{ + int i, j; + int scale = 1. / d.y.cols; + int eps = .1; + for(i = 0; i < d.y.rows; ++i){ + for(j = 0; j < d.y.cols; ++j){ + d.y.vals[i][j] = eps * scale + (1-eps) * d.y.vals[i][j]; + } + } +} data load_all_cifar10() { @@ -894,9 +910,55 @@ data load_all_cifar10() //normalize_data_rows(d); //translate_data_rows(d, -128); scale_data_rows(d, 1./255); + // smooth_data(d); return d; } +data load_go(char *filename) +{ + FILE *fp = fopen(filename, "rb"); + matrix X = make_matrix(128, 361); + matrix y = make_matrix(128, 361); + int row, col; + + if(!fp) file_error(filename); + char *label; + int count = 0; + while((label = fgetl(fp))){ + int i; + if(count == X.rows){ + X = resize_matrix(X, count*2); + y = resize_matrix(y, count*2); + } + sscanf(label, "%d %d", &row, &col); + char *board = fgetl(fp); + + int index = row*19 + col; + y.vals[count][index] = 1; + + for(i = 0; i < 19*19; ++i){ + float val = 0; + if(board[i] == '1') val = 1; + else if(board[i] == '2') val = -1; + X.vals[count][i] = val; + } + ++count; + } + X = resize_matrix(X, count); + y = resize_matrix(y, count); + + data d; + d.shallow = 0; + d.X = X; + d.y = y; + + + fclose(fp); + + return d; +} + + void randomize_data(data d) { int i; @@ -936,6 +998,29 @@ void normalize_data_rows(data d) } } +data get_random_data(data d, int num) +{ + data r = {0}; + r.shallow = 1; + + r.X.rows = num; + r.y.rows = num; + + r.X.cols = d.X.cols; + r.y.cols = d.y.cols; + + r.X.vals = calloc(num, sizeof(float *)); + r.y.vals = calloc(num, sizeof(float *)); + + int i; + for(i = 0; i < num; ++i){ + int index = rand()%d.X.rows; + r.X.vals[i] = d.X.vals[index]; + r.y.vals[i] = d.y.vals[index]; + } + return r; +} + data *split_data(data d, int part, int total) { data *split = calloc(2, sizeof(data)); diff --git a/src/data.h b/src/data.h index a3036a81..f928ade7 100644 --- a/src/data.h +++ b/src/data.h @@ -70,6 +70,7 @@ data load_data(char **paths, int n, int m, char **labels, int k, int w, int h); data load_data_detection(int n, char **paths, int m, int classes, int w, int h, int num_boxes, int background); data load_data_tag(char **paths, int n, int m, int k, int min, int max, int size); data load_data_augment(char **paths, int n, int m, char **labels, int k, int min, int max, int size); +data load_go(char *filename); box_label *read_boxes(char *filename, int *n); data load_cifar10_data(char *filename); @@ -80,6 +81,7 @@ data load_data_writing(char **paths, int n, int m, int w, int h, int out_w, int list *get_paths(char *filename); char **get_labels(char *filename); void get_random_batch(data d, int n, float *X, float *y); +data get_random_data(data d, int num); void get_next_batch(data d, int n, int offset, float *X, float *y); data load_categorical_data_csv(char *filename, int target, int k); void normalize_data_rows(data d); diff --git a/src/go.c b/src/go.c new file mode 100644 index 00000000..a7da283c --- /dev/null +++ b/src/go.c @@ -0,0 +1,249 @@ +#include "network.h" +#include "utils.h" +#include "parser.h" +#include "option_list.h" +#include "blas.h" + +#ifdef OPENCV +#include "opencv2/highgui/highgui_c.h" +#endif + +void train_go(char *cfgfile, char *weightfile) +{ + data_seed = time(0); + srand(time(0)); + float avg_loss = -1; + char *base = basecfg(cfgfile); + printf("%s\n", base); + network net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); + + char *backup_directory = "/home/pjreddie/backup/"; + + data train = load_go("/home/pjreddie/backup/go.train"); + int N = train.X.rows; + int epoch = (*net.seen)/N; + while(get_current_batch(net) < net.max_batches || net.max_batches == 0){ + clock_t time=clock(); + + data batch = get_random_data(train, net.batch); + int i; + for(i = 0; i < batch.X.rows; ++i){ + int flip = rand()%2; + int rotate = rand()%4; + image in = float_to_image(19, 19, 1, batch.X.vals[i]); + image out = float_to_image(19, 19, 1, batch.y.vals[i]); + //show_image_normalized(in, "in"); + //show_image_normalized(out, "out"); + if(flip){ + flip_image(in); + flip_image(out); + } + rotate_image_cw(in, rotate); + rotate_image_cw(out, rotate); + //show_image_normalized(in, "in2"); + //show_image_normalized(out, "out2"); + //cvWaitKey(0); + } + float loss = train_network(net, batch); + free_data(batch); + if(avg_loss == -1) avg_loss = loss; + avg_loss = avg_loss*.95 + loss*.05; + printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen); + if(*net.seen/N > epoch){ + epoch = *net.seen/N; + char buff[256]; + sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch); + save_weights(net, buff); + } + if(get_current_batch(net)%100 == 0){ + char buff[256]; + sprintf(buff, "%s/%s.backup",backup_directory,base); + save_weights(net, buff); + } + } + char buff[256]; + sprintf(buff, "%s/%s.weights", backup_directory, base); + save_weights(net, buff); + + free_network(net); + free(base); + free_data(train); +} + +void propagate_liberty(float *board, int *lib, int *visited, int row, int col, int num, int side) +{ + if (!num) return; + if (row < 0 || row > 18 || col < 0 || col > 18) return; + int index = row*19 + col; + if (board[index] != side) return; + if (visited[index]) return; + visited[index] = 1; + lib[index] += num; + propagate_liberty(board, lib, visited, row+1, col, num, side); + propagate_liberty(board, lib, visited, row-1, col, num, side); + propagate_liberty(board, lib, visited, row, col+1, num, side); + propagate_liberty(board, lib, visited, row, col-1, num, side); +} + +int *calculate_liberties(float *board) +{ + int *lib = calloc(19*19, sizeof(int)); + int visited[361]; + int i, j; + for(j = 0; j < 19; ++j){ + for(i = 0; i < 19; ++i){ + memset(visited, 0, 19*19*sizeof(int)); + int index = j*19 + i; + if(board[index]){ + printf("%d %d\n", j, i); + int side = board[index]; + int num = 0; + if (i > 0 && board[j*19 + i - 1] == 0) ++num; + if (i < 18 && board[j*19 + i + 1] == 0) ++num; + if (j > 0 && board[j*19 + i - 19] == 0) ++num; + if (j < 18 && board[j*19 + i + 19] == 0) ++num; + propagate_liberty(board, lib, visited, j, i, num, side); + } + } + } + return lib; +} + +void update_board(float *board) +{ + int i; + int *l = calculate_liberties(board); + for(i = 0; i < 19*19; ++i){ + if (board[i] && !l[i]) board[i] = 0; + } + free(l); +} + +void print_board(float *board) +{ + int i,j; + printf("\n\n"); + printf(" "); + for(i = 0; i < 19; ++i){ + printf("%c ", 'A' + i + 1*(i > 7)); + } + printf("\n"); + for(j = 0; j < 19; ++j){ + printf("%2d ", 19-j); + for(i = 0; i < 19; ++i){ + int index = j*19 + i; + if(board[index] > 0) printf("\u25C9 "); + else if(board[index] < 0) printf("\u25EF "); + else printf(" "); + } + printf("\n"); + } +} + +void flip_board(float *board) +{ + int i; + for(i = 0; i < 19*19; ++i){ + board[i] = -board[i]; + } +} + +void test_go(char *filename, char *weightfile) +{ + network net = parse_network_cfg(filename); + if(weightfile){ + load_weights(&net, weightfile); + } + srand(time(0)); + set_batch_network(&net, 1); + float *board = calloc(19*19, sizeof(float)); + float *move = calloc(19*19, sizeof(float)); + image bim = float_to_image(19, 19, 1, board); + while(1){ + float *output = network_predict(net, board); + copy_cpu(19*19, output, 1, move, 1); + int i; + for(i = 1; i < 8; ++i){ + rotate_image_cw(bim, i); + if(i >= 4) flip_image(bim); + + float *output = network_predict(net, board); + image oim = float_to_image(19, 19, 1, output); + + if(i >= 4) flip_image(oim); + rotate_image_cw(oim, -i); + + int index = max_index(output, 19*19); + int row = index / 19; + int col = index % 19; + printf("Suggested: %c %d, %.2f%%\n", col + 'A' + 1*(col > 7), 19 - row, output[index]*100); + + axpy_cpu(19*19, 1, output, 1, move, 1); + + if(i >= 4) flip_image(bim); + rotate_image_cw(bim, -i); + } + scal_cpu(19*19, 1./8., move, 1); + for(i = 0; i < 19*19; ++i){ + if(board[i]) move[i] = 0; + } + + int indexes[3]; + int row, col; + top_k(move, 19*19, 3, indexes); + print_board(board); + for(i = 0; i < 3; ++i){ + int index = indexes[i]; + row = index / 19; + col = index % 19; + printf("Suggested: %c %d, %.2f%%\n", col + 'A' + 1*(col > 7), 19 - row, move[index]*100); + } + int index = indexes[0]; + row = index / 19; + col = index % 19; + + printf("\u25C9 Enter move: "); + char c; + char *line = fgetl(stdin); + int num = sscanf(line, "%c %d", &c, &row); + if (c < 'A' || c > 'T'){ + if (c == 'p'){ + board[row*19 + col] = 1; + }else{ + char g; + num = sscanf(line, "%c %c %d", &g, &c, &row); + row = 19 - row; + col = c - 'A'; + if (col > 7) col -= 1; + board[row*19 + col] = 0; + } + } else { + row = 19 - row; + col = c - 'A'; + if (col > 7) col -= 1; + if(num == 2) board[row*19 + col] = 1; + } + update_board(board); + flip_board(board); + } + +} + +void run_go(int argc, char **argv) +{ + if(argc < 4){ + fprintf(stderr, "usage: %s %s [train/test/valid] [cfg] [weights (optional)]\n", argv[0], argv[1]); + return; + } + + char *cfg = argv[3]; + char *weights = (argc > 4) ? argv[4] : 0; + if(0==strcmp(argv[2], "train")) train_go(cfg, weights); + else if(0==strcmp(argv[2], "test")) test_go(cfg, weights); +} + + diff --git a/src/image.c b/src/image.c index e2cf97fc..0d714fe1 100644 --- a/src/image.c +++ b/src/image.c @@ -137,6 +137,42 @@ void draw_detections(image im, int num, float thresh, box *boxes, float **probs, } } +void transpose_image(image im) +{ + assert(im.w == im.h); + int n, m; + int c; + for(c = 0; c < im.c; ++c){ + for(n = 0; n < im.w-1; ++n){ + for(m = n + 1; m < im.w; ++m){ + float swap = im.data[m + im.w*(n + im.h*c)]; + im.data[m + im.w*(n + im.h*c)] = im.data[n + im.w*(m + im.h*c)]; + im.data[n + im.w*(m + im.h*c)] = swap; + } + } + } +} + +void rotate_image_cw(image im, int times) +{ + assert(im.w == im.h); + times = (times + 400) % 4; + int i, x, y, c; + int n = im.w; + for(i = 0; i < times; ++i){ + for(c = 0; c < im.c; ++c){ + for(x = 0; x < n/2; ++x){ + for(y = 0; y < (n-1)/2 + 1; ++y){ + float temp = im.data[y + im.w*(x + im.h*c)]; + im.data[y + im.w*(x + im.h*c)] = im.data[n-1-x + im.w*(y + im.h*c)]; + im.data[n-1-x + im.w*(y + im.h*c)] = im.data[n-1-y + im.w*(n-1-x + im.h*c)]; + im.data[n-1-y + im.w*(n-1-x + im.h*c)] = im.data[x + im.w*(n-1-y + im.h*c)]; + im.data[x + im.w*(n-1-y + im.h*c)] = temp; + } + } + } + } +} void flip_image(image a) { @@ -294,739 +330,747 @@ void show_image_cv(image p, const char *name) } cvShowImage(buff, disp); cvReleaseImage(&disp); -} + } #endif -void show_image(image p, const char *name) -{ + void show_image(image p, const char *name) + { #ifdef OPENCV - show_image_cv(p, name); + show_image_cv(p, name); #else - fprintf(stderr, "Not compiled with OpenCV, saving to %s.png instead\n", name); - save_image(p, name); + fprintf(stderr, "Not compiled with OpenCV, saving to %s.png instead\n", name); + save_image(p, name); #endif -} - -void save_image(image im, const char *name) -{ - char buff[256]; - //sprintf(buff, "%s (%d)", name, windows); - sprintf(buff, "%s.png", name); - unsigned char *data = calloc(im.w*im.h*im.c, sizeof(char)); - int i,k; - for(k = 0; k < im.c; ++k){ - for(i = 0; i < im.w*im.h; ++i){ - data[i*im.c+k] = (unsigned char) (255*im.data[i + k*im.w*im.h]); - } } - int success = stbi_write_png(buff, im.w, im.h, im.c, data, im.w*im.c); - free(data); - if(!success) fprintf(stderr, "Failed to write image %s\n", buff); -} + + void save_image(image im, const char *name) + { + char buff[256]; + //sprintf(buff, "%s (%d)", name, windows); + sprintf(buff, "%s.png", name); + unsigned char *data = calloc(im.w*im.h*im.c, sizeof(char)); + int i,k; + for(k = 0; k < im.c; ++k){ + for(i = 0; i < im.w*im.h; ++i){ + data[i*im.c+k] = (unsigned char) (255*im.data[i + k*im.w*im.h]); + } + } + int success = stbi_write_png(buff, im.w, im.h, im.c, data, im.w*im.c); + free(data); + if(!success) fprintf(stderr, "Failed to write image %s\n", buff); + } #ifdef OPENCV -image get_image_from_stream(CvCapture *cap) -{ - IplImage* src = cvQueryFrame(cap); - image im = ipl_to_image(src); - rgbgr_image(im); - return im; -} + image get_image_from_stream(CvCapture *cap) + { + IplImage* src = cvQueryFrame(cap); + image im = ipl_to_image(src); + rgbgr_image(im); + return im; + } #endif #ifdef OPENCV -void save_image_jpg(image p, char *name) -{ - image copy = copy_image(p); - rgbgr_image(copy); - int x,y,k; + void save_image_jpg(image p, char *name) + { + image copy = copy_image(p); + rgbgr_image(copy); + int x,y,k; - char buff[256]; - sprintf(buff, "%s.jpg", name); + char buff[256]; + sprintf(buff, "%s.jpg", name); - IplImage *disp = cvCreateImage(cvSize(p.w,p.h), IPL_DEPTH_8U, p.c); - int step = disp->widthStep; - for(y = 0; y < p.h; ++y){ - for(x = 0; x < p.w; ++x){ - for(k= 0; k < p.c; ++k){ - disp->imageData[y*step + x*p.c + k] = (unsigned char)(get_pixel(copy,x,y,k)*255); + IplImage *disp = cvCreateImage(cvSize(p.w,p.h), IPL_DEPTH_8U, p.c); + int step = disp->widthStep; + for(y = 0; y < p.h; ++y){ + for(x = 0; x < p.w; ++x){ + for(k= 0; k < p.c; ++k){ + disp->imageData[y*step + x*p.c + k] = (unsigned char)(get_pixel(copy,x,y,k)*255); + } } } + cvSaveImage(buff, disp,0); + cvReleaseImage(&disp); + free_image(copy); } - cvSaveImage(buff, disp,0); - cvReleaseImage(&disp); - free_image(copy); -} #endif -void show_image_layers(image p, char *name) -{ - int i; - char buff[256]; - for(i = 0; i < p.c; ++i){ - sprintf(buff, "%s - Layer %d", name, i); - image layer = get_image_layer(p, i); - show_image(layer, buff); - free_image(layer); - } -} - -void show_image_collapsed(image p, char *name) -{ - image c = collapse_image_layers(p, 1); - show_image(c, name); - free_image(c); -} - -image make_empty_image(int w, int h, int c) -{ - image out; - out.data = 0; - out.h = h; - out.w = w; - out.c = c; - return out; -} - -image make_image(int w, int h, int c) -{ - image out = make_empty_image(w,h,c); - out.data = calloc(h*w*c, sizeof(float)); - return out; -} - -image make_random_image(int w, int h, int c) -{ - image out = make_empty_image(w,h,c); - out.data = calloc(h*w*c, sizeof(float)); - int i; - for(i = 0; i < w*h*c; ++i){ - out.data[i] = (rand_normal() * .25) + .5; - } - return out; -} - -image float_to_image(int w, int h, int c, float *data) -{ - image out = make_empty_image(w,h,c); - out.data = data; - return out; -} - -image rotate_image(image im, float rad) -{ - int x, y, c; - float cx = im.w/2.; - float cy = im.h/2.; - image rot = make_image(im.w, im.h, im.c); - for(c = 0; c < im.c; ++c){ - for(y = 0; y < im.h; ++y){ - for(x = 0; x < im.w; ++x){ - float rx = cos(rad)*(x-cx) - sin(rad)*(y-cy) + cx; - float ry = sin(rad)*(x-cx) + cos(rad)*(y-cy) + cy; - float val = bilinear_interpolate(im, rx, ry, c); - set_pixel(rot, x, y, c, val); - } + void show_image_layers(image p, char *name) + { + int i; + char buff[256]; + for(i = 0; i < p.c; ++i){ + sprintf(buff, "%s - Layer %d", name, i); + image layer = get_image_layer(p, i); + show_image(layer, buff); + free_image(layer); } } - return rot; -} -void translate_image(image m, float s) -{ - int i; - for(i = 0; i < m.h*m.w*m.c; ++i) m.data[i] += s; -} + void show_image_collapsed(image p, char *name) + { + image c = collapse_image_layers(p, 1); + show_image(c, name); + free_image(c); + } -void scale_image(image m, float s) -{ - int i; - for(i = 0; i < m.h*m.w*m.c; ++i) m.data[i] *= s; -} + image make_empty_image(int w, int h, int c) + { + image out; + out.data = 0; + out.h = h; + out.w = w; + out.c = c; + return out; + } -image crop_image(image im, int dx, int dy, int w, int h) -{ - image cropped = make_image(w, h, im.c); - int i, j, k; - for(k = 0; k < im.c; ++k){ - for(j = 0; j < h; ++j){ - for(i = 0; i < w; ++i){ - int r = j + dy; - int c = i + dx; - float val = 0; - if (r >= 0 && r < im.h && c >= 0 && c < im.w) { - val = get_pixel(im, c, r, k); - } - set_pixel(cropped, i, j, k, val); - } + image make_image(int w, int h, int c) + { + image out = make_empty_image(w,h,c); + out.data = calloc(h*w*c, sizeof(float)); + return out; + } + + image make_random_image(int w, int h, int c) + { + image out = make_empty_image(w,h,c); + out.data = calloc(h*w*c, sizeof(float)); + int i; + for(i = 0; i < w*h*c; ++i){ + out.data[i] = (rand_normal() * .25) + .5; } + return out; } - return cropped; -} -image resize_min(image im, int min) -{ - int w = im.w; - int h = im.h; - if(w < h){ - h = (h * min) / w; - w = min; - } else { - w = (w * min) / h; - h = min; + image float_to_image(int w, int h, int c, float *data) + { + image out = make_empty_image(w,h,c); + out.data = data; + return out; } - image resized = resize_image(im, w, h); - return resized; -} -image random_crop_image(image im, int low, int high, int size) -{ - int r = rand_int(low, high); - image resized = resize_min(im, r); - int dx = rand_int(0, resized.w - size); - int dy = rand_int(0, resized.h - size); - image crop = crop_image(resized, dx, dy, size, size); - - /* - show_image(im, "orig"); - show_image(crop, "cropped"); - cvWaitKey(0); - */ - - free_image(resized); - return crop; -} - -float three_way_max(float a, float b, float c) -{ - return (a > b) ? ( (a > c) ? a : c) : ( (b > c) ? b : c) ; -} - -float three_way_min(float a, float b, float c) -{ - return (a < b) ? ( (a < c) ? a : c) : ( (b < c) ? b : c) ; -} - -// http://www.cs.rit.edu/~ncs/color/t_convert.html -void rgb_to_hsv(image im) -{ - assert(im.c == 3); - int i, j; - float r, g, b; - float h, s, v; - for(j = 0; j < im.h; ++j){ - for(i = 0; i < im.w; ++i){ - r = get_pixel(im, i , j, 0); - g = get_pixel(im, i , j, 1); - b = get_pixel(im, i , j, 2); - float max = three_way_max(r,g,b); - float min = three_way_min(r,g,b); - float delta = max - min; - v = max; - if(max == 0){ - s = 0; - h = -1; - }else{ - s = delta/max; - if(r == max){ - h = (g - b) / delta; - } else if (g == max) { - h = 2 + (b - r) / delta; - } else { - h = 4 + (r - g) / delta; - } - if (h < 0) h += 6; - } - set_pixel(im, i, j, 0, h); - set_pixel(im, i, j, 1, s); - set_pixel(im, i, j, 2, v); - } - } -} - -void hsv_to_rgb(image im) -{ - assert(im.c == 3); - int i, j; - float r, g, b; - float h, s, v; - float f, p, q, t; - for(j = 0; j < im.h; ++j){ - for(i = 0; i < im.w; ++i){ - h = get_pixel(im, i , j, 0); - s = get_pixel(im, i , j, 1); - v = get_pixel(im, i , j, 2); - if (s == 0) { - r = g = b = v; - } else { - int index = floor(h); - f = h - index; - p = v*(1-s); - q = v*(1-s*f); - t = v*(1-s*(1-f)); - if(index == 0){ - r = v; g = t; b = p; - } else if(index == 1){ - r = q; g = v; b = p; - } else if(index == 2){ - r = p; g = v; b = t; - } else if(index == 3){ - r = p; g = q; b = v; - } else if(index == 4){ - r = t; g = p; b = v; - } else { - r = v; g = p; b = q; + image rotate_image(image im, float rad) + { + int x, y, c; + float cx = im.w/2.; + float cy = im.h/2.; + image rot = make_image(im.w, im.h, im.c); + for(c = 0; c < im.c; ++c){ + for(y = 0; y < im.h; ++y){ + for(x = 0; x < im.w; ++x){ + float rx = cos(rad)*(x-cx) - sin(rad)*(y-cy) + cx; + float ry = sin(rad)*(x-cx) + cos(rad)*(y-cy) + cy; + float val = bilinear_interpolate(im, rx, ry, c); + set_pixel(rot, x, y, c, val); } } - set_pixel(im, i, j, 0, r); - set_pixel(im, i, j, 1, g); - set_pixel(im, i, j, 2, b); } + return rot; } -} -image grayscale_image(image im) -{ - assert(im.c == 3); - int i, j, k; - image gray = make_image(im.w, im.h, 1); - float scale[] = {0.587, 0.299, 0.114}; - for(k = 0; k < im.c; ++k){ + void translate_image(image m, float s) + { + int i; + for(i = 0; i < m.h*m.w*m.c; ++i) m.data[i] += s; + } + + void scale_image(image m, float s) + { + int i; + for(i = 0; i < m.h*m.w*m.c; ++i) m.data[i] *= s; + } + + image crop_image(image im, int dx, int dy, int w, int h) + { + image cropped = make_image(w, h, im.c); + int i, j, k; + for(k = 0; k < im.c; ++k){ + for(j = 0; j < h; ++j){ + for(i = 0; i < w; ++i){ + int r = j + dy; + int c = i + dx; + float val = 0; + if (r >= 0 && r < im.h && c >= 0 && c < im.w) { + val = get_pixel(im, c, r, k); + } + set_pixel(cropped, i, j, k, val); + } + } + } + return cropped; + } + + image resize_min(image im, int min) + { + int w = im.w; + int h = im.h; + if(w < h){ + h = (h * min) / w; + w = min; + } else { + w = (w * min) / h; + h = min; + } + image resized = resize_image(im, w, h); + return resized; + } + + image random_crop_image(image im, int low, int high, int size) + { + int r = rand_int(low, high); + image resized = resize_min(im, r); + int dx = rand_int(0, resized.w - size); + int dy = rand_int(0, resized.h - size); + image crop = crop_image(resized, dx, dy, size, size); + + /* + show_image(im, "orig"); + show_image(crop, "cropped"); + cvWaitKey(0); + */ + + free_image(resized); + return crop; + } + + float three_way_max(float a, float b, float c) + { + return (a > b) ? ( (a > c) ? a : c) : ( (b > c) ? b : c) ; + } + + float three_way_min(float a, float b, float c) + { + return (a < b) ? ( (a < c) ? a : c) : ( (b < c) ? b : c) ; + } + + // http://www.cs.rit.edu/~ncs/color/t_convert.html + void rgb_to_hsv(image im) + { + assert(im.c == 3); + int i, j; + float r, g, b; + float h, s, v; for(j = 0; j < im.h; ++j){ for(i = 0; i < im.w; ++i){ - gray.data[i+im.w*j] += scale[k]*get_pixel(im, i, j, k); - } - } - } - return gray; -} - -image threshold_image(image im, float thresh) -{ - int i; - image t = make_image(im.w, im.h, im.c); - for(i = 0; i < im.w*im.h*im.c; ++i){ - t.data[i] = im.data[i]>thresh ? 1 : 0; - } - return t; -} - -image blend_image(image fore, image back, float alpha) -{ - assert(fore.w == back.w && fore.h == back.h && fore.c == back.c); - image blend = make_image(fore.w, fore.h, fore.c); - int i, j, k; - for(k = 0; k < fore.c; ++k){ - for(j = 0; j < fore.h; ++j){ - for(i = 0; i < fore.w; ++i){ - float val = alpha * get_pixel(fore, i, j, k) + - (1 - alpha)* get_pixel(back, i, j, k); - set_pixel(blend, i, j, k, val); - } - } - } - return blend; -} - -void scale_image_channel(image im, int c, float v) -{ - int i, j; - for(j = 0; j < im.h; ++j){ - for(i = 0; i < im.w; ++i){ - float pix = get_pixel(im, i, j, c); - pix = pix*v; - set_pixel(im, i, j, c, pix); - } - } -} - -void saturate_image(image im, float sat) -{ - rgb_to_hsv(im); - scale_image_channel(im, 1, sat); - hsv_to_rgb(im); - constrain_image(im); -} - -void exposure_image(image im, float sat) -{ - rgb_to_hsv(im); - scale_image_channel(im, 2, sat); - hsv_to_rgb(im); - constrain_image(im); -} - -void saturate_exposure_image(image im, float sat, float exposure) -{ - rgb_to_hsv(im); - scale_image_channel(im, 1, sat); - scale_image_channel(im, 2, exposure); - hsv_to_rgb(im); - constrain_image(im); -} - -/* - image saturate_image(image im, float sat) - { - image gray = grayscale_image(im); - image blend = blend_image(im, gray, sat); - free_image(gray); - constrain_image(blend); - return blend; - } - - image brightness_image(image im, float b) - { - image bright = make_image(im.w, im.h, im.c); - return bright; - } - */ - -float bilinear_interpolate(image im, float x, float y, int c) -{ - int ix = (int) floorf(x); - int iy = (int) floorf(y); - - float dx = x - ix; - float dy = y - iy; - - float val = (1-dy) * (1-dx) * get_pixel_extend(im, ix, iy, c) + - dy * (1-dx) * get_pixel_extend(im, ix, iy+1, c) + - (1-dy) * dx * get_pixel_extend(im, ix+1, iy, c) + - dy * dx * get_pixel_extend(im, ix+1, iy+1, c); - return val; -} - -image resize_image(image im, int w, int h) -{ - image resized = make_image(w, h, im.c); - image part = make_image(w, im.h, im.c); - int r, c, k; - float w_scale = (float)(im.w - 1) / (w - 1); - float h_scale = (float)(im.h - 1) / (h - 1); - for(k = 0; k < im.c; ++k){ - for(r = 0; r < im.h; ++r){ - for(c = 0; c < w; ++c){ - float val = 0; - if(c == w-1 || im.w == 1){ - val = get_pixel(im, im.w-1, r, k); - } else { - float sx = c*w_scale; - int ix = (int) sx; - float dx = sx - ix; - val = (1 - dx) * get_pixel(im, ix, r, k) + dx * get_pixel(im, ix+1, r, k); + r = get_pixel(im, i , j, 0); + g = get_pixel(im, i , j, 1); + b = get_pixel(im, i , j, 2); + float max = three_way_max(r,g,b); + float min = three_way_min(r,g,b); + float delta = max - min; + v = max; + if(max == 0){ + s = 0; + h = -1; + }else{ + s = delta/max; + if(r == max){ + h = (g - b) / delta; + } else if (g == max) { + h = 2 + (b - r) / delta; + } else { + h = 4 + (r - g) / delta; + } + if (h < 0) h += 6; } - set_pixel(part, c, r, k, val); - } - } - } - for(k = 0; k < im.c; ++k){ - for(r = 0; r < h; ++r){ - float sy = r*h_scale; - int iy = (int) sy; - float dy = sy - iy; - for(c = 0; c < w; ++c){ - float val = (1-dy) * get_pixel(part, c, iy, k); - set_pixel(resized, c, r, k, val); - } - if(r == h-1 || im.h == 1) continue; - for(c = 0; c < w; ++c){ - float val = dy * get_pixel(part, c, iy+1, k); - add_pixel(resized, c, r, k, val); + set_pixel(im, i, j, 0, h); + set_pixel(im, i, j, 1, s); + set_pixel(im, i, j, 2, v); } } } - free_image(part); - return resized; -} + void hsv_to_rgb(image im) + { + assert(im.c == 3); + int i, j; + float r, g, b; + float h, s, v; + float f, p, q, t; + for(j = 0; j < im.h; ++j){ + for(i = 0; i < im.w; ++i){ + h = get_pixel(im, i , j, 0); + s = get_pixel(im, i , j, 1); + v = get_pixel(im, i , j, 2); + if (s == 0) { + r = g = b = v; + } else { + int index = floor(h); + f = h - index; + p = v*(1-s); + q = v*(1-s*f); + t = v*(1-s*(1-f)); + if(index == 0){ + r = v; g = t; b = p; + } else if(index == 1){ + r = q; g = v; b = p; + } else if(index == 2){ + r = p; g = v; b = t; + } else if(index == 3){ + r = p; g = q; b = v; + } else if(index == 4){ + r = t; g = p; b = v; + } else { + r = v; g = p; b = q; + } + } + set_pixel(im, i, j, 0, r); + set_pixel(im, i, j, 1, g); + set_pixel(im, i, j, 2, b); + } + } + } + + image grayscale_image(image im) + { + assert(im.c == 3); + int i, j, k; + image gray = make_image(im.w, im.h, 1); + float scale[] = {0.587, 0.299, 0.114}; + for(k = 0; k < im.c; ++k){ + for(j = 0; j < im.h; ++j){ + for(i = 0; i < im.w; ++i){ + gray.data[i+im.w*j] += scale[k]*get_pixel(im, i, j, k); + } + } + } + return gray; + } + + image threshold_image(image im, float thresh) + { + int i; + image t = make_image(im.w, im.h, im.c); + for(i = 0; i < im.w*im.h*im.c; ++i){ + t.data[i] = im.data[i]>thresh ? 1 : 0; + } + return t; + } + + image blend_image(image fore, image back, float alpha) + { + assert(fore.w == back.w && fore.h == back.h && fore.c == back.c); + image blend = make_image(fore.w, fore.h, fore.c); + int i, j, k; + for(k = 0; k < fore.c; ++k){ + for(j = 0; j < fore.h; ++j){ + for(i = 0; i < fore.w; ++i){ + float val = alpha * get_pixel(fore, i, j, k) + + (1 - alpha)* get_pixel(back, i, j, k); + set_pixel(blend, i, j, k, val); + } + } + } + return blend; + } + + void scale_image_channel(image im, int c, float v) + { + int i, j; + for(j = 0; j < im.h; ++j){ + for(i = 0; i < im.w; ++i){ + float pix = get_pixel(im, i, j, c); + pix = pix*v; + set_pixel(im, i, j, c, pix); + } + } + } + + void saturate_image(image im, float sat) + { + rgb_to_hsv(im); + scale_image_channel(im, 1, sat); + hsv_to_rgb(im); + constrain_image(im); + } + + void exposure_image(image im, float sat) + { + rgb_to_hsv(im); + scale_image_channel(im, 2, sat); + hsv_to_rgb(im); + constrain_image(im); + } + + void saturate_exposure_image(image im, float sat, float exposure) + { + rgb_to_hsv(im); + scale_image_channel(im, 1, sat); + scale_image_channel(im, 2, exposure); + hsv_to_rgb(im); + constrain_image(im); + } + + /* + image saturate_image(image im, float sat) + { + image gray = grayscale_image(im); + image blend = blend_image(im, gray, sat); + free_image(gray); + constrain_image(blend); + return blend; + } + + image brightness_image(image im, float b) + { + image bright = make_image(im.w, im.h, im.c); + return bright; + } + */ + + float bilinear_interpolate(image im, float x, float y, int c) + { + int ix = (int) floorf(x); + int iy = (int) floorf(y); + + float dx = x - ix; + float dy = y - iy; + + float val = (1-dy) * (1-dx) * get_pixel_extend(im, ix, iy, c) + + dy * (1-dx) * get_pixel_extend(im, ix, iy+1, c) + + (1-dy) * dx * get_pixel_extend(im, ix+1, iy, c) + + dy * dx * get_pixel_extend(im, ix+1, iy+1, c); + return val; + } + + image resize_image(image im, int w, int h) + { + image resized = make_image(w, h, im.c); + image part = make_image(w, im.h, im.c); + int r, c, k; + float w_scale = (float)(im.w - 1) / (w - 1); + float h_scale = (float)(im.h - 1) / (h - 1); + for(k = 0; k < im.c; ++k){ + for(r = 0; r < im.h; ++r){ + for(c = 0; c < w; ++c){ + float val = 0; + if(c == w-1 || im.w == 1){ + val = get_pixel(im, im.w-1, r, k); + } else { + float sx = c*w_scale; + int ix = (int) sx; + float dx = sx - ix; + val = (1 - dx) * get_pixel(im, ix, r, k) + dx * get_pixel(im, ix+1, r, k); + } + set_pixel(part, c, r, k, val); + } + } + } + for(k = 0; k < im.c; ++k){ + for(r = 0; r < h; ++r){ + float sy = r*h_scale; + int iy = (int) sy; + float dy = sy - iy; + for(c = 0; c < w; ++c){ + float val = (1-dy) * get_pixel(part, c, iy, k); + set_pixel(resized, c, r, k, val); + } + if(r == h-1 || im.h == 1) continue; + for(c = 0; c < w; ++c){ + float val = dy * get_pixel(part, c, iy+1, k); + add_pixel(resized, c, r, k, val); + } + } + } + + free_image(part); + return resized; + } #include "cuda.h" -void test_resize(char *filename) -{ - image im = load_image(filename, 0,0, 3); - float mag = mag_array(im.data, im.w*im.h*im.c); - printf("L2 Norm: %f\n", mag); - image gray = grayscale_image(im); + void test_resize(char *filename) + { + image im = load_image(filename, 0,0, 3); + float mag = mag_array(im.data, im.w*im.h*im.c); + printf("L2 Norm: %f\n", mag); + image gray = grayscale_image(im); - image sat2 = copy_image(im); - saturate_image(sat2, 2); + image sat2 = copy_image(im); + saturate_image(sat2, 2); - image sat5 = copy_image(im); - saturate_image(sat5, .5); + image sat5 = copy_image(im); + saturate_image(sat5, .5); - image exp2 = copy_image(im); - exposure_image(exp2, 2); + image exp2 = copy_image(im); + exposure_image(exp2, 2); - image exp5 = copy_image(im); - exposure_image(exp5, .5); + image exp5 = copy_image(im); + exposure_image(exp5, .5); #ifdef GPU - image r = resize_image(im, im.w, im.h); - image black = make_image(im.w*2 + 3, im.h*2 + 3, 9); - image black2 = make_image(im.w, im.h, 3); + image r = resize_image(im, im.w, im.h); + image black = make_image(im.w*2 + 3, im.h*2 + 3, 9); + image black2 = make_image(im.w, im.h, 3); - float *r_gpu = cuda_make_array(r.data, r.w*r.h*r.c); - float *black_gpu = cuda_make_array(black.data, black.w*black.h*black.c); - float *black2_gpu = cuda_make_array(black2.data, black2.w*black2.h*black2.c); - shortcut_gpu(3, r.w, r.h, 1, r_gpu, black.w, black.h, 3, black_gpu); - //flip_image(r); - //shortcut_gpu(3, r.w, r.h, 1, r.data, black.w, black.h, 3, black.data); + float *r_gpu = cuda_make_array(r.data, r.w*r.h*r.c); + float *black_gpu = cuda_make_array(black.data, black.w*black.h*black.c); + float *black2_gpu = cuda_make_array(black2.data, black2.w*black2.h*black2.c); + shortcut_gpu(3, r.w, r.h, 1, r_gpu, black.w, black.h, 3, black_gpu); + //flip_image(r); + //shortcut_gpu(3, r.w, r.h, 1, r.data, black.w, black.h, 3, black.data); - shortcut_gpu(3, black.w, black.h, 3, black_gpu, black2.w, black2.h, 1, black2_gpu); - cuda_pull_array(black_gpu, black.data, black.w*black.h*black.c); - cuda_pull_array(black2_gpu, black2.data, black2.w*black2.h*black2.c); - show_image_layers(black, "Black"); - show_image(black2, "Recreate"); + shortcut_gpu(3, black.w, black.h, 3, black_gpu, black2.w, black2.h, 1, black2_gpu); + cuda_pull_array(black_gpu, black.data, black.w*black.h*black.c); + cuda_pull_array(black2_gpu, black2.data, black2.w*black2.h*black2.c); + show_image_layers(black, "Black"); + show_image(black2, "Recreate"); #endif - show_image(im, "Original"); - show_image(gray, "Gray"); - show_image(sat2, "Saturation-2"); - show_image(sat5, "Saturation-.5"); - show_image(exp2, "Exposure-2"); - show_image(exp5, "Exposure-.5"); + show_image(im, "Original"); + show_image(gray, "Gray"); + show_image(sat2, "Saturation-2"); + show_image(sat5, "Saturation-.5"); + show_image(exp2, "Exposure-2"); + show_image(exp5, "Exposure-.5"); #ifdef OPENCV - cvWaitKey(0); + cvWaitKey(0); #endif -} + } #ifdef OPENCV -image ipl_to_image(IplImage* src) -{ - unsigned char *data = (unsigned char *)src->imageData; - int h = src->height; - int w = src->width; - int c = src->nChannels; - int step = src->widthStep; - image out = make_image(w, h, c); - int i, j, k, count=0;; - - for(k= 0; k < c; ++k){ - for(i = 0; i < h; ++i){ - for(j = 0; j < w; ++j){ - out.data[count++] = data[i*step + j*c + k]/255.; - } - } - } - return out; -} - -image load_image_cv(char *filename, int channels) -{ - IplImage* src = 0; - int flag = -1; - if (channels == 0) flag = -1; - else if (channels == 1) flag = 0; - else if (channels == 3) flag = 1; - else { - fprintf(stderr, "OpenCV can't force load with %d channels\n", channels); - } - - if( (src = cvLoadImage(filename, flag)) == 0 ) + image ipl_to_image(IplImage* src) { - fprintf(stderr, "Cannot load image \"%s\"\n", filename); - char buff[256]; - sprintf(buff, "echo %s >> bad.list", filename); - system(buff); - return make_image(10,10,3); - //exit(0); - } - image out = ipl_to_image(src); - cvReleaseImage(&src); - rgbgr_image(out); - return out; -} + unsigned char *data = (unsigned char *)src->imageData; + int h = src->height; + int w = src->width; + int c = src->nChannels; + int step = src->widthStep; + image out = make_image(w, h, c); + int i, j, k, count=0;; -#endif - - -image load_image_stb(char *filename, int channels) -{ - int w, h, c; - unsigned char *data = stbi_load(filename, &w, &h, &c, channels); - if (!data) { - fprintf(stderr, "Cannot load image \"%s\"\nSTB Reason: %s\n", filename, stbi_failure_reason()); - exit(0); - } - if(channels) c = channels; - int i,j,k; - image im = make_image(w, h, c); - for(k = 0; k < c; ++k){ - for(j = 0; j < h; ++j){ - for(i = 0; i < w; ++i){ - int dst_index = i + w*j + w*h*k; - int src_index = k + c*i + c*w*j; - im.data[dst_index] = (float)data[src_index]/255.; + for(k= 0; k < c; ++k){ + for(i = 0; i < h; ++i){ + for(j = 0; j < w; ++j){ + out.data[count++] = data[i*step + j*c + k]/255.; + } } } + return out; + } + + image load_image_cv(char *filename, int channels) + { + IplImage* src = 0; + int flag = -1; + if (channels == 0) flag = -1; + else if (channels == 1) flag = 0; + else if (channels == 3) flag = 1; + else { + fprintf(stderr, "OpenCV can't force load with %d channels\n", channels); + } + + if( (src = cvLoadImage(filename, flag)) == 0 ) + { + fprintf(stderr, "Cannot load image \"%s\"\n", filename); + char buff[256]; + sprintf(buff, "echo %s >> bad.list", filename); + system(buff); + return make_image(10,10,3); + //exit(0); + } + image out = ipl_to_image(src); + cvReleaseImage(&src); + rgbgr_image(out); + return out; } - free(data); - return im; -} -image load_image(char *filename, int w, int h, int c) -{ -#ifdef OPENCV - image out = load_image_cv(filename, c); -#else - image out = load_image_stb(filename, c); #endif - if((h && w) && (h != out.h || w != out.w)){ - image resized = resize_image(out, w, h); - free_image(out); - out = resized; + + image load_image_stb(char *filename, int channels) + { + int w, h, c; + unsigned char *data = stbi_load(filename, &w, &h, &c, channels); + if (!data) { + fprintf(stderr, "Cannot load image \"%s\"\nSTB Reason: %s\n", filename, stbi_failure_reason()); + exit(0); + } + if(channels) c = channels; + int i,j,k; + image im = make_image(w, h, c); + for(k = 0; k < c; ++k){ + for(j = 0; j < h; ++j){ + for(i = 0; i < w; ++i){ + int dst_index = i + w*j + w*h*k; + int src_index = k + c*i + c*w*j; + im.data[dst_index] = (float)data[src_index]/255.; + } + } + } + free(data); + return im; } - return out; -} -image load_image_color(char *filename, int w, int h) -{ - return load_image(filename, w, h, 3); -} + image load_image(char *filename, int w, int h, int c) + { +#ifdef OPENCV + image out = load_image_cv(filename, c); +#else + image out = load_image_stb(filename, c); +#endif -image get_image_layer(image m, int l) -{ - image out = make_image(m.w, m.h, 1); - int i; - for(i = 0; i < m.h*m.w; ++i){ - out.data[i] = m.data[i+l*m.h*m.w]; + if((h && w) && (h != out.h || w != out.w)){ + image resized = resize_image(out, w, h); + free_image(out); + out = resized; + } + return out; } - return out; -} -float get_pixel(image m, int x, int y, int c) -{ - assert(x < m.w && y < m.h && c < m.c); - return m.data[c*m.h*m.w + y*m.w + x]; -} -float get_pixel_extend(image m, int x, int y, int c) -{ - if(x < 0 || x >= m.w || y < 0 || y >= m.h || c < 0 || c >= m.c) return 0; - return get_pixel(m, x, y, c); -} -void set_pixel(image m, int x, int y, int c, float val) -{ - assert(x < m.w && y < m.h && c < m.c); - m.data[c*m.h*m.w + y*m.w + x] = val; -} -void add_pixel(image m, int x, int y, int c, float val) -{ - assert(x < m.w && y < m.h && c < m.c); - m.data[c*m.h*m.w + y*m.w + x] += val; -} + image load_image_color(char *filename, int w, int h) + { + return load_image(filename, w, h, 3); + } -void print_image(image m) -{ - int i, j, k; - for(i =0 ; i < m.c; ++i){ - for(j =0 ; j < m.h; ++j){ - for(k = 0; k < m.w; ++k){ - printf("%.2lf, ", m.data[i*m.h*m.w + j*m.w + k]); - if(k > 30) break; + image get_image_layer(image m, int l) + { + image out = make_image(m.w, m.h, 1); + int i; + for(i = 0; i < m.h*m.w; ++i){ + out.data[i] = m.data[i+l*m.h*m.w]; + } + return out; + } + + float get_pixel(image m, int x, int y, int c) + { + assert(x < m.w && y < m.h && c < m.c); + return m.data[c*m.h*m.w + y*m.w + x]; + } + float get_pixel_extend(image m, int x, int y, int c) + { + if(x < 0 || x >= m.w || y < 0 || y >= m.h || c < 0 || c >= m.c) return 0; + return get_pixel(m, x, y, c); + } + void set_pixel(image m, int x, int y, int c, float val) + { + assert(x < m.w && y < m.h && c < m.c); + m.data[c*m.h*m.w + y*m.w + x] = val; + } + void add_pixel(image m, int x, int y, int c, float val) + { + assert(x < m.w && y < m.h && c < m.c); + m.data[c*m.h*m.w + y*m.w + x] += val; + } + + void print_image(image m) + { + int i, j, k; + for(i =0 ; i < m.c; ++i){ + for(j =0 ; j < m.h; ++j){ + for(k = 0; k < m.w; ++k){ + printf("%.2lf, ", m.data[i*m.h*m.w + j*m.w + k]); + if(k > 30) break; + } + printf("\n"); + if(j > 30) break; } printf("\n"); - if(j > 30) break; } printf("\n"); } - printf("\n"); -} -image collapse_images_vert(image *ims, int n) -{ - int color = 1; - int border = 1; - int h,w,c; - w = ims[0].w; - h = (ims[0].h + border) * n - border; - c = ims[0].c; - if(c != 3 || !color){ - w = (w+border)*c - border; - c = 1; - } - - image filters = make_image(w, h, c); - int i,j; - for(i = 0; i < n; ++i){ - int h_offset = i*(ims[0].h+border); - image copy = copy_image(ims[i]); - //normalize_image(copy); - if(c == 3 && color){ - embed_image(copy, filters, 0, h_offset); + image collapse_images_vert(image *ims, int n) + { + int color = 1; + int border = 1; + int h,w,c; + w = ims[0].w; + h = (ims[0].h + border) * n - border; + c = ims[0].c; + if(c != 3 || !color){ + w = (w+border)*c - border; + c = 1; } - else{ - for(j = 0; j < copy.c; ++j){ - int w_offset = j*(ims[0].w+border); - image layer = get_image_layer(copy, j); - embed_image(layer, filters, w_offset, h_offset); - free_image(layer); + + image filters = make_image(w, h, c); + int i,j; + for(i = 0; i < n; ++i){ + int h_offset = i*(ims[0].h+border); + image copy = copy_image(ims[i]); + //normalize_image(copy); + if(c == 3 && color){ + embed_image(copy, filters, 0, h_offset); } - } - free_image(copy); - } - return filters; -} - -image collapse_images_horz(image *ims, int n) -{ - int color = 1; - int border = 1; - int h,w,c; - int size = ims[0].h; - h = size; - w = (ims[0].w + border) * n - border; - c = ims[0].c; - if(c != 3 || !color){ - h = (h+border)*c - border; - c = 1; - } - - image filters = make_image(w, h, c); - int i,j; - for(i = 0; i < n; ++i){ - int w_offset = i*(size+border); - image copy = copy_image(ims[i]); - //normalize_image(copy); - if(c == 3 && color){ - embed_image(copy, filters, w_offset, 0); - } - else{ - for(j = 0; j < copy.c; ++j){ - int h_offset = j*(size+border); - image layer = get_image_layer(copy, j); - embed_image(layer, filters, w_offset, h_offset); - free_image(layer); + else{ + for(j = 0; j < copy.c; ++j){ + int w_offset = j*(ims[0].w+border); + image layer = get_image_layer(copy, j); + embed_image(layer, filters, w_offset, h_offset); + free_image(layer); + } } + free_image(copy); } - free_image(copy); + return filters; + } + + image collapse_images_horz(image *ims, int n) + { + int color = 1; + int border = 1; + int h,w,c; + int size = ims[0].h; + h = size; + w = (ims[0].w + border) * n - border; + c = ims[0].c; + if(c != 3 || !color){ + h = (h+border)*c - border; + c = 1; + } + + image filters = make_image(w, h, c); + int i,j; + for(i = 0; i < n; ++i){ + int w_offset = i*(size+border); + image copy = copy_image(ims[i]); + //normalize_image(copy); + if(c == 3 && color){ + embed_image(copy, filters, w_offset, 0); + } + else{ + for(j = 0; j < copy.c; ++j){ + int h_offset = j*(size+border); + image layer = get_image_layer(copy, j); + embed_image(layer, filters, w_offset, h_offset); + free_image(layer); + } + } + free_image(copy); + } + return filters; + } + + void show_image_normalized(image im, const char *name) + { + image c = copy_image(im); + normalize_image(c); + show_image(c, name); + free_image(c); } - return filters; -} -void show_images(image *ims, int n, char *window) -{ - image m = collapse_images_vert(ims, n); - /* - int w = 448; - int h = ((float)m.h/m.w) * 448; - if(h > 896){ - h = 896; - w = ((float)m.w/m.h) * 896; - } - image sized = resize_image(m, w, h); - */ - normalize_image(m); - image sized = resize_image(m, m.w, m.h); - save_image(sized, window); - show_image(sized, window); - free_image(sized); - free_image(m); -} + void show_images(image *ims, int n, char *window) + { + image m = collapse_images_vert(ims, n); + /* + int w = 448; + int h = ((float)m.h/m.w) * 448; + if(h > 896){ + h = 896; + w = ((float)m.w/m.h) * 896; + } + image sized = resize_image(m, w, h); + */ + normalize_image(m); + image sized = resize_image(m, m.w, m.h); + save_image(sized, window); + show_image(sized, window); + free_image(sized); + free_image(m); + } -void free_image(image m) -{ - free(m.data); -} + void free_image(image m) + { + free(m.data); + } diff --git a/src/image.h b/src/image.h index b4a7a233..bf6ef999 100644 --- a/src/image.h +++ b/src/image.h @@ -36,6 +36,7 @@ image resize_min(image im, int min); void translate_image(image m, float s); void normalize_image(image p); image rotate_image(image m, float rad); +void rotate_image_cw(image im, int times); void embed_image(image source, image dest, int dx, int dy); void saturate_image(image im, float sat); void exposure_image(image im, float sat); @@ -52,6 +53,7 @@ image collapse_images_horz(image *ims, int n); image collapse_images_vert(image *ims, int n); void show_image(image p, const char *name); +void show_image_normalized(image im, const char *name); void save_image(image p, const char *name); void show_images(image *ims, int n, char *window); void show_image_layers(image p, char *name); diff --git a/src/layer.h b/src/layer.h index 93083708..3efd5977 100644 --- a/src/layer.h +++ b/src/layer.h @@ -56,6 +56,7 @@ struct layer{ int binary; int steps; int hidden; + float dot; float angle; float jitter; float saturation; diff --git a/src/matrix.c b/src/matrix.c index 2e7001ed..ee149799 100644 --- a/src/matrix.c +++ b/src/matrix.c @@ -33,6 +33,35 @@ float matrix_topk_accuracy(matrix truth, matrix guess, int k) return (float)correct/truth.rows; } +void scale_matrix(matrix m, float scale) +{ + int i,j; + for(i = 0; i < m.rows; ++i){ + for(j = 0; j < m.cols; ++j){ + m.vals[i][j] *= scale; + } + } +} + +matrix resize_matrix(matrix m, int size) +{ + int i; + if (m.rows == size) return m; + if (m.rows < size) { + m.vals = realloc(m.vals, size*sizeof(float*)); + for (i = m.rows; i < size; ++i) { + m.vals[i] = calloc(m.cols, sizeof(float)); + } + } else if (m.rows > size) { + for (i = size; i < m.rows; ++i) { + free(m.vals[i]); + } + m.vals = realloc(m.vals, size*sizeof(float*)); + } + m.rows = size; + return m; +} + void matrix_add_matrix(matrix from, matrix to) { assert(from.rows == to.rows && from.cols == to.cols); @@ -114,6 +143,19 @@ matrix csv_to_matrix(char *filename) return m; } +void matrix_to_csv(matrix m) +{ + int i, j; + + for(i = 0; i < m.rows; ++i){ + for(j = 0; j < m.cols; ++j){ + if(j > 0) printf(","); + printf("%.17g", m.vals[i][j]); + } + printf("\n"); + } +} + void print_matrix(matrix m) { int i, j; diff --git a/src/matrix.h b/src/matrix.h index d84431c7..641b5965 100644 --- a/src/matrix.h +++ b/src/matrix.h @@ -10,9 +10,12 @@ void free_matrix(matrix m); void print_matrix(matrix m); matrix csv_to_matrix(char *filename); +void matrix_to_csv(matrix m); matrix hold_out_matrix(matrix *m, int n); float matrix_topk_accuracy(matrix truth, matrix guess, int k); void matrix_add_matrix(matrix from, matrix to); +void scale_matrix(matrix m, float scale); +matrix resize_matrix(matrix m, int size); float *pop_column(matrix *m, int c); diff --git a/src/parser.c b/src/parser.c index 97ce7a16..923e24c4 100644 --- a/src/parser.c +++ b/src/parser.c @@ -160,6 +160,7 @@ convolutional_layer parse_convolutional(list *options, size_params params) convolutional_layer layer = make_convolutional_layer(batch,h,w,c,n,size,stride,pad,activation, batch_normalize, binary); layer.flipped = option_find_int_quiet(options, "flipped", 0); + layer.dot = option_find_float_quiet(options, "dot", 0); char *weights = option_find_str(options, "weights", 0); char *biases = option_find_str(options, "biases", 0); @@ -850,7 +851,15 @@ void load_convolutional_weights(layer l, FILE *fp) fread(l.scales, sizeof(float), l.n, fp); fread(l.rolling_mean, sizeof(float), l.n, fp); fread(l.rolling_variance, sizeof(float), l.n, fp); + /* + int i; + for(i = 0; i < l.n; ++i){ + if(l.rolling_mean[i] > 1 || l.rolling_mean[i] < -1 || l.rolling_variance[i] > 1 || l.rolling_variance[i] < -1) + printf("%f %f\n", l.rolling_mean[i], l.rolling_variance[i]); + } + */ } + fflush(stdout); fread(l.filters, sizeof(float), num, fp); if (l.flipped) { transpose_matrix(l.filters, l.c*l.size*l.size, l.n); diff --git a/src/tag.c b/src/tag.c index 8b63d31c..a00a161a 100644 --- a/src/tag.c +++ b/src/tag.c @@ -99,6 +99,7 @@ void test_tag(char *cfgfile, char *weightfile, char *filename) int indexes[10]; char buff[256]; char *input = buff; + int size = net.w; while(1){ if(filename){ strncpy(input, filename, 256); @@ -109,11 +110,12 @@ void test_tag(char *cfgfile, char *weightfile, char *filename) if(!input) return; strtok(input, "\n"); } - image im = load_image_color(input, net.w, net.h); - //resize_network(&net, im.w, im.h); - printf("%d %d\n", im.w, im.h); + image im = load_image_color(input, 0, 0); + image r = resize_min(im, size); + resize_network(&net, r.w, r.h); + printf("%d %d\n", r.w, r.h); - float *X = im.data; + float *X = r.data; time=clock(); float *predictions = network_predict(net, X); top_predictions(net, 10, indexes); @@ -122,6 +124,7 @@ void test_tag(char *cfgfile, char *weightfile, char *filename) int index = indexes[i]; printf("%.1f%%: %s\n", predictions[index]*100, names[index]); } + free_image(r); free_image(im); if (filename) break; } diff --git a/src/yolo.c b/src/yolo.c index 382cbaa9..02c4fba2 100644 --- a/src/yolo.c +++ b/src/yolo.c @@ -395,13 +395,7 @@ void demo_swag(char *cfgfile, char *weightfile, float thresh){} #endif */ -void demo_yolo(char *cfgfile, char *weightfile, float thresh, int cam_index); -#ifndef GPU -void demo_yolo(char *cfgfile, char *weightfile, float thresh, int cam_index) -{ - fprintf(stderr, "Darknet must be compiled with CUDA for YOLO demo.\n"); -} -#endif +void demo_yolo(char *cfgfile, char *weightfile, float thresh, int cam_index, char *filename); void run_yolo(int argc, char **argv) { @@ -426,5 +420,5 @@ void run_yolo(int argc, char **argv) else if(0==strcmp(argv[2], "train")) train_yolo(cfg, weights); else if(0==strcmp(argv[2], "valid")) validate_yolo(cfg, weights); else if(0==strcmp(argv[2], "recall")) validate_yolo_recall(cfg, weights); - else if(0==strcmp(argv[2], "demo")) demo_yolo(cfg, weights, thresh, cam_index); + else if(0==strcmp(argv[2], "demo")) demo_yolo(cfg, weights, thresh, cam_index, filename); } diff --git a/src/coco_kernels.cu b/src/yolo_demo.c similarity index 50% rename from src/coco_kernels.cu rename to src/yolo_demo.c index 0a5f8407..4e3f8392 100644 --- a/src/coco_kernels.cu +++ b/src/yolo_demo.c @@ -1,8 +1,3 @@ -#include "cuda_runtime.h" -#include "curand.h" -#include "cublas_v2.h" - -extern "C" { #include "network.h" #include "detection_layer.h" #include "cost_layer.h" @@ -11,16 +6,16 @@ extern "C" { #include "box.h" #include "image.h" #include -} #ifdef OPENCV #include "opencv2/highgui/highgui.hpp" #include "opencv2/imgproc/imgproc.hpp" -extern "C" image ipl_to_image(IplImage* src); -extern "C" void convert_coco_detections(float *predictions, int classes, int num, int square, int side, int w, int h, float thresh, float **probs, box *boxes, int only_objectness); +image ipl_to_image(IplImage* src); +void convert_yolo_detections(float *predictions, int classes, int num, int square, int side, int w, int h, float thresh, float **probs, box *boxes, int only_objectness); +void draw_yolo(image im, int num, float thresh, box *boxes, float **probs); -extern "C" char *coco_classes[]; -extern "C" image coco_labels[]; +extern char *voc_names[]; +extern image voc_labels[]; static float **probs; static box *boxes; @@ -30,55 +25,36 @@ static image in_s ; static image det ; static image det_s; static image disp ; -static cv::VideoCapture cap; +static CvCapture * cap; static float fps = 0; static float demo_thresh = 0; -static const int frames = 3; -static float *predictions[frames]; -static int demo_index = 0; -static image images[frames]; -static float *avg; - -void *fetch_in_thread_coco(void *ptr) +void *fetch_in_thread(void *ptr) { - cv::Mat frame_m; - cap >> frame_m; - IplImage frame = frame_m; - in = ipl_to_image(&frame); - rgbgr_image(in); + in = get_image_from_stream(cap); in_s = resize_image(in, net.w, net.h); return 0; } -void *detect_in_thread_coco(void *ptr) +void *detect_in_thread(void *ptr) { float nms = .4; detection_layer l = net.layers[net.n-1]; float *X = det_s.data; - float *prediction = network_predict(net, X); - - memcpy(predictions[demo_index], prediction, l.outputs*sizeof(float)); - mean_arrays(predictions, frames, l.outputs, avg); - + float *predictions = network_predict(net, X); free_image(det_s); - convert_coco_detections(avg, l.classes, l.n, l.sqrt, l.side, 1, 1, demo_thresh, probs, boxes, 0); + convert_yolo_detections(predictions, l.classes, l.n, l.sqrt, l.side, 1, 1, demo_thresh, probs, boxes, 0); if (nms > 0) do_nms(boxes, probs, l.side*l.side*l.n, l.classes, nms); printf("\033[2J"); printf("\033[1;1H"); printf("\nFPS:%.0f\n",fps); printf("Objects:\n\n"); - - images[demo_index] = det; - det = images[(demo_index + frames/2 + 1)%frames]; - demo_index = (demo_index + 1)%frames; - - draw_detections(det, l.side*l.side*l.n, demo_thresh, boxes, probs, coco_classes, coco_labels, 80); + draw_detections(det, l.side*l.side*l.n, demo_thresh, boxes, probs, voc_names, voc_labels, 20); return 0; } -extern "C" void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam_index, const char *filename) +void demo_yolo(char *cfgfile, char *weightfile, float thresh, int cam_index, char *filename) { demo_thresh = thresh; printf("YOLO demo\n"); @@ -91,20 +67,18 @@ extern "C" void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam srand(2222222); if(filename){ - cap.open(filename); + cap = cvCaptureFromFile(filename); }else{ - cap.open(cam_index); + cap = cvCaptureFromCAM(cam_index); } - if(!cap.isOpened()) error("Couldn't connect to webcam.\n"); + if(!cap) error("Couldn't connect to webcam.\n"); + cvNamedWindow("YOLO", CV_WINDOW_NORMAL); + cvResizeWindow("YOLO", 512, 512); detection_layer l = net.layers[net.n-1]; int j; - avg = (float *) calloc(l.outputs, sizeof(float)); - for(j = 0; j < frames; ++j) predictions[j] = (float *) calloc(l.outputs, sizeof(float)); - for(j = 0; j < frames; ++j) images[j] = make_image(1,1,3); - boxes = (box *)calloc(l.side*l.side*l.n, sizeof(box)); probs = (float **)calloc(l.side*l.side*l.n, sizeof(float *)); for(j = 0; j < l.side*l.side*l.n; ++j) probs[j] = (float *)calloc(l.classes, sizeof(float *)); @@ -112,12 +86,12 @@ extern "C" void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam pthread_t fetch_thread; pthread_t detect_thread; - fetch_in_thread_coco(0); + fetch_in_thread(0); det = in; det_s = in_s; - fetch_in_thread_coco(0); - detect_in_thread_coco(0); + fetch_in_thread(0); + detect_in_thread(0); disp = det; det = in; det_s = in_s; @@ -125,8 +99,8 @@ extern "C" void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam while(1){ struct timeval tval_before, tval_after, tval_result; gettimeofday(&tval_before, NULL); - if(pthread_create(&fetch_thread, 0, fetch_in_thread_coco, 0)) error("Thread creation failed"); - if(pthread_create(&detect_thread, 0, detect_in_thread_coco, 0)) error("Thread creation failed"); + if(pthread_create(&fetch_thread, 0, fetch_in_thread, 0)) error("Thread creation failed"); + if(pthread_create(&detect_thread, 0, detect_in_thread, 0)) error("Thread creation failed"); show_image(disp, "YOLO"); free_image(disp); cvWaitKey(1); @@ -144,8 +118,8 @@ extern "C" void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam } } #else -extern "C" void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam_index){ - fprintf(stderr, "YOLO-COCO demo needs OpenCV for webcam images.\n"); +void demo_yolo(char *cfgfile, char *weightfile, float thresh, int cam_index, char *filename){ + fprintf(stderr, "YOLO demo needs OpenCV for webcam images.\n"); } #endif