From 16d06ec0db241261d0d030722e440206ed8aad77 Mon Sep 17 00:00:00 2001 From: Joseph Redmon Date: Mon, 29 Feb 2016 13:54:12 -0800 Subject: [PATCH] stuff --- Makefile | 8 +- src/blas.c | 23 ++- src/blas.h | 8 +- src/blas_kernels.cu | 33 ++++- src/cifar.c | 95 ++++++++++++ src/classifier.c | 169 +++++++++++++++++++-- src/coco.c | 4 +- src/coco_demo.c | 152 +++++++++++++++++++ src/convolutional_kernels.cu | 16 +- src/cost_layer.c | 26 ++-- src/crnn_layer.c | 277 +++++++++++++++++++++++++++++++++++ src/crnn_layer.h | 24 +++ src/darknet.c | 9 ++ src/data.c | 89 ++++++++++- src/data.h | 5 +- src/image.c | 60 ++++++-- src/image.h | 10 +- src/imagenet.c | 4 +- src/layer.h | 3 +- src/network.c | 11 +- src/network.h | 1 + src/network_kernels.cu | 7 + src/nightmare.c | 88 +++++------ src/parser.c | 102 +++++++++---- src/rnn.c | 1 + src/rnn_layer.c | 2 +- src/rnn_vid.c | 210 ++++++++++++++++++++++++++ src/tag.c | 144 ++++++++++++++++++ src/utils.c | 19 ++- src/utils.h | 1 + 30 files changed, 1453 insertions(+), 148 deletions(-) create mode 100644 src/cifar.c create mode 100644 src/coco_demo.c create mode 100644 src/crnn_layer.c create mode 100644 src/crnn_layer.h create mode 100644 src/rnn_vid.c create mode 100644 src/tag.c diff --git a/Makefile b/Makefile index c9b6ecac..528437d9 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -GPU=0 -OPENCV=0 +GPU=1 +OPENCV=1 DEBUG=0 ARCH= --gpu-architecture=compute_20 --gpu-code=compute_20 @@ -34,9 +34,9 @@ CFLAGS+= -DGPU LDFLAGS+= -L/usr/local/cuda/lib64 -lcuda -lcudart -lcublas -lcurand endif -OBJ=gemm.o utils.o cuda.o deconvolutional_layer.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o darknet.o detection_layer.o imagenet.o captcha.o route_layer.o writing.o box.o nightmare.o normalization_layer.o avgpool_layer.o coco.o dice.o yolo.o layer.o compare.o classifier.o local_layer.o swag.o shortcut_layer.o activation_layer.o rnn_layer.o rnn.o +OBJ=gemm.o utils.o cuda.o deconvolutional_layer.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o darknet.o detection_layer.o imagenet.o captcha.o route_layer.o writing.o box.o nightmare.o normalization_layer.o avgpool_layer.o coco.o dice.o yolo.o layer.o compare.o classifier.o local_layer.o swag.o shortcut_layer.o activation_layer.o rnn_layer.o rnn.o rnn_vid.o crnn_layer.o coco_demo.o tag.o cifar.o ifeq ($(GPU), 1) -OBJ+=convolutional_kernels.o deconvolutional_kernels.o activation_kernels.o im2col_kernels.o col2im_kernels.o blas_kernels.o crop_layer_kernels.o dropout_layer_kernels.o maxpool_layer_kernels.o softmax_layer_kernels.o network_kernels.o avgpool_layer_kernels.o yolo_kernels.o coco_kernels.o +OBJ+=convolutional_kernels.o deconvolutional_kernels.o activation_kernels.o im2col_kernels.o col2im_kernels.o blas_kernels.o crop_layer_kernels.o dropout_layer_kernels.o maxpool_layer_kernels.o softmax_layer_kernels.o network_kernels.o avgpool_layer_kernels.o yolo_kernels.o endif OBJS = $(addprefix $(OBJDIR), $(OBJ)) diff --git a/src/blas.c b/src/blas.c index d7948bb1..978f1ed7 100644 --- a/src/blas.c +++ b/src/blas.c @@ -115,13 +115,30 @@ void copy_cpu(int N, float *X, int INCX, float *Y, int INCY) for(i = 0; i < N; ++i) Y[i*INCY] = X[i*INCX]; } -void smooth_l1_cpu(int n, float *pred, float *truth, float *delta) +void smooth_l1_cpu(int n, float *pred, float *truth, float *delta, float *error) { int i; for(i = 0; i < n; ++i){ float diff = truth[i] - pred[i]; - if(fabs(diff) > 1) delta[i] = diff; - else delta[i] = (diff > 0) ? 1 : -1; + float abs_val = fabs(diff); + if(abs_val < 1) { + error[i] = diff * diff; + delta[i] = diff; + } + else { + error[i] = 2*abs_val - 1; + delta[i] = (diff < 0) ? -1 : 1; + } + } +} + +void l2_cpu(int n, float *pred, float *truth, float *delta, float *error) +{ + int i; + for(i = 0; i < n; ++i){ + float diff = truth[i] - pred[i]; + error[i] = diff * diff; + delta[i] = diff; } } diff --git a/src/blas.h b/src/blas.h index f5189e5e..030ef668 100644 --- a/src/blas.h +++ b/src/blas.h @@ -17,7 +17,6 @@ void fill_cpu(int N, float ALPHA, float * X, int INCX); float dot_cpu(int N, float *X, int INCX, float *Y, int INCY); void test_gpu_blas(); void shortcut_cpu(int batch, int w1, int h1, int c1, float *add, int w2, int h2, int c2, float *out); -void smooth_l1_cpu(int n, float *pred, float *truth, float *delta); void mean_cpu(float *x, int batch, int filters, int spatial, float *mean); void variance_cpu(float *x, float *mean, int batch, int filters, int spatial, float *variance); @@ -29,6 +28,9 @@ void mean_delta_cpu(float *delta, float *variance, int batch, int filters, int s void variance_delta_cpu(float *x, float *delta, float *mean, float *variance, int batch, int filters, int spatial, float *variance_delta); void normalize_delta_cpu(float *x, float *mean, float *variance, float *mean_delta, float *variance_delta, int batch, int filters, int spatial, float *delta); +void smooth_l1_cpu(int n, float *pred, float *truth, float *delta, float *error); +void l2_cpu(int n, float *pred, float *truth, float *delta, float *error); + #ifdef GPU void axpy_ongpu(int N, float ALPHA, float * X, int INCX, float * Y, int INCY); void axpy_ongpu_offset(int N, float ALPHA, float * X, int OFFX, int INCX, float * Y, int OFFY, int INCY); @@ -53,9 +55,11 @@ void fast_variance_delta_gpu(float *x, float *delta, float *mean, float *varianc void fast_variance_gpu(float *x, float *mean, int batch, int filters, int spatial, float *variance); void fast_mean_gpu(float *x, int batch, int filters, int spatial, float *mean); void shortcut_gpu(int batch, int w1, int h1, int c1, float *add, int w2, int h2, int c2, float *out); -void smooth_l1_gpu(int n, float *pred, float *truth, float *delta); void scale_bias_gpu(float *output, float *biases, int batch, int n, int size); void backward_scale_gpu(float *x_norm, float *delta, int batch, int n, int size, float *scale_updates); void scale_bias_gpu(float *output, float *biases, int batch, int n, int size); + +void smooth_l1_gpu(int n, float *pred, float *truth, float *delta, float *error); +void l2_gpu(int n, float *pred, float *truth, float *delta, float *error); #endif #endif diff --git a/src/blas_kernels.cu b/src/blas_kernels.cu index 61db29f2..be0e553b 100644 --- a/src/blas_kernels.cu +++ b/src/blas_kernels.cu @@ -410,18 +410,41 @@ extern "C" void shortcut_gpu(int batch, int w1, int h1, int c1, float *add, int check_error(cudaPeekAtLastError()); } -__global__ void smooth_l1_kernel(int n, float *pred, float *truth, float *delta) +__global__ void smooth_l1_kernel(int n, float *pred, float *truth, float *delta, float *error) { int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; if(i < n){ float diff = truth[i] - pred[i]; - if(abs(diff) > 1) delta[i] = diff; - else delta[i] = (diff > 0) ? 1 : -1; + float abs_val = abs(diff); + if(abs_val < 1) { + error[i] = diff * diff; + delta[i] = diff; + } + else { + error[i] = 2*abs_val - 1; + delta[i] = (diff < 0) ? -1 : 1; + } } } -extern "C" void smooth_l1_gpu(int n, float *pred, float *truth, float *delta) +extern "C" void smooth_l1_gpu(int n, float *pred, float *truth, float *delta, float *error) { - smooth_l1_kernel<<>>(n, pred, truth, delta); + smooth_l1_kernel<<>>(n, pred, truth, delta, error); + check_error(cudaPeekAtLastError()); +} + +__global__ void l2_kernel(int n, float *pred, float *truth, float *delta, float *error) +{ + int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x; + if(i < n){ + float diff = truth[i] - pred[i]; + error[i] = diff * diff; //I know this is technically wrong, deal with it. + delta[i] = diff; + } +} + +extern "C" void l2_gpu(int n, float *pred, float *truth, float *delta, float *error) +{ + l2_kernel<<>>(n, pred, truth, delta, error); check_error(cudaPeekAtLastError()); } diff --git a/src/cifar.c b/src/cifar.c new file mode 100644 index 00000000..f8878770 --- /dev/null +++ b/src/cifar.c @@ -0,0 +1,95 @@ +#include "network.h" +#include "utils.h" +#include "parser.h" +#include "option_list.h" +#include "blas.h" + +#ifdef OPENCV +#include "opencv2/highgui/highgui_c.h" +#endif + +void train_cifar(char *cfgfile, char *weightfile) +{ + data_seed = time(0); + srand(time(0)); + float avg_loss = -1; + char *base = basecfg(cfgfile); + printf("%s\n", base); + network net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); + + char *backup_directory = "/home/pjreddie/backup/"; + int classes = 10; + int N = 50000; + + char **labels = get_labels("data/cifar/labels.txt"); + int epoch = (*net.seen)/N; + data train = load_all_cifar10(); + while(get_current_batch(net) < net.max_batches || net.max_batches == 0){ + clock_t time=clock(); + + float loss = train_network_sgd(net, train, 1); + if(avg_loss == -1) avg_loss = loss; + avg_loss = avg_loss*.9 + loss*.1; + printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen); + if(*net.seen/N > epoch){ + epoch = *net.seen/N; + char buff[256]; + sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch); + save_weights(net, buff); + } + if(get_current_batch(net)%100 == 0){ + char buff[256]; + sprintf(buff, "%s/%s.backup",backup_directory,base); + save_weights(net, buff); + } + } + char buff[256]; + sprintf(buff, "%s/%s.weights", backup_directory, base); + save_weights(net, buff); + + free_network(net); + free_ptrs((void**)labels, classes); + free(base); + free_data(train); +} + +void test_cifar(char *filename, char *weightfile) +{ + network net = parse_network_cfg(filename); + if(weightfile){ + load_weights(&net, weightfile); + } + srand(time(0)); + + clock_t time; + float avg_acc = 0; + float avg_top5 = 0; + data test = load_cifar10_data("data/cifar/cifar-10-batches-bin/test_batch.bin"); + + time=clock(); + + float *acc = network_accuracies(net, test, 2); + avg_acc += acc[0]; + avg_top5 += acc[1]; + printf("top1: %f, %lf seconds, %d images\n", avg_acc, sec(clock()-time), test.X.rows); + free_data(test); +} + +void run_cifar(int argc, char **argv) +{ + if(argc < 4){ + fprintf(stderr, "usage: %s %s [train/test/valid] [cfg] [weights (optional)]\n", argv[0], argv[1]); + return; + } + + char *cfg = argv[3]; + char *weights = (argc > 4) ? argv[4] : 0; + if(0==strcmp(argv[2], "train")) train_cifar(cfg, weights); + else if(0==strcmp(argv[2], "test")) test_cifar(cfg, weights); +} + + diff --git a/src/classifier.c b/src/classifier.c index 9924c371..fdbe5344 100644 --- a/src/classifier.c +++ b/src/classifier.c @@ -70,6 +70,11 @@ void train_classifier(char *datacfg, char *cfgfile, char *weightfile) load_args args = {0}; args.w = net.w; args.h = net.h; + + args.min = net.w; + args.max = net.max_crop; + args.size = net.w; + args.paths = paths; args.classes = classes; args.n = imgs; @@ -88,6 +93,16 @@ void train_classifier(char *datacfg, char *cfgfile, char *weightfile) load_thread = load_data_in_thread(args); printf("Loaded: %lf seconds\n", sec(clock()-time)); time=clock(); + +/* + int u; + for(u = 0; u < net.batch; ++u){ + image im = float_to_image(net.w, net.h, 3, train.X.vals[u]); + show_image(im, "loaded"); + cvWaitKey(0); + } + */ + float loss = train_network(net, train); if(avg_loss == -1) avg_loss = loss; avg_loss = avg_loss*.9 + loss*.1; @@ -99,7 +114,7 @@ void train_classifier(char *datacfg, char *cfgfile, char *weightfile) sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch); save_weights(net, buff); } - if(*net.seen%1000 == 0){ + if(*net.seen%100 == 0){ char buff[256]; sprintf(buff, "%s/%s.backup",backup_directory,base); save_weights(net, buff); @@ -152,13 +167,14 @@ void validate_classifier(char *datacfg, char *filename, char *weightfile) load_args args = {0}; args.w = net.w; args.h = net.h; + args.paths = paths; args.classes = classes; args.n = num; args.m = 0; args.labels = labels; args.d = &buffer; - args.type = CLASSIFICATION_DATA; + args.type = OLD_CLASSIFICATION_DATA; pthread_t load_thread = load_data_in_thread(args); for(i = 1; i <= splits; ++i){ @@ -221,19 +237,22 @@ void validate_classifier_10(char *datacfg, char *filename, char *weightfile) break; } } - image im = load_image_color(paths[i], 256, 256); + int w = net.w; + int h = net.h; + image im = load_image_color(paths[i], w, h); + int shift = 32; image images[10]; - images[0] = crop_image(im, -16, -16, 256, 256); - images[1] = crop_image(im, 16, -16, 256, 256); - images[2] = crop_image(im, 0, 0, 256, 256); - images[3] = crop_image(im, -16, 16, 256, 256); - images[4] = crop_image(im, 16, 16, 256, 256); + images[0] = crop_image(im, -shift, -shift, w, h); + images[1] = crop_image(im, shift, -shift, w, h); + images[2] = crop_image(im, 0, 0, w, h); + images[3] = crop_image(im, -shift, shift, w, h); + images[4] = crop_image(im, shift, shift, w, h); flip_image(im); - images[5] = crop_image(im, -16, -16, 256, 256); - images[6] = crop_image(im, 16, -16, 256, 256); - images[7] = crop_image(im, 0, 0, 256, 256); - images[8] = crop_image(im, -16, 16, 256, 256); - images[9] = crop_image(im, 16, 16, 256, 256); + images[5] = crop_image(im, -shift, -shift, w, h); + images[6] = crop_image(im, shift, -shift, w, h); + images[7] = crop_image(im, 0, 0, w, h); + images[8] = crop_image(im, -shift, shift, w, h); + images[9] = crop_image(im, shift, shift, w, h); float *pred = calloc(classes, sizeof(float)); for(j = 0; j < 10; ++j){ float *p = network_predict(net, images[j].data); @@ -252,6 +271,122 @@ void validate_classifier_10(char *datacfg, char *filename, char *weightfile) } } +void validate_classifier_full(char *datacfg, char *filename, char *weightfile) +{ + int i, j; + network net = parse_network_cfg(filename); + set_batch_network(&net, 1); + if(weightfile){ + load_weights(&net, weightfile); + } + srand(time(0)); + + list *options = read_data_cfg(datacfg); + + char *label_list = option_find_str(options, "labels", "data/labels.list"); + char *valid_list = option_find_str(options, "valid", "data/train.list"); + int classes = option_find_int(options, "classes", 2); + int topk = option_find_int(options, "top", 1); + + char **labels = get_labels(label_list); + list *plist = get_paths(valid_list); + + char **paths = (char **)list_to_array(plist); + int m = plist->size; + free_list(plist); + + float avg_acc = 0; + float avg_topk = 0; + int *indexes = calloc(topk, sizeof(int)); + + for(i = 0; i < m; ++i){ + int class = -1; + char *path = paths[i]; + for(j = 0; j < classes; ++j){ + if(strstr(path, labels[j])){ + class = j; + break; + } + } + image im = load_image_color(paths[i], 0, 0); + resize_network(&net, im.w, im.h); + //show_image(im, "orig"); + //show_image(crop, "cropped"); + //cvWaitKey(0); + float *pred = network_predict(net, im.data); + + free_image(im); + top_k(pred, classes, topk, indexes); + + if(indexes[0] == class) avg_acc += 1; + for(j = 0; j < topk; ++j){ + if(indexes[j] == class) avg_topk += 1; + } + + printf("%d: top 1: %f, top %d: %f\n", i, avg_acc/(i+1), topk, avg_topk/(i+1)); + } +} + + +void validate_classifier_single(char *datacfg, char *filename, char *weightfile) +{ + int i, j; + network net = parse_network_cfg(filename); + set_batch_network(&net, 1); + if(weightfile){ + load_weights(&net, weightfile); + } + srand(time(0)); + + list *options = read_data_cfg(datacfg); + + char *label_list = option_find_str(options, "labels", "data/labels.list"); + char *valid_list = option_find_str(options, "valid", "data/train.list"); + int classes = option_find_int(options, "classes", 2); + int topk = option_find_int(options, "top", 1); + + char **labels = get_labels(label_list); + list *plist = get_paths(valid_list); + + char **paths = (char **)list_to_array(plist); + int m = plist->size; + free_list(plist); + + float avg_acc = 0; + float avg_topk = 0; + int *indexes = calloc(topk, sizeof(int)); + + for(i = 0; i < m; ++i){ + int class = -1; + char *path = paths[i]; + for(j = 0; j < classes; ++j){ + if(strstr(path, labels[j])){ + class = j; + break; + } + } + image im = load_image_color(paths[i], 0, 0); + image resized = resize_min(im, net.w); + image crop = crop_image(resized, (resized.w - net.w)/2, (resized.h - net.h)/2, net.w, net.h); + //show_image(im, "orig"); + //show_image(crop, "cropped"); + //cvWaitKey(0); + float *pred = network_predict(net, crop.data); + + free_image(im); + free_image(resized); + free_image(crop); + top_k(pred, classes, topk, indexes); + + if(indexes[0] == class) avg_acc += 1; + for(j = 0; j < topk; ++j){ + if(indexes[j] == class) avg_topk += 1; + } + + printf("%d: top 1: %f, top %d: %f\n", i, avg_acc/(i+1), topk, avg_topk/(i+1)); + } +} + void validate_classifier_multi(char *datacfg, char *filename, char *weightfile) { int i, j; @@ -271,7 +406,7 @@ void validate_classifier_multi(char *datacfg, char *filename, char *weightfile) char **labels = get_labels(label_list); list *plist = get_paths(valid_list); - int scales[] = {224, 256, 384, 480, 640}; + int scales[] = {224, 256, 384, 480, 512}; int nscales = sizeof(scales)/sizeof(scales[0]); char **paths = (char **)list_to_array(plist); @@ -402,7 +537,7 @@ void test_classifier(char *datacfg, char *cfgfile, char *weightfile, int target_ args.m = 0; args.labels = 0; args.d = &buffer; - args.type = CLASSIFICATION_DATA; + args.type = OLD_CLASSIFICATION_DATA; pthread_t load_thread = load_data_in_thread(args); for(curr = net.batch; curr < m; curr += net.batch){ @@ -420,7 +555,7 @@ void test_classifier(char *datacfg, char *cfgfile, char *weightfile, int target_ time=clock(); matrix pred = network_predict_data(net, val); - + int i, j; if (target_layer >= 0){ //layer l = net.layers[target_layer]; @@ -461,6 +596,8 @@ void run_classifier(int argc, char **argv) else if(0==strcmp(argv[2], "valid")) validate_classifier(data, cfg, weights); else if(0==strcmp(argv[2], "valid10")) validate_classifier_10(data, cfg, weights); else if(0==strcmp(argv[2], "validmulti")) validate_classifier_multi(data, cfg, weights); + else if(0==strcmp(argv[2], "validsingle")) validate_classifier_single(data, cfg, weights); + else if(0==strcmp(argv[2], "validfull")) validate_classifier_full(data, cfg, weights); } diff --git a/src/coco.c b/src/coco.c index 41c2d80c..947bef21 100644 --- a/src/coco.c +++ b/src/coco.c @@ -389,10 +389,10 @@ void test_coco(char *cfgfile, char *weightfile, char *filename, float thresh) void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam_index, char *filename); static void demo(char *cfgfile, char *weightfile, float thresh, int cam_index, char* filename) { - #if defined(OPENCV) && defined(GPU) + #if defined(OPENCV) demo_coco(cfgfile, weightfile, thresh, cam_index, filename); #else - fprintf(stderr, "Need to compile with GPU and OpenCV for demo.\n"); + fprintf(stderr, "Need to compile with OpenCV for demo.\n"); #endif } diff --git a/src/coco_demo.c b/src/coco_demo.c new file mode 100644 index 00000000..4ba8eef2 --- /dev/null +++ b/src/coco_demo.c @@ -0,0 +1,152 @@ +#include "network.h" +#include "detection_layer.h" +#include "cost_layer.h" +#include "utils.h" +#include "parser.h" +#include "box.h" +#include "image.h" +#include + +#define FRAMES 1 + +#ifdef OPENCV +#include "opencv2/highgui/highgui.hpp" +#include "opencv2/imgproc/imgproc.hpp" +void convert_coco_detections(float *predictions, int classes, int num, int square, int side, int w, int h, float thresh, float **probs, box *boxes, int only_objectness); + +extern char *coco_classes[]; +extern image coco_labels[]; + +static float **probs; +static box *boxes; +static network net; +static image in ; +static image in_s ; +static image det ; +static image det_s; +static image disp ; +static CvCapture * cap; +static float fps = 0; +static float demo_thresh = 0; + +static float *predictions[FRAMES]; +static int demo_index = 0; +static image images[FRAMES]; +static float *avg; + +void *fetch_in_thread_coco(void *ptr) +{ + in = get_image_from_stream(cap); + in_s = resize_image(in, net.w, net.h); + return 0; +} + +void *detect_in_thread_coco(void *ptr) +{ + float nms = .4; + + detection_layer l = net.layers[net.n-1]; + float *X = det_s.data; + float *prediction = network_predict(net, X); + + memcpy(predictions[demo_index], prediction, l.outputs*sizeof(float)); + mean_arrays(predictions, FRAMES, l.outputs, avg); + + free_image(det_s); + convert_coco_detections(avg, l.classes, l.n, l.sqrt, l.side, 1, 1, demo_thresh, probs, boxes, 0); + if (nms > 0) do_nms(boxes, probs, l.side*l.side*l.n, l.classes, nms); + printf("\033[2J"); + printf("\033[1;1H"); + printf("\nFPS:%.0f\n",fps); + printf("Objects:\n\n"); + + images[demo_index] = det; + det = images[(demo_index + FRAMES/2 + 1)%FRAMES]; + demo_index = (demo_index + 1)%FRAMES; + + draw_detections(det, l.side*l.side*l.n, demo_thresh, boxes, probs, coco_classes, coco_labels, 80); + return 0; +} + +void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam_index, const char *filename) +{ + demo_thresh = thresh; + printf("YOLO demo\n"); + net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + set_batch_network(&net, 1); + + srand(2222222); + + if(filename){ + cap = cvCaptureFromFile(filename); + }else{ + cap = cvCaptureFromCAM(cam_index); + } + + if(!cap) error("Couldn't connect to webcam.\n"); + cvNamedWindow("YOLO", CV_WINDOW_NORMAL); + cvResizeWindow("YOLO", 512, 512); + + detection_layer l = net.layers[net.n-1]; + int j; + + avg = (float *) calloc(l.outputs, sizeof(float)); + for(j = 0; j < FRAMES; ++j) predictions[j] = (float *) calloc(l.outputs, sizeof(float)); + for(j = 0; j < FRAMES; ++j) images[j] = make_image(1,1,3); + + boxes = (box *)calloc(l.side*l.side*l.n, sizeof(box)); + probs = (float **)calloc(l.side*l.side*l.n, sizeof(float *)); + for(j = 0; j < l.side*l.side*l.n; ++j) probs[j] = (float *)calloc(l.classes, sizeof(float *)); + + pthread_t fetch_thread; + pthread_t detect_thread; + + fetch_in_thread_coco(0); + det = in; + det_s = in_s; + + fetch_in_thread_coco(0); + detect_in_thread_coco(0); + disp = det; + det = in; + det_s = in_s; + + for(j = 0; j < FRAMES/2; ++j){ + fetch_in_thread_coco(0); + detect_in_thread_coco(0); + disp = det; + det = in; + det_s = in_s; + } + + while(1){ + struct timeval tval_before, tval_after, tval_result; + gettimeofday(&tval_before, NULL); + if(pthread_create(&fetch_thread, 0, fetch_in_thread_coco, 0)) error("Thread creation failed"); + if(pthread_create(&detect_thread, 0, detect_in_thread_coco, 0)) error("Thread creation failed"); + show_image(disp, "YOLO"); + save_image(disp, "YOLO"); + free_image(disp); + cvWaitKey(10); + pthread_join(fetch_thread, 0); + pthread_join(detect_thread, 0); + + disp = det; + det = in; + det_s = in_s; + + gettimeofday(&tval_after, NULL); + timersub(&tval_after, &tval_before, &tval_result); + float curr = 1000000.f/((long int)tval_result.tv_usec); + fps = .9*fps + .1*curr; + } +} +#else +void demo_coco(char *cfgfile, char *weightfile, float thresh, int cam_index){ + fprintf(stderr, "YOLO-COCO demo needs OpenCV for webcam images.\n"); +} +#endif + diff --git a/src/convolutional_kernels.cu b/src/convolutional_kernels.cu index 4fdc1a1e..4f474d68 100644 --- a/src/convolutional_kernels.cu +++ b/src/convolutional_kernels.cu @@ -121,11 +121,11 @@ void backward_bias_gpu(float *bias_updates, float *delta, int batch, int n, int check_error(cudaPeekAtLastError()); } -void swap_binary(convolutional_layer l) +void swap_binary(convolutional_layer *l) { - float *swap = l.filters_gpu; - l.filters_gpu = l.binary_filters_gpu; - l.binary_filters_gpu = swap; + float *swap = l->filters_gpu; + l->filters_gpu = l->binary_filters_gpu; + l->binary_filters_gpu = swap; } void forward_convolutional_layer_gpu(convolutional_layer l, network_state state) @@ -139,7 +139,7 @@ void forward_convolutional_layer_gpu(convolutional_layer l, network_state state) fill_ongpu(l.outputs*l.batch, 0, l.output_gpu, 1); if(l.binary){ binarize_filters_gpu(l.filters_gpu, l.n, l.c*l.size*l.size, l.binary_filters_gpu); - swap_binary(l); + swap_binary(&l); } for(i = 0; i < l.batch; ++i){ @@ -172,7 +172,7 @@ void forward_convolutional_layer_gpu(convolutional_layer l, network_state state) add_bias_gpu(l.output_gpu, l.biases_gpu, l.batch, l.n, n); activate_array_ongpu(l.output_gpu, m*n*l.batch, l.activation); - if(l.binary) swap_binary(l); + if(l.binary) swap_binary(&l); } void backward_convolutional_layer_gpu(convolutional_layer l, network_state state) @@ -206,7 +206,7 @@ void backward_convolutional_layer_gpu(convolutional_layer l, network_state state gemm_ongpu(0,1,m,n,k,1,a + i*m*k,k,b,k,1,c,n); if(state.delta){ - if(l.binary) swap_binary(l); + if(l.binary) swap_binary(&l); float * a = l.filters_gpu; float * b = l.delta_gpu; float * c = l.col_image_gpu; @@ -214,7 +214,7 @@ void backward_convolutional_layer_gpu(convolutional_layer l, network_state state gemm_ongpu(1,0,n,k,m,1,a,n,b + i*k*m,k,0,c,k); col2im_ongpu(l.col_image_gpu, l.c, l.h, l.w, l.size, l.stride, l.pad, state.delta + i*l.c*l.h*l.w); - if(l.binary) swap_binary(l); + if(l.binary) swap_binary(&l); } } } diff --git a/src/cost_layer.c b/src/cost_layer.c index 39ae8096..fdba7771 100644 --- a/src/cost_layer.c +++ b/src/cost_layer.c @@ -41,9 +41,11 @@ cost_layer make_cost_layer(int batch, int inputs, COST_TYPE cost_type, float sca l.outputs = inputs; l.cost_type = cost_type; l.delta = calloc(inputs*batch, sizeof(float)); - l.output = calloc(1, sizeof(float)); + l.output = calloc(inputs*batch, sizeof(float)); + l.cost = calloc(1, sizeof(float)); #ifdef GPU - l.delta_gpu = cuda_make_array(l.delta, inputs*batch); + l.delta_gpu = cuda_make_array(l.output, inputs*batch); + l.output_gpu = cuda_make_array(l.delta, inputs*batch); #endif return l; } @@ -53,9 +55,12 @@ void resize_cost_layer(cost_layer *l, int inputs) l->inputs = inputs; l->outputs = inputs; l->delta = realloc(l->delta, inputs*l->batch*sizeof(float)); + l->output = realloc(l->output, inputs*l->batch*sizeof(float)); #ifdef GPU cuda_free(l->delta_gpu); + cuda_free(l->output_gpu); l->delta_gpu = cuda_make_array(l->delta, inputs*l->batch); + l->output_gpu = cuda_make_array(l->output, inputs*l->batch); #endif } @@ -69,13 +74,11 @@ void forward_cost_layer(cost_layer l, network_state state) } } if(l.cost_type == SMOOTH){ - smooth_l1_cpu(l.batch*l.inputs, state.input, state.truth, l.delta); + smooth_l1_cpu(l.batch*l.inputs, state.input, state.truth, l.delta, l.output); } else { - copy_cpu(l.batch*l.inputs, state.truth, 1, l.delta, 1); - axpy_cpu(l.batch*l.inputs, -1, state.input, 1, l.delta, 1); + l2_cpu(l.batch*l.inputs, state.input, state.truth, l.delta, l.output); } - *(l.output) = dot_cpu(l.batch*l.inputs, l.delta, 1, l.delta, 1); - //printf("cost: %f\n", *l.output); + l.cost[0] = sum_array(l.output, l.batch*l.inputs); } void backward_cost_layer(const cost_layer l, network_state state) @@ -103,14 +106,13 @@ void forward_cost_layer_gpu(cost_layer l, network_state state) } if(l.cost_type == SMOOTH){ - smooth_l1_gpu(l.batch*l.inputs, state.input, state.truth, l.delta_gpu); + smooth_l1_gpu(l.batch*l.inputs, state.input, state.truth, l.delta_gpu, l.output_gpu); } else { - copy_ongpu(l.batch*l.inputs, state.truth, 1, l.delta_gpu, 1); - axpy_ongpu(l.batch*l.inputs, -1, state.input, 1, l.delta_gpu, 1); + l2_gpu(l.batch*l.inputs, state.input, state.truth, l.delta_gpu, l.output_gpu); } - cuda_pull_array(l.delta_gpu, l.delta, l.batch*l.inputs); - *(l.output) = dot_cpu(l.batch*l.inputs, l.delta, 1, l.delta, 1); + cuda_pull_array(l.output_gpu, l.output, l.batch*l.inputs); + l.cost[0] = sum_array(l.output, l.batch*l.inputs); } void backward_cost_layer_gpu(const cost_layer l, network_state state) diff --git a/src/crnn_layer.c b/src/crnn_layer.c new file mode 100644 index 00000000..ed65665f --- /dev/null +++ b/src/crnn_layer.c @@ -0,0 +1,277 @@ +#include "crnn_layer.h" +#include "convolutional_layer.h" +#include "utils.h" +#include "cuda.h" +#include "blas.h" +#include "gemm.h" + +#include +#include +#include +#include + +static void increment_layer(layer *l, int steps) +{ + int num = l->outputs*l->batch*steps; + l->output += num; + l->delta += num; + l->x += num; + l->x_norm += num; + +#ifdef GPU + l->output_gpu += num; + l->delta_gpu += num; + l->x_gpu += num; + l->x_norm_gpu += num; +#endif +} + +layer make_crnn_layer(int batch, int h, int w, int c, int hidden_filters, int output_filters, int steps, ACTIVATION activation, int batch_normalize) +{ + fprintf(stderr, "CRNN Layer: %d x %d x %d image, %d filters\n", h,w,c,output_filters); + batch = batch / steps; + layer l = {0}; + l.batch = batch; + l.type = CRNN; + l.steps = steps; + l.h = h; + l.w = w; + l.c = c; + l.out_h = h; + l.out_w = w; + l.out_c = output_filters; + l.inputs = h*w*c; + l.hidden = h * w * hidden_filters; + l.outputs = l.out_h * l.out_w * l.out_c; + + l.state = calloc(l.hidden*batch*(steps+1), sizeof(float)); + + l.input_layer = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.input_layer) = make_convolutional_layer(batch*steps, h, w, c, hidden_filters, 3, 1, 1, activation, batch_normalize, 0); + l.input_layer->batch = batch; + + l.self_layer = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.self_layer) = make_convolutional_layer(batch*steps, h, w, hidden_filters, hidden_filters, 3, 1, 1, activation, batch_normalize, 0); + l.self_layer->batch = batch; + + l.output_layer = malloc(sizeof(layer)); + fprintf(stderr, "\t\t"); + *(l.output_layer) = make_convolutional_layer(batch*steps, h, w, hidden_filters, output_filters, 3, 1, 1, activation, batch_normalize, 0); + l.output_layer->batch = batch; + + l.output = l.output_layer->output; + l.delta = l.output_layer->delta; + +#ifdef GPU + l.state_gpu = cuda_make_array(l.state, l.hidden*batch*(steps+1)); + l.output_gpu = l.output_layer->output_gpu; + l.delta_gpu = l.output_layer->delta_gpu; +#endif + + return l; +} + +void update_crnn_layer(layer l, int batch, float learning_rate, float momentum, float decay) +{ + update_convolutional_layer(*(l.input_layer), batch, learning_rate, momentum, decay); + update_convolutional_layer(*(l.self_layer), batch, learning_rate, momentum, decay); + update_convolutional_layer(*(l.output_layer), batch, learning_rate, momentum, decay); +} + +void forward_crnn_layer(layer l, network_state state) +{ + network_state s = {0}; + s.train = state.train; + int i; + layer input_layer = *(l.input_layer); + layer self_layer = *(l.self_layer); + layer output_layer = *(l.output_layer); + + fill_cpu(l.outputs * l.batch * l.steps, 0, output_layer.delta, 1); + fill_cpu(l.hidden * l.batch * l.steps, 0, self_layer.delta, 1); + fill_cpu(l.hidden * l.batch * l.steps, 0, input_layer.delta, 1); + if(state.train) fill_cpu(l.hidden * l.batch, 0, l.state, 1); + + for (i = 0; i < l.steps; ++i) { + s.input = state.input; + forward_convolutional_layer(input_layer, s); + + s.input = l.state; + forward_convolutional_layer(self_layer, s); + + float *old_state = l.state; + if(state.train) l.state += l.hidden*l.batch; + if(l.shortcut){ + copy_cpu(l.hidden * l.batch, old_state, 1, l.state, 1); + }else{ + fill_cpu(l.hidden * l.batch, 0, l.state, 1); + } + axpy_cpu(l.hidden * l.batch, 1, input_layer.output, 1, l.state, 1); + axpy_cpu(l.hidden * l.batch, 1, self_layer.output, 1, l.state, 1); + + s.input = l.state; + forward_convolutional_layer(output_layer, s); + + state.input += l.inputs*l.batch; + increment_layer(&input_layer, 1); + increment_layer(&self_layer, 1); + increment_layer(&output_layer, 1); + } +} + +void backward_crnn_layer(layer l, network_state state) +{ + network_state s = {0}; + s.train = state.train; + int i; + layer input_layer = *(l.input_layer); + layer self_layer = *(l.self_layer); + layer output_layer = *(l.output_layer); + + increment_layer(&input_layer, l.steps-1); + increment_layer(&self_layer, l.steps-1); + increment_layer(&output_layer, l.steps-1); + + l.state += l.hidden*l.batch*l.steps; + for (i = l.steps-1; i >= 0; --i) { + copy_cpu(l.hidden * l.batch, input_layer.output, 1, l.state, 1); + axpy_cpu(l.hidden * l.batch, 1, self_layer.output, 1, l.state, 1); + + s.input = l.state; + s.delta = self_layer.delta; + backward_convolutional_layer(output_layer, s); + + l.state -= l.hidden*l.batch; + /* + if(i > 0){ + copy_cpu(l.hidden * l.batch, input_layer.output - l.hidden*l.batch, 1, l.state, 1); + axpy_cpu(l.hidden * l.batch, 1, self_layer.output - l.hidden*l.batch, 1, l.state, 1); + }else{ + fill_cpu(l.hidden * l.batch, 0, l.state, 1); + } + */ + + s.input = l.state; + s.delta = self_layer.delta - l.hidden*l.batch; + if (i == 0) s.delta = 0; + backward_convolutional_layer(self_layer, s); + + copy_cpu(l.hidden*l.batch, self_layer.delta, 1, input_layer.delta, 1); + if (i > 0 && l.shortcut) axpy_cpu(l.hidden*l.batch, 1, self_layer.delta, 1, self_layer.delta - l.hidden*l.batch, 1); + s.input = state.input + i*l.inputs*l.batch; + if(state.delta) s.delta = state.delta + i*l.inputs*l.batch; + else s.delta = 0; + backward_convolutional_layer(input_layer, s); + + increment_layer(&input_layer, -1); + increment_layer(&self_layer, -1); + increment_layer(&output_layer, -1); + } +} + +#ifdef GPU + +void pull_crnn_layer(layer l) +{ + pull_convolutional_layer(*(l.input_layer)); + pull_convolutional_layer(*(l.self_layer)); + pull_convolutional_layer(*(l.output_layer)); +} + +void push_crnn_layer(layer l) +{ + push_convolutional_layer(*(l.input_layer)); + push_convolutional_layer(*(l.self_layer)); + push_convolutional_layer(*(l.output_layer)); +} + +void update_crnn_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay) +{ + update_convolutional_layer_gpu(*(l.input_layer), batch, learning_rate, momentum, decay); + update_convolutional_layer_gpu(*(l.self_layer), batch, learning_rate, momentum, decay); + update_convolutional_layer_gpu(*(l.output_layer), batch, learning_rate, momentum, decay); +} + +void forward_crnn_layer_gpu(layer l, network_state state) +{ + network_state s = {0}; + s.train = state.train; + int i; + layer input_layer = *(l.input_layer); + layer self_layer = *(l.self_layer); + layer output_layer = *(l.output_layer); + + fill_ongpu(l.outputs * l.batch * l.steps, 0, output_layer.delta_gpu, 1); + fill_ongpu(l.hidden * l.batch * l.steps, 0, self_layer.delta_gpu, 1); + fill_ongpu(l.hidden * l.batch * l.steps, 0, input_layer.delta_gpu, 1); + if(state.train) fill_ongpu(l.hidden * l.batch, 0, l.state_gpu, 1); + + for (i = 0; i < l.steps; ++i) { + s.input = state.input; + forward_convolutional_layer_gpu(input_layer, s); + + s.input = l.state_gpu; + forward_convolutional_layer_gpu(self_layer, s); + + float *old_state = l.state_gpu; + if(state.train) l.state_gpu += l.hidden*l.batch; + if(l.shortcut){ + copy_ongpu(l.hidden * l.batch, old_state, 1, l.state_gpu, 1); + }else{ + fill_ongpu(l.hidden * l.batch, 0, l.state_gpu, 1); + } + axpy_ongpu(l.hidden * l.batch, 1, input_layer.output_gpu, 1, l.state_gpu, 1); + axpy_ongpu(l.hidden * l.batch, 1, self_layer.output_gpu, 1, l.state_gpu, 1); + + s.input = l.state_gpu; + forward_convolutional_layer_gpu(output_layer, s); + + state.input += l.inputs*l.batch; + increment_layer(&input_layer, 1); + increment_layer(&self_layer, 1); + increment_layer(&output_layer, 1); + } +} + +void backward_crnn_layer_gpu(layer l, network_state state) +{ + network_state s = {0}; + s.train = state.train; + int i; + layer input_layer = *(l.input_layer); + layer self_layer = *(l.self_layer); + layer output_layer = *(l.output_layer); + increment_layer(&input_layer, l.steps - 1); + increment_layer(&self_layer, l.steps - 1); + increment_layer(&output_layer, l.steps - 1); + l.state_gpu += l.hidden*l.batch*l.steps; + for (i = l.steps-1; i >= 0; --i) { + copy_ongpu(l.hidden * l.batch, input_layer.output_gpu, 1, l.state_gpu, 1); + axpy_ongpu(l.hidden * l.batch, 1, self_layer.output_gpu, 1, l.state_gpu, 1); + + s.input = l.state_gpu; + s.delta = self_layer.delta_gpu; + backward_convolutional_layer_gpu(output_layer, s); + + l.state_gpu -= l.hidden*l.batch; + + s.input = l.state_gpu; + s.delta = self_layer.delta_gpu - l.hidden*l.batch; + if (i == 0) s.delta = 0; + backward_convolutional_layer_gpu(self_layer, s); + + copy_ongpu(l.hidden*l.batch, self_layer.delta_gpu, 1, input_layer.delta_gpu, 1); + if (i > 0 && l.shortcut) axpy_ongpu(l.hidden*l.batch, 1, self_layer.delta_gpu, 1, self_layer.delta_gpu - l.hidden*l.batch, 1); + s.input = state.input + i*l.inputs*l.batch; + if(state.delta) s.delta = state.delta + i*l.inputs*l.batch; + else s.delta = 0; + backward_convolutional_layer_gpu(input_layer, s); + + increment_layer(&input_layer, -1); + increment_layer(&self_layer, -1); + increment_layer(&output_layer, -1); + } +} +#endif diff --git a/src/crnn_layer.h b/src/crnn_layer.h new file mode 100644 index 00000000..0da942ee --- /dev/null +++ b/src/crnn_layer.h @@ -0,0 +1,24 @@ + +#ifndef CRNN_LAYER_H +#define CRNN_LAYER_H + +#include "activations.h" +#include "layer.h" +#include "network.h" + +layer make_crnn_layer(int batch, int h, int w, int c, int hidden_filters, int output_filters, int steps, ACTIVATION activation, int batch_normalize); + +void forward_crnn_layer(layer l, network_state state); +void backward_crnn_layer(layer l, network_state state); +void update_crnn_layer(layer l, int batch, float learning_rate, float momentum, float decay); + +#ifdef GPU +void forward_crnn_layer_gpu(layer l, network_state state); +void backward_crnn_layer_gpu(layer l, network_state state); +void update_crnn_layer_gpu(layer l, int batch, float learning_rate, float momentum, float decay); +void push_crnn_layer(layer l); +void pull_crnn_layer(layer l); +#endif + +#endif + diff --git a/src/darknet.c b/src/darknet.c index c4006cee..5722729a 100644 --- a/src/darknet.c +++ b/src/darknet.c @@ -21,6 +21,9 @@ extern void run_dice(int argc, char **argv); extern void run_compare(int argc, char **argv); extern void run_classifier(int argc, char **argv); extern void run_char_rnn(int argc, char **argv); +extern void run_vid_rnn(int argc, char **argv); +extern void run_tag(int argc, char **argv); +extern void run_cifar(int argc, char **argv); void change_rate(char *filename, float scale, float add) { @@ -223,12 +226,18 @@ int main(int argc, char **argv) average(argc, argv); } else if (0 == strcmp(argv[1], "yolo")){ run_yolo(argc, argv); + } else if (0 == strcmp(argv[1], "cifar")){ + run_cifar(argc, argv); } else if (0 == strcmp(argv[1], "rnn")){ run_char_rnn(argc, argv); + } else if (0 == strcmp(argv[1], "vid")){ + run_vid_rnn(argc, argv); } else if (0 == strcmp(argv[1], "coco")){ run_coco(argc, argv); } else if (0 == strcmp(argv[1], "classifier")){ run_classifier(argc, argv); + } else if (0 == strcmp(argv[1], "tag")){ + run_tag(argc, argv); } else if (0 == strcmp(argv[1], "compare")){ run_compare(argc, argv); } else if (0 == strcmp(argv[1], "dice")){ diff --git a/src/data.c b/src/data.c index 88c89917..c429a73a 100644 --- a/src/data.c +++ b/src/data.c @@ -82,6 +82,27 @@ matrix load_image_paths(char **paths, int n, int w, int h) return X; } +matrix load_image_cropped_paths(char **paths, int n, int min, int max, int size) +{ + int i; + matrix X; + X.rows = n; + X.vals = calloc(X.rows, sizeof(float*)); + X.cols = 0; + + for(i = 0; i < n; ++i){ + image im = load_image_color(paths[i], 0, 0); + image crop = random_crop_image(im, min, max, size); + int flip = rand_r(&data_seed)%2; + if (flip) flip_image(crop); + free_image(im); + X.vals[i] = crop.data; + X.cols = crop.h*crop.w*crop.c; + } + return X; +} + + box_label *read_boxes(char *filename, int *n) { box_label *boxes = calloc(1, sizeof(box_label)); @@ -386,6 +407,33 @@ matrix load_labels_paths(char **paths, int n, char **labels, int k) return y; } +matrix load_tags_paths(char **paths, int n, int k) +{ + matrix y = make_matrix(n, k); + int i; + int count = 0; + for(i = 0; i < n; ++i){ + char *label = find_replace(paths[i], "imgs", "labels"); + label = find_replace(label, "_iconl.jpeg", ".txt"); + FILE *file = fopen(label, "r"); + if(!file){ + label = find_replace(label, "labels", "labels2"); + file = fopen(label, "r"); + if(!file) continue; + } + ++count; + int tag; + while(fscanf(file, "%d", &tag) == 1){ + if(tag < k){ + y.vals[i][tag] = 1; + } + } + fclose(file); + } + printf("%d/%d\n", count, n); + return y; +} + char **get_labels(char *filename) { list *plist = get_paths(filename); @@ -641,8 +689,10 @@ void *load_thread(void *ptr) //printf("Loading data: %d\n", rand_r(&data_seed)); load_args a = *(struct load_args*)ptr; - if (a.type == CLASSIFICATION_DATA){ + if (a.type == OLD_CLASSIFICATION_DATA){ *a.d = load_data(a.paths, a.n, a.m, a.labels, a.classes, a.w, a.h); + } else if (a.type == CLASSIFICATION_DATA){ + *a.d = load_data_augment(a.paths, a.n, a.m, a.labels, a.classes, a.min, a.max, a.size); } else if (a.type == DETECTION_DATA){ *a.d = load_data_detection(a.n, a.paths, a.m, a.classes, a.w, a.h, a.num_boxes, a.background); } else if (a.type == WRITING_DATA){ @@ -656,6 +706,9 @@ void *load_thread(void *ptr) } else if (a.type == IMAGE_DATA){ *(a.im) = load_image_color(a.path, 0, 0); *(a.resized) = resize_image(*(a.im), a.w, a.h); + } else if (a.type == TAG_DATA){ + *a.d = load_data_tag(a.paths, a.n, a.m, a.classes, a.min, a.max, a.size); + //*a.d = load_data(a.paths, a.n, a.m, a.labels, a.classes, a.w, a.h); } free(ptr); return 0; @@ -696,6 +749,30 @@ data load_data(char **paths, int n, int m, char **labels, int k, int w, int h) return d; } +data load_data_augment(char **paths, int n, int m, char **labels, int k, int min, int max, int size) +{ + if(m) paths = get_random_paths(paths, n, m); + data d; + d.shallow = 0; + d.X = load_image_cropped_paths(paths, n, min, max, size); + d.y = load_labels_paths(paths, n, labels, k); + if(m) free(paths); + return d; +} + +data load_data_tag(char **paths, int n, int m, int k, int min, int max, int size) +{ + if(m) paths = get_random_paths(paths, n, m); + data d = {0}; + d.w = size; + d.h = size; + d.shallow = 0; + d.X = load_image_cropped_paths(paths, n, min, max, size); + d.y = load_tags_paths(paths, n, k); + if(m) free(paths); + return d; +} + matrix concat_matrix(matrix m1, matrix m2) { int i, count = 0; @@ -759,8 +836,8 @@ data load_cifar10_data(char *filename) X.vals[i][j] = (double)bytes[j+1]; } } - translate_data_rows(d, -128); - scale_data_rows(d, 1./128); + //translate_data_rows(d, -128); + scale_data_rows(d, 1./255); //normalize_data_rows(d); fclose(fp); return d; @@ -800,7 +877,7 @@ data load_all_cifar10() for(b = 0; b < 5; ++b){ char buff[256]; - sprintf(buff, "data/cifar10/data_batch_%d.bin", b+1); + sprintf(buff, "data/cifar/cifar-10-batches-bin/data_batch_%d.bin", b+1); FILE *fp = fopen(buff, "rb"); if(!fp) file_error(buff); for(i = 0; i < 10000; ++i){ @@ -815,8 +892,8 @@ data load_all_cifar10() fclose(fp); } //normalize_data_rows(d); - translate_data_rows(d, -128); - scale_data_rows(d, 1./128); + //translate_data_rows(d, -128); + scale_data_rows(d, 1./255); return d; } diff --git a/src/data.h b/src/data.h index 0ebdfc3f..a3036a81 100644 --- a/src/data.h +++ b/src/data.h @@ -27,7 +27,7 @@ typedef struct{ } data; typedef enum { - CLASSIFICATION_DATA, DETECTION_DATA, CAPTCHA_DATA, REGION_DATA, IMAGE_DATA, COMPARE_DATA, WRITING_DATA, SWAG_DATA + CLASSIFICATION_DATA, DETECTION_DATA, CAPTCHA_DATA, REGION_DATA, IMAGE_DATA, COMPARE_DATA, WRITING_DATA, SWAG_DATA, TAG_DATA, OLD_CLASSIFICATION_DATA } data_type; typedef struct load_args{ @@ -43,6 +43,7 @@ typedef struct load_args{ int nh; int nw; int num_boxes; + int min, max, size; int classes; int background; float jitter; @@ -67,6 +68,8 @@ data load_data_captcha(char **paths, int n, int m, int k, int w, int h); data load_data_captcha_encode(char **paths, int n, int m, int w, int h); data load_data(char **paths, int n, int m, char **labels, int k, int w, int h); data load_data_detection(int n, char **paths, int m, int classes, int w, int h, int num_boxes, int background); +data load_data_tag(char **paths, int n, int m, int k, int min, int max, int size); +data load_data_augment(char **paths, int n, int m, char **labels, int k, int min, int max, int size); box_label *read_boxes(char *filename, int *n); data load_cifar10_data(char *filename); diff --git a/src/image.c b/src/image.c index 60ccfb8c..e2cf97fc 100644 --- a/src/image.c +++ b/src/image.c @@ -4,11 +4,6 @@ #include #include -#ifdef OPENCV -#include "opencv2/highgui/highgui_c.h" -#include "opencv2/imgproc/imgproc_c.h" -#endif - #define STB_IMAGE_IMPLEMENTATION #include "stb_image.h" #define STB_IMAGE_WRITE_IMPLEMENTATION @@ -329,6 +324,16 @@ void save_image(image im, const char *name) if(!success) fprintf(stderr, "Failed to write image %s\n", buff); } +#ifdef OPENCV +image get_image_from_stream(CvCapture *cap) +{ + IplImage* src = cvQueryFrame(cap); + image im = ipl_to_image(src); + rgbgr_image(im); + return im; +} +#endif + #ifdef OPENCV void save_image_jpg(image p, char *name) { @@ -459,6 +464,39 @@ image crop_image(image im, int dx, int dy, int w, int h) return cropped; } +image resize_min(image im, int min) +{ + int w = im.w; + int h = im.h; + if(w < h){ + h = (h * min) / w; + w = min; + } else { + w = (w * min) / h; + h = min; + } + image resized = resize_image(im, w, h); + return resized; +} + +image random_crop_image(image im, int low, int high, int size) +{ + int r = rand_int(low, high); + image resized = resize_min(im, r); + int dx = rand_int(0, resized.w - size); + int dy = rand_int(0, resized.h - size); + image crop = crop_image(resized, dx, dy, size, size); + + /* + show_image(im, "orig"); + show_image(crop, "cropped"); + cvWaitKey(0); + */ + + free_image(resized); + return crop; +} + float three_way_max(float a, float b, float c) { return (a > b) ? ( (a > c) ? a : c) : ( (b > c) ? b : c) ; @@ -724,7 +762,7 @@ void test_resize(char *filename) image exp5 = copy_image(im); exposure_image(exp5, .5); - #ifdef GPU +#ifdef GPU image r = resize_image(im, im.w, im.h); image black = make_image(im.w*2 + 3, im.h*2 + 3, 9); image black2 = make_image(im.w, im.h, 3); @@ -741,7 +779,7 @@ void test_resize(char *filename) cuda_pull_array(black2_gpu, black2.data, black2.w*black2.h*black2.c); show_image_layers(black, "Black"); show_image(black2, "Recreate"); - #endif +#endif show_image(im, "Original"); show_image(gray, "Gray"); @@ -788,8 +826,12 @@ image load_image_cv(char *filename, int channels) if( (src = cvLoadImage(filename, flag)) == 0 ) { - printf("Cannot load image \"%s\"\n", filename); - exit(0); + fprintf(stderr, "Cannot load image \"%s\"\n", filename); + char buff[256]; + sprintf(buff, "echo %s >> bad.list", filename); + system(buff); + return make_image(10,10,3); + //exit(0); } image out = ipl_to_image(src); cvReleaseImage(&src); diff --git a/src/image.h b/src/image.h index 4846bc19..b4a7a233 100644 --- a/src/image.h +++ b/src/image.h @@ -8,6 +8,11 @@ #include #include "box.h" +#ifdef OPENCV +#include "opencv2/highgui/highgui_c.h" +#include "opencv2/imgproc/imgproc_c.h" +#endif + typedef struct { int h; int w; @@ -25,8 +30,9 @@ void draw_detections(image im, int num, float thresh, box *boxes, float **probs, image image_distance(image a, image b); void scale_image(image m, float s); image crop_image(image im, int dx, int dy, int w, int h); +image random_crop_image(image im, int low, int high, int size); image resize_image(image im, int w, int h); -image resize_image2(image im, int w, int h); +image resize_min(image im, int min); void translate_image(image m, float s); void normalize_image(image p); image rotate_image(image m, float rad); @@ -53,6 +59,8 @@ void show_image_collapsed(image p, char *name); #ifdef OPENCV void save_image_jpg(image p, char *name); +image get_image_from_stream(CvCapture *cap); +image ipl_to_image(IplImage* src); #endif void print_image(image m); diff --git a/src/imagenet.c b/src/imagenet.c index 4c4d2bd6..16255263 100644 --- a/src/imagenet.c +++ b/src/imagenet.c @@ -39,7 +39,7 @@ void train_imagenet(char *cfgfile, char *weightfile) args.m = N; args.labels = labels; args.d = &buffer; - args.type = CLASSIFICATION_DATA; + args.type = OLD_CLASSIFICATION_DATA; load_thread = load_data_in_thread(args); int epoch = (*net.seen)/N; @@ -115,7 +115,7 @@ void validate_imagenet(char *filename, char *weightfile) args.m = 0; args.labels = labels; args.d = &buffer; - args.type = CLASSIFICATION_DATA; + args.type = OLD_CLASSIFICATION_DATA; pthread_t load_thread = load_data_in_thread(args); for(i = 1; i <= splits; ++i){ diff --git a/src/layer.h b/src/layer.h index 91042a21..93083708 100644 --- a/src/layer.h +++ b/src/layer.h @@ -22,7 +22,8 @@ typedef enum { LOCAL, SHORTCUT, ACTIVE, - RNN + RNN, + CRNN } LAYER_TYPE; typedef enum{ diff --git a/src/network.c b/src/network.c index 32c3ba14..e6fb51e5 100644 --- a/src/network.c +++ b/src/network.c @@ -9,6 +9,7 @@ #include "crop_layer.h" #include "connected_layer.h" #include "rnn_layer.h" +#include "crnn_layer.h" #include "local_layer.h" #include "convolutional_layer.h" #include "activation_layer.h" @@ -85,6 +86,8 @@ char *get_layer_string(LAYER_TYPE a) return "connected"; case RNN: return "rnn"; + case CRNN: + return "crnn"; case MAXPOOL: return "maxpool"; case AVGPOOL: @@ -149,6 +152,8 @@ void forward_network(network net, network_state state) forward_connected_layer(l, state); } else if(l.type == RNN){ forward_rnn_layer(l, state); + } else if(l.type == CRNN){ + forward_crnn_layer(l, state); } else if(l.type == CROP){ forward_crop_layer(l, state); } else if(l.type == COST){ @@ -185,6 +190,8 @@ void update_network(network net) update_connected_layer(l, update_batch, rate, net.momentum, net.decay); } else if(l.type == RNN){ update_rnn_layer(l, update_batch, rate, net.momentum, net.decay); + } else if(l.type == CRNN){ + update_crnn_layer(l, update_batch, rate, net.momentum, net.decay); } else if(l.type == LOCAL){ update_local_layer(l, update_batch, rate, net.momentum, net.decay); } @@ -205,7 +212,7 @@ float get_network_cost(network net) int count = 0; for(i = 0; i < net.n; ++i){ if(net.layers[i].type == COST){ - sum += net.layers[i].output[0]; + sum += net.layers[i].cost[0]; ++count; } if(net.layers[i].type == DETECTION){ @@ -261,6 +268,8 @@ void backward_network(network net, network_state state) backward_connected_layer(l, state); } else if(l.type == RNN){ backward_rnn_layer(l, state); + } else if(l.type == CRNN){ + backward_crnn_layer(l, state); } else if(l.type == LOCAL){ backward_local_layer(l, state); } else if(l.type == COST){ diff --git a/src/network.h b/src/network.h index 3d7c5746..f4f8b5cd 100644 --- a/src/network.h +++ b/src/network.h @@ -36,6 +36,7 @@ typedef struct network{ int inputs; int h, w, c; + int max_crop; #ifdef GPU float **input_gpu; diff --git a/src/network_kernels.cu b/src/network_kernels.cu index ea128194..730634ef 100644 --- a/src/network_kernels.cu +++ b/src/network_kernels.cu @@ -16,6 +16,7 @@ extern "C" { #include "crop_layer.h" #include "connected_layer.h" #include "rnn_layer.h" +#include "crnn_layer.h" #include "detection_layer.h" #include "convolutional_layer.h" #include "activation_layer.h" @@ -59,6 +60,8 @@ void forward_network_gpu(network net, network_state state) forward_connected_layer_gpu(l, state); } else if(l.type == RNN){ forward_rnn_layer_gpu(l, state); + } else if(l.type == CRNN){ + forward_crnn_layer_gpu(l, state); } else if(l.type == CROP){ forward_crop_layer_gpu(l, state); } else if(l.type == COST){ @@ -122,6 +125,8 @@ void backward_network_gpu(network net, network_state state) backward_connected_layer_gpu(l, state); } else if(l.type == RNN){ backward_rnn_layer_gpu(l, state); + } else if(l.type == CRNN){ + backward_crnn_layer_gpu(l, state); } else if(l.type == COST){ backward_cost_layer_gpu(l, state); } else if(l.type == ROUTE){ @@ -147,6 +152,8 @@ void update_network_gpu(network net) update_connected_layer_gpu(l, update_batch, rate, net.momentum, net.decay); } else if(l.type == RNN){ update_rnn_layer_gpu(l, update_batch, rate, net.momentum, net.decay); + } else if(l.type == CRNN){ + update_crnn_layer_gpu(l, update_batch, rate, net.momentum, net.decay); } else if(l.type == LOCAL){ update_local_layer_gpu(l, update_batch, rate, net.momentum, net.decay); } diff --git a/src/nightmare.c b/src/nightmare.c index 2b1c76cd..ec7166cc 100644 --- a/src/nightmare.c +++ b/src/nightmare.c @@ -8,6 +8,8 @@ #include "opencv2/highgui/highgui_c.h" #endif +// ./darknet nightmare cfg/extractor.recon.cfg ~/trained/yolo-coco.conv frame6.png -reconstruct -iters 500 -i 3 -lambda .1 -rate .01 -smooth 2 + float abs_mean(float *x, int n) { int i; @@ -31,8 +33,8 @@ void calculate_loss(float *output, float *delta, int n, float thresh) void optimize_picture(network *net, image orig, int max_layer, float scale, float rate, float thresh, int norm) { - scale_image(orig, 2); - translate_image(orig, -1); + //scale_image(orig, 2); + //translate_image(orig, -1); net->n = max_layer + 1; int dx = rand()%16 - 8; @@ -98,8 +100,8 @@ void optimize_picture(network *net, image orig, int max_layer, float scale, floa translate_image(orig, mean); */ - translate_image(orig, 1); - scale_image(orig, .5); + //translate_image(orig, 1); + //scale_image(orig, .5); //normalize_image(orig); constrain_image(orig); @@ -133,50 +135,47 @@ void smooth(image recon, image update, float lambda, int num) } } -void reconstruct_picture(network net, float *features, image recon, image update, float rate, float momentum, float lambda, int smooth_size) +void reconstruct_picture(network net, float *features, image recon, image update, float rate, float momentum, float lambda, int smooth_size, int iters) { - scale_image(recon, 2); - translate_image(recon, -1); + int iter = 0; + for (iter = 0; iter < iters; ++iter) { + image delta = make_image(recon.w, recon.h, recon.c); - image delta = make_image(recon.w, recon.h, recon.c); - - network_state state = {0}; + network_state state = {0}; #ifdef GPU - state.input = cuda_make_array(recon.data, recon.w*recon.h*recon.c); - state.delta = cuda_make_array(delta.data, delta.w*delta.h*delta.c); - state.truth = cuda_make_array(features, get_network_output_size(net)); + state.input = cuda_make_array(recon.data, recon.w*recon.h*recon.c); + state.delta = cuda_make_array(delta.data, delta.w*delta.h*delta.c); + state.truth = cuda_make_array(features, get_network_output_size(net)); - forward_network_gpu(net, state); - backward_network_gpu(net, state); + forward_network_gpu(net, state); + backward_network_gpu(net, state); - cuda_pull_array(state.delta, delta.data, delta.w*delta.h*delta.c); + cuda_pull_array(state.delta, delta.data, delta.w*delta.h*delta.c); - cuda_free(state.input); - cuda_free(state.delta); - cuda_free(state.truth); + cuda_free(state.input); + cuda_free(state.delta); + cuda_free(state.truth); #else - state.input = recon.data; - state.delta = delta.data; - state.truth = features; + state.input = recon.data; + state.delta = delta.data; + state.truth = features; - forward_network(net, state); - backward_network(net, state); + forward_network(net, state); + backward_network(net, state); #endif - axpy_cpu(recon.w*recon.h*recon.c, 1, delta.data, 1, update.data, 1); - smooth(recon, update, lambda, smooth_size); + axpy_cpu(recon.w*recon.h*recon.c, 1, delta.data, 1, update.data, 1); + smooth(recon, update, lambda, smooth_size); - axpy_cpu(recon.w*recon.h*recon.c, rate, update.data, 1, recon.data, 1); - scal_cpu(recon.w*recon.h*recon.c, momentum, update.data, 1); + axpy_cpu(recon.w*recon.h*recon.c, rate, update.data, 1, recon.data, 1); + scal_cpu(recon.w*recon.h*recon.c, momentum, update.data, 1); - translate_image(recon, 1); - scale_image(recon, .5); + //float mag = mag_array(recon.data, recon.w*recon.h*recon.c); + //scal_cpu(recon.w*recon.h*recon.c, 600/mag, recon.data, 1); - float mag = mag_array(recon.data, recon.w*recon.h*recon.c); - scal_cpu(recon.w*recon.h*recon.c, 600/mag, recon.data, 1); - - constrain_image(recon); - free_image(delta); + constrain_image(recon); + free_image(delta); + } } @@ -226,7 +225,7 @@ void run_nightmare(int argc, char **argv) im = resized; } - float *features; + float *features = 0; image update; if (reconstruct){ resize_network(&net, im.w, im.h); @@ -241,13 +240,19 @@ void run_nightmare(int argc, char **argv) printf("%d features\n", out_im.w*out_im.h*out_im.c); - im = resize_image(im, im.w*2, im.h); - f_im = resize_image(f_im, f_im.w*2, f_im.h); + im = resize_image(im, im.w, im.h); + f_im = resize_image(f_im, f_im.w, f_im.h); features = f_im.data; + int i; + for(i = 0; i < 14*14*512; ++i){ + features[i] += rand_uniform(-.19, .19); + } + free_image(im); im = make_random_image(im.w, im.h, im.c); update = make_image(im.w, im.h, im.c); + } int e; @@ -259,11 +264,12 @@ void run_nightmare(int argc, char **argv) fprintf(stderr, "%d, ", n); fflush(stderr); if(reconstruct){ - reconstruct_picture(net, features, im, update, rate, momentum, lambda, smooth_size); + reconstruct_picture(net, features, im, update, rate, momentum, lambda, smooth_size, 1); + //if ((n+1)%30 == 0) rate *= .5; show_image(im, "reconstruction"); - #ifdef OPENCV +#ifdef OPENCV cvWaitKey(10); - #endif +#endif }else{ int layer = max_layer + rand()%range - range/2; int octave = rand()%octaves; diff --git a/src/parser.c b/src/parser.c index 8051fd7f..97ce7a16 100644 --- a/src/parser.c +++ b/src/parser.c @@ -12,6 +12,7 @@ #include "deconvolutional_layer.h" #include "connected_layer.h" #include "rnn_layer.h" +#include "crnn_layer.h" #include "maxpool_layer.h" #include "softmax_layer.h" #include "dropout_layer.h" @@ -36,6 +37,7 @@ int is_local(section *s); int is_deconvolutional(section *s); int is_connected(section *s); int is_rnn(section *s); +int is_crnn(section *s); int is_maxpool(section *s); int is_avgpool(section *s); int is_dropout(section *s); @@ -169,6 +171,21 @@ convolutional_layer parse_convolutional(list *options, size_params params) return layer; } +layer parse_crnn(list *options, size_params params) +{ + int output_filters = option_find_int(options, "output_filters",1); + int hidden_filters = option_find_int(options, "hidden_filters",1); + char *activation_s = option_find_str(options, "activation", "logistic"); + ACTIVATION activation = get_activation(activation_s); + int batch_normalize = option_find_int_quiet(options, "batch_normalize", 0); + + layer l = make_crnn_layer(params.batch, params.w, params.h, params.c, hidden_filters, output_filters, params.time_steps, activation, batch_normalize); + + l.shortcut = option_find_int_quiet(options, "shortcut", 0); + + return l; +} + layer parse_rnn(list *options, size_params params) { int output = option_find_int(options, "output",1); @@ -419,6 +436,7 @@ void parse_net_options(list *options, network *net) net->w = option_find_int_quiet(options, "width",0); net->c = option_find_int_quiet(options, "channels",0); net->inputs = option_find_int_quiet(options, "inputs", net->h * net->w * net->c); + net->max_crop = option_find_int_quiet(options, "max_crop",net->w*2); if(!net->inputs && !(net->h && net->w && net->c)) error("No input parameters supplied"); @@ -501,6 +519,8 @@ network parse_network_cfg(char *filename) l = parse_deconvolutional(options, params); }else if(is_rnn(s)){ l = parse_rnn(options, params); + }else if(is_crnn(s)){ + l = parse_crnn(options, params); }else if(is_connected(s)){ l = parse_connected(options, params); }else if(is_crop(s)){ @@ -591,6 +611,10 @@ int is_network(section *s) return (strcmp(s->type, "[net]")==0 || strcmp(s->type, "[network]")==0); } +int is_crnn(section *s) +{ + return (strcmp(s->type, "[crnn]")==0); +} int is_rnn(section *s) { return (strcmp(s->type, "[rnn]")==0); @@ -705,6 +729,23 @@ void save_weights_double(network net, char *filename) fclose(fp); } +void save_convolutional_weights(layer l, FILE *fp) +{ +#ifdef GPU + if(gpu_index >= 0){ + pull_convolutional_layer(l); + } +#endif + int num = l.n*l.c*l.size*l.size; + fwrite(l.biases, sizeof(float), l.n, fp); + if (l.batch_normalize){ + fwrite(l.scales, sizeof(float), l.n, fp); + fwrite(l.rolling_mean, sizeof(float), l.n, fp); + fwrite(l.rolling_variance, sizeof(float), l.n, fp); + } + fwrite(l.filters, sizeof(float), num, fp); +} + void save_connected_weights(layer l, FILE *fp) { #ifdef GPU @@ -739,25 +780,17 @@ void save_weights_upto(network net, char *filename, int cutoff) for(i = 0; i < net.n && i < cutoff; ++i){ layer l = net.layers[i]; if(l.type == CONVOLUTIONAL){ -#ifdef GPU - if(gpu_index >= 0){ - pull_convolutional_layer(l); - } -#endif - int num = l.n*l.c*l.size*l.size; - fwrite(l.biases, sizeof(float), l.n, fp); - if (l.batch_normalize){ - fwrite(l.scales, sizeof(float), l.n, fp); - fwrite(l.rolling_mean, sizeof(float), l.n, fp); - fwrite(l.rolling_variance, sizeof(float), l.n, fp); - } - fwrite(l.filters, sizeof(float), num, fp); + save_convolutional_weights(l, fp); } if(l.type == CONNECTED){ save_connected_weights(l, fp); } if(l.type == RNN){ save_connected_weights(*(l.input_layer), fp); save_connected_weights(*(l.self_layer), fp); save_connected_weights(*(l.output_layer), fp); + } if(l.type == CRNN){ + save_convolutional_weights(*(l.input_layer), fp); + save_convolutional_weights(*(l.self_layer), fp); + save_convolutional_weights(*(l.output_layer), fp); } if(l.type == LOCAL){ #ifdef GPU if(gpu_index >= 0){ @@ -809,6 +842,27 @@ void load_connected_weights(layer l, FILE *fp, int transpose) #endif } +void load_convolutional_weights(layer l, FILE *fp) +{ + int num = l.n*l.c*l.size*l.size; + fread(l.biases, sizeof(float), l.n, fp); + if (l.batch_normalize && (!l.dontloadscales)){ + fread(l.scales, sizeof(float), l.n, fp); + fread(l.rolling_mean, sizeof(float), l.n, fp); + fread(l.rolling_variance, sizeof(float), l.n, fp); + } + fread(l.filters, sizeof(float), num, fp); + if (l.flipped) { + transpose_matrix(l.filters, l.c*l.size*l.size, l.n); + } +#ifdef GPU + if(gpu_index >= 0){ + push_convolutional_layer(l); + } +#endif +} + + void load_weights_upto(network *net, char *filename, int cutoff) { fprintf(stderr, "Loading weights from %s...", filename); @@ -830,22 +884,7 @@ void load_weights_upto(network *net, char *filename, int cutoff) layer l = net->layers[i]; if (l.dontload) continue; if(l.type == CONVOLUTIONAL){ - int num = l.n*l.c*l.size*l.size; - fread(l.biases, sizeof(float), l.n, fp); - if (l.batch_normalize && (!l.dontloadscales)){ - fread(l.scales, sizeof(float), l.n, fp); - fread(l.rolling_mean, sizeof(float), l.n, fp); - fread(l.rolling_variance, sizeof(float), l.n, fp); - } - fread(l.filters, sizeof(float), num, fp); - if (l.flipped) { - transpose_matrix(l.filters, l.c*l.size*l.size, l.n); - } -#ifdef GPU - if(gpu_index >= 0){ - push_convolutional_layer(l); - } -#endif + load_convolutional_weights(l, fp); } if(l.type == DECONVOLUTIONAL){ int num = l.n*l.c*l.size*l.size; @@ -860,6 +899,11 @@ void load_weights_upto(network *net, char *filename, int cutoff) if(l.type == CONNECTED){ load_connected_weights(l, fp, transpose); } + if(l.type == CRNN){ + load_convolutional_weights(*(l.input_layer), fp); + load_convolutional_weights(*(l.self_layer), fp); + load_convolutional_weights(*(l.output_layer), fp); + } if(l.type == RNN){ load_connected_weights(*(l.input_layer), fp, transpose); load_connected_weights(*(l.self_layer), fp, transpose); diff --git a/src/rnn.c b/src/rnn.c index 3865209b..30fa4bd0 100644 --- a/src/rnn.c +++ b/src/rnn.c @@ -71,6 +71,7 @@ void train_char_rnn(char *cfgfile, char *weightfile, char *filename) fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); int batch = net.batch; int steps = net.time_steps; + //*net.seen = 0; int i = (*net.seen)/net.batch; clock_t time; diff --git a/src/rnn_layer.c b/src/rnn_layer.c index 384169a4..35cf9923 100644 --- a/src/rnn_layer.c +++ b/src/rnn_layer.c @@ -10,7 +10,7 @@ #include #include -void increment_layer(layer *l, int steps) +static void increment_layer(layer *l, int steps) { int num = l->outputs*l->batch*steps; l->output += num; diff --git a/src/rnn_vid.c b/src/rnn_vid.c new file mode 100644 index 00000000..183ae779 --- /dev/null +++ b/src/rnn_vid.c @@ -0,0 +1,210 @@ +#include "network.h" +#include "cost_layer.h" +#include "utils.h" +#include "parser.h" +#include "blas.h" + +#ifdef OPENCV +#include "opencv2/highgui/highgui_c.h" + +void reconstruct_picture(network net, float *features, image recon, image update, float rate, float momentum, float lambda, int smooth_size, int iters); + + +typedef struct { + float *x; + float *y; +} float_pair; + +float_pair get_rnn_vid_data(network net, char **files, int n, int batch, int steps) +{ + int b; + assert(net.batch == steps + 1); + image out_im = get_network_image(net); + int output_size = out_im.w*out_im.h*out_im.c; + printf("%d %d %d\n", out_im.w, out_im.h, out_im.c); + float *feats = calloc(net.batch*batch*output_size, sizeof(float)); + for(b = 0; b < batch; ++b){ + int input_size = net.w*net.h*net.c; + float *input = calloc(input_size*net.batch, sizeof(float)); + char *filename = files[rand()%n]; + CvCapture *cap = cvCaptureFromFile(filename); + int frames = cvGetCaptureProperty(cap, CV_CAP_PROP_FRAME_COUNT); + int index = rand() % (frames - steps - 2); + if (frames < (steps + 4)){ + --b; + free(input); + continue; + } + + printf("frames: %d, index: %d\n", frames, index); + cvSetCaptureProperty(cap, CV_CAP_PROP_POS_FRAMES, index); + + int i; + for(i = 0; i < net.batch; ++i){ + IplImage* src = cvQueryFrame(cap); + image im = ipl_to_image(src); + rgbgr_image(im); + image re = resize_image(im, net.w, net.h); + //show_image(re, "loaded"); + //cvWaitKey(10); + memcpy(input + i*input_size, re.data, input_size*sizeof(float)); + free_image(im); + free_image(re); + } + float *output = network_predict(net, input); + + free(input); + + for(i = 0; i < net.batch; ++i){ + memcpy(feats + (b + i*batch)*output_size, output + i*output_size, output_size*sizeof(float)); + } + + cvReleaseCapture(&cap); + } + + //printf("%d %d %d\n", out_im.w, out_im.h, out_im.c); + float_pair p = {0}; + p.x = feats; + p.y = feats + output_size*batch; //+ out_im.w*out_im.h*out_im.c; + + return p; +} + + +void train_vid_rnn(char *cfgfile, char *weightfile) +{ + char *train_videos = "data/vid/train.txt"; + char *backup_directory = "/home/pjreddie/backup/"; + srand(time(0)); + data_seed = time(0); + char *base = basecfg(cfgfile); + printf("%s\n", base); + float avg_loss = -1; + network net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); + int imgs = net.batch*net.subdivisions; + int i = *net.seen/imgs; + + list *plist = get_paths(train_videos); + int N = plist->size; + char **paths = (char **)list_to_array(plist); + clock_t time; + int steps = net.time_steps; + int batch = net.batch / net.time_steps; + + network extractor = parse_network_cfg("cfg/extractor.cfg"); + load_weights(&extractor, "/home/pjreddie/trained/yolo-coco.conv"); + + while(get_current_batch(net) < net.max_batches){ + i += 1; + time=clock(); + float_pair p = get_rnn_vid_data(extractor, paths, N, batch, steps); + + float loss = train_network_datum(net, p.x, p.y) / (net.batch); + + + free(p.x); + if (avg_loss < 0) avg_loss = loss; + avg_loss = avg_loss*.9 + loss*.1; + + fprintf(stderr, "%d: %f, %f avg, %f rate, %lf seconds\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time)); + if(i%100==0){ + char buff[256]; + sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i); + save_weights(net, buff); + } + if(i%10==0){ + char buff[256]; + sprintf(buff, "%s/%s.backup", backup_directory, base); + save_weights(net, buff); + } + } + char buff[256]; + sprintf(buff, "%s/%s_final.weights", backup_directory, base); + save_weights(net, buff); +} + + +image save_reconstruction(network net, image *init, float *feat, char *name, int i) +{ + image recon; + if (init) { + recon = copy_image(*init); + } else { + recon = make_random_image(net.w, net.h, 3); + } + + image update = make_image(net.w, net.h, 3); + reconstruct_picture(net, feat, recon, update, .01, .9, .1, 2, 50); + char buff[256]; + sprintf(buff, "%s%d", name, i); + save_image(recon, buff); + free_image(update); + return recon; +} + +void generate_vid_rnn(char *cfgfile, char *weightfile) +{ + network extractor = parse_network_cfg("cfg/extractor.recon.cfg"); + load_weights(&extractor, "/home/pjreddie/trained/yolo-coco.conv"); + + network net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + set_batch_network(&extractor, 1); + set_batch_network(&net, 1); + + int i; + CvCapture *cap = cvCaptureFromFile("/extra/vid/ILSVRC2015/Data/VID/snippets/val/ILSVRC2015_val_00007030.mp4"); + float *feat; + float *next; + image last; + for(i = 0; i < 25; ++i){ + image im = get_image_from_stream(cap); + image re = resize_image(im, extractor.w, extractor.h); + feat = network_predict(extractor, re.data); + if(i > 0){ + printf("%f %f\n", mean_array(feat, 14*14*512), variance_array(feat, 14*14*512)); + printf("%f %f\n", mean_array(next, 14*14*512), variance_array(next, 14*14*512)); + printf("%f\n", mse_array(feat, 14*14*512)); + axpy_cpu(14*14*512, -1, feat, 1, next, 1); + printf("%f\n", mse_array(next, 14*14*512)); + } + next = network_predict(net, feat); + + free_image(im); + + free_image(save_reconstruction(extractor, 0, feat, "feat", i)); + free_image(save_reconstruction(extractor, 0, next, "next", i)); + if (i==24) last = copy_image(re); + free_image(re); + } + for(i = 0; i < 30; ++i){ + next = network_predict(net, next); + image new = save_reconstruction(extractor, &last, next, "new", i); + free_image(last); + last = new; + } +} + +void run_vid_rnn(int argc, char **argv) +{ + if(argc < 4){ + fprintf(stderr, "usage: %s %s [train/test/valid] [cfg] [weights (optional)]\n", argv[0], argv[1]); + return; + } + + char *cfg = argv[3]; + char *weights = (argc > 4) ? argv[4] : 0; + //char *filename = (argc > 5) ? argv[5]: 0; + if(0==strcmp(argv[2], "train")) train_vid_rnn(cfg, weights); + else if(0==strcmp(argv[2], "generate")) generate_vid_rnn(cfg, weights); +} +#else +void run_vid_rnn(int argc, char **argv){} +#endif + diff --git a/src/tag.c b/src/tag.c new file mode 100644 index 00000000..8b63d31c --- /dev/null +++ b/src/tag.c @@ -0,0 +1,144 @@ +#include "network.h" +#include "utils.h" +#include "parser.h" + +#ifdef OPENCV +#include "opencv2/highgui/highgui_c.h" +#endif + +void train_tag(char *cfgfile, char *weightfile) +{ + data_seed = time(0); + srand(time(0)); + float avg_loss = -1; + char *base = basecfg(cfgfile); + char *backup_directory = "/home/pjreddie/backup/"; + printf("%s\n", base); + network net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay); + int imgs = 1024; + list *plist = get_paths("/home/pjreddie/tag/train.list"); + char **paths = (char **)list_to_array(plist); + printf("%d\n", plist->size); + int N = plist->size; + clock_t time; + pthread_t load_thread; + data train; + data buffer; + + load_args args = {0}; + args.w = net.w; + args.h = net.h; + + args.min = net.w; + args.max = net.max_crop; + args.size = net.w; + + args.paths = paths; + args.classes = net.outputs; + args.n = imgs; + args.m = N; + args.d = &buffer; + args.type = TAG_DATA; + + fprintf(stderr, "%d classes\n", net.outputs); + + load_thread = load_data_in_thread(args); + int epoch = (*net.seen)/N; + while(get_current_batch(net) < net.max_batches || net.max_batches == 0){ + time=clock(); + pthread_join(load_thread, 0); + train = buffer; + + load_thread = load_data_in_thread(args); + printf("Loaded: %lf seconds\n", sec(clock()-time)); + time=clock(); + float loss = train_network(net, train); + if(avg_loss == -1) avg_loss = loss; + avg_loss = avg_loss*.9 + loss*.1; + printf("%d, %.3f: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), (float)(*net.seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen); + free_data(train); + if(*net.seen/N > epoch){ + epoch = *net.seen/N; + char buff[256]; + sprintf(buff, "%s/%s_%d.weights",backup_directory,base, epoch); + save_weights(net, buff); + } + if(get_current_batch(net)%100 == 0){ + char buff[256]; + sprintf(buff, "%s/%s.backup",backup_directory,base); + save_weights(net, buff); + } + } + char buff[256]; + sprintf(buff, "%s/%s.weights", backup_directory, base); + save_weights(net, buff); + + pthread_join(load_thread, 0); + free_data(buffer); + free_network(net); + free_ptrs((void**)paths, plist->size); + free_list(plist); + free(base); +} + +void test_tag(char *cfgfile, char *weightfile, char *filename) +{ + network net = parse_network_cfg(cfgfile); + if(weightfile){ + load_weights(&net, weightfile); + } + set_batch_network(&net, 1); + srand(2222222); + int i = 0; + char **names = get_labels("data/tags.txt"); + clock_t time; + int indexes[10]; + char buff[256]; + char *input = buff; + while(1){ + if(filename){ + strncpy(input, filename, 256); + }else{ + printf("Enter Image Path: "); + fflush(stdout); + input = fgets(input, 256, stdin); + if(!input) return; + strtok(input, "\n"); + } + image im = load_image_color(input, net.w, net.h); + //resize_network(&net, im.w, im.h); + printf("%d %d\n", im.w, im.h); + + float *X = im.data; + time=clock(); + float *predictions = network_predict(net, X); + top_predictions(net, 10, indexes); + printf("%s: Predicted in %f seconds.\n", input, sec(clock()-time)); + for(i = 0; i < 10; ++i){ + int index = indexes[i]; + printf("%.1f%%: %s\n", predictions[index]*100, names[index]); + } + free_image(im); + if (filename) break; + } +} + + +void run_tag(int argc, char **argv) +{ + if(argc < 4){ + fprintf(stderr, "usage: %s %s [train/test/valid] [cfg] [weights (optional)]\n", argv[0], argv[1]); + return; + } + + char *cfg = argv[3]; + char *weights = (argc > 4) ? argv[4] : 0; + char *filename = (argc > 5) ? argv[5] : 0; + if(0==strcmp(argv[2], "train")) train_tag(cfg, weights); + else if(0==strcmp(argv[2], "test")) test_tag(cfg, weights, filename); +} + diff --git a/src/utils.c b/src/utils.c index ec87a265..398d18a8 100644 --- a/src/utils.c +++ b/src/utils.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -137,15 +138,18 @@ void pm(int M, int N, float *A) char *find_replace(char *str, char *orig, char *rep) { static char buffer[4096]; + static char buffer2[4096]; + static char buffer3[4096]; char *p; if(!(p = strstr(str, orig))) // Is 'orig' even in 'str'? return str; - strncpy(buffer, str, p-str); // Copy characters from 'str' start to 'orig' st$ - buffer[p-str] = '\0'; + strncpy(buffer2, str, p-str); // Copy characters from 'str' start to 'orig' st$ + buffer2[p-str] = '\0'; - sprintf(buffer+(p-str), "%s%s", rep, p+strlen(orig)); + sprintf(buffer3, "%s%s%s", buffer2, rep, p+strlen(orig)); + sprintf(buffer, "%s", buffer3); return buffer; } @@ -174,7 +178,8 @@ void top_k(float *a, int n, int k, int *index) void error(const char *s) { perror(s); - exit(0); + assert(0); + exit(-1); } void malloc_error() @@ -450,6 +455,12 @@ int max_index(float *a, int n) return max_i; } +int rand_int(int min, int max) +{ + int r = (rand()%(max - min + 1)) + min; + return r; +} + // From http://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform #define TWO_PI 6.2831853071795864769252866 float rand_normal() diff --git a/src/utils.h b/src/utils.h index 96bd6cfe..3af85d33 100644 --- a/src/utils.h +++ b/src/utils.h @@ -35,6 +35,7 @@ float constrain(float min, float max, float a); float mse_array(float *a, int n); float rand_normal(); float rand_uniform(float min, float max); +int rand_int(int min, int max); float sum_array(float *a, int n); float mean_array(float *a, int n); void mean_arrays(float **a, int n, int els, float *avg);