From 3fb3eec6502ff8e9b77ce6d43d835b41d0c3241f Mon Sep 17 00:00:00 2001
From: Joseph Redmon <pjreddie@gmail.com>
Date: Tue, 7 Nov 2017 16:10:33 -0800
Subject: [PATCH 1/3] OK SHOULD I START WORKING ON CVPR OR WHAT?

---
 examples/attention.c              | 195 +++++--
 examples/classifier.c             |   2 +-
 examples/darknet.c                |   3 +
 examples/detector-scipy-opencv.py |  56 ++
 examples/detector.c               |   2 +-
 examples/go.c                     | 813 +++++++++++++++++++++++-------
 include/darknet.h                 |  10 +
 python/darknet.py                 |   9 +
 src/activation_kernels.cu         |  35 ++
 src/data.c                        |  55 +-
 src/utils.c                       |  30 ++
 src/utils.h                       |   1 -
 12 files changed, 1003 insertions(+), 208 deletions(-)
 create mode 100644 examples/detector-scipy-opencv.py

diff --git a/examples/attention.c b/examples/attention.c
index e7f15245..1afd8489 100644
--- a/examples/attention.c
+++ b/examples/attention.c
@@ -3,16 +3,64 @@
 #include <sys/time.h>
 #include <assert.h>
 
-void train_attention(char *datacfg, char *cfgfile, char *weightfile, char *cfgfile2, char *weightfile2, int *gpus, int ngpus, int clear)
+void extend_data_truth(data *d, int n, float val)
 {
-    int i;
+    int i, j;
+    for(i = 0; i < d->y.rows; ++i){
+        d->y.vals[i] = realloc(d->y.vals[i], (d->y.cols+n)*sizeof(float));
+        for(j = 0; j < n; ++j){
+            d->y.vals[i][d->y.cols + j] = val;
+        }
+    }
+    d->y.cols += n;
+}
 
-    float avg_loss = -1;
+matrix network_loss_data(network *net, data test)
+{
+    int i,b;
+    int k = 1;
+    matrix pred = make_matrix(test.X.rows, k);
+    float *X = calloc(net->batch*test.X.cols, sizeof(float));
+    float *y = calloc(net->batch*test.y.cols, sizeof(float));
+    for(i = 0; i < test.X.rows; i += net->batch){
+        for(b = 0; b < net->batch; ++b){
+            if(i+b == test.X.rows) break;
+            memcpy(X+b*test.X.cols, test.X.vals[i+b], test.X.cols*sizeof(float));
+            memcpy(y+b*test.y.cols, test.y.vals[i+b], test.y.cols*sizeof(float));
+        }
+
+        network orig = *net;
+        net->input = X;
+        net->truth = y;
+        net->train = 0;
+        net->delta = 0;
+        forward_network(net);
+        *net = orig;
+
+        float *delta = net->layers[net->n-1].output;
+        for(b = 0; b < net->batch; ++b){
+            if(i+b == test.X.rows) break;
+            int t = max_index(y + b*test.y.cols, 1000);
+            float err = sum_array(delta + b*net->outputs, net->outputs);
+            pred.vals[i+b][0] = -err;
+            //pred.vals[i+b][0] = 1-delta[b*net->outputs + t];
+        }
+    }
+    free(X);
+    free(y);
+    return pred;   
+}
+
+void train_attention(char *datacfg, char *cfgfile, char *weightfile, int *gpus, int ngpus, int clear)
+{
+    int i, j;
+
+    float avg_cls_loss = -1;
+    float avg_att_loss = -1;
     char *base = basecfg(cfgfile);
     printf("%s\n", base);
     printf("%d\n", ngpus);
-    network **attnets = calloc(ngpus, sizeof(network*));
-    network **clsnets = calloc(ngpus, sizeof(network*));
+    network **nets = calloc(ngpus, sizeof(network*));
 
     srand(time(0));
     int seed = rand();
@@ -21,14 +69,11 @@ void train_attention(char *datacfg, char *cfgfile, char *weightfile, char *cfgfi
 #ifdef GPU
         cuda_set_device(gpus[i]);
 #endif
-        attnets[i] = load_network(cfgfile, weightfile, clear);
-        attnets[i]->learning_rate *= ngpus;
-        clsnets[i] = load_network(cfgfile2, weightfile2, clear);
-        clsnets[i]->learning_rate *= ngpus;
+        nets[i] = load_network(cfgfile, weightfile, clear);
+        nets[i]->learning_rate *= ngpus;
     }
     srand(time(0));
-    network *net = attnets[0];
-    //network *clsnet = clsnets[0];
+    network *net = nets[0];
 
     int imgs = net->batch * net->subdivisions * ngpus;
 
@@ -47,15 +92,18 @@ void train_attention(char *datacfg, char *cfgfile, char *weightfile, char *cfgfi
     int N = plist->size;
     double time;
 
+    int divs=3;
+    int size=2;
+
     load_args args = {0};
-    args.w = 4*net->w;
-    args.h = 4*net->h;
-    args.size = 4*net->w;
+    args.w = divs*net->w/size;
+    args.h = divs*net->h/size;
+    args.size = divs*net->w/size;
     args.threads = 32;
     args.hierarchy = net->hierarchy;
 
-    args.min = net->min_ratio*net->w;
-    args.max = net->max_ratio*net->w;
+    args.min = net->min_ratio*args.w;
+    args.max = net->max_ratio*args.w;
     args.angle = net->angle;
     args.aspect = net->aspect;
     args.exposure = net->exposure;
@@ -83,25 +131,81 @@ void train_attention(char *datacfg, char *cfgfile, char *weightfile, char *cfgfi
         train = buffer;
         load_thread = load_data(args);
         data resized = resize_data(train, net->w, net->h);
+        extend_data_truth(&resized, divs*divs, 0);
+        data *tiles = tile_data(train, divs, size);
 
         printf("Loaded: %lf seconds\n", what_time_is_it_now()-time);
         time = what_time_is_it_now();
 
-        float loss = 0;
-#ifdef GPU
-        if(ngpus == 1){
-            loss = train_network(net, train);
-        } else {
-            loss = train_networks(attnets, ngpus, train, 4);
+        float aloss = 0;
+        float closs = 0;
+        int z;
+        for (i = 0; i < divs*divs/ngpus; ++i) {
+#pragma omp parallel for
+            for(j = 0; j < ngpus; ++j){
+                int index = i*ngpus + j;
+                extend_data_truth(tiles+index, divs*divs, SECRET_NUM);
+                matrix deltas = network_loss_data(nets[j], tiles[index]);
+                for(z = 0; z < resized.y.rows; ++z){
+                    resized.y.vals[z][train.y.cols + index] = deltas.vals[z][0];
+                }
+                free_matrix(deltas);
+            }
+        }
+        int *inds = calloc(resized.y.rows, sizeof(int));
+        for(z = 0; z < resized.y.rows; ++z){
+            int index = max_index(resized.y.vals[z] + train.y.cols, divs*divs);
+            inds[z] = index;
+            for(i = 0; i < divs*divs; ++i){
+                resized.y.vals[z][train.y.cols + i] = (i == index)? 1 : 0;
+            }
+        }
+        data best = select_data(tiles, inds);
+        free(inds);
+        #ifdef GPU
+        if (ngpus == 1) {
+            closs = train_network(net, best);
+        } else {
+            closs = train_networks(nets, ngpus, best, 4);
+        }
+        #endif
+        for (i = 0; i < divs*divs; ++i) {
+            printf("%.2f ", resized.y.vals[0][train.y.cols + i]);
+            if((i+1)%divs == 0) printf("\n");
+            free_data(tiles[i]);
+        }
+        free_data(best);
+        printf("\n");
+        image im = float_to_image(64,64,3,resized.X.vals[0]);
+        //show_image(im, "orig");
+        //cvWaitKey(100);
+        /*
+           image im1 = float_to_image(64,64,3,tiles[i].X.vals[0]);
+           image im2 = float_to_image(64,64,3,resized.X.vals[0]);
+           show_image(im1, "tile");
+           show_image(im2, "res");
+         */
+#ifdef GPU
+        if (ngpus == 1) {
+            aloss = train_network(net, resized);
+        } else {
+            aloss = train_networks(nets, ngpus, resized, 4);
         }
-#else
-        loss = train_network(net, train);
 #endif
+        for(i = 0; i < divs*divs; ++i){
+            printf("%f ", nets[0]->output[1000 + i]);
+            if ((i+1) % divs == 0) printf("\n");
+        }
+        printf("\n");
+
         free_data(resized);
-        if(avg_loss == -1) avg_loss = loss;
-        avg_loss = avg_loss*.9 + loss*.1;
-        printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, loss, avg_loss, get_current_rate(net), what_time_is_it_now()-time, *net->seen);
         free_data(train);
+        if(avg_cls_loss == -1) avg_cls_loss = closs;
+        if(avg_att_loss == -1) avg_att_loss = aloss;
+        avg_cls_loss = avg_cls_loss*.9 + closs*.1;
+        avg_att_loss = avg_att_loss*.9 + aloss*.1;
+
+        printf("%ld, %.3f: Att: %f, %f avg, Class: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, aloss, avg_att_loss, closs, avg_cls_loss, get_current_rate(net), what_time_is_it_now()-time, *net->seen);
         if(*net->seen/N > epoch){
             epoch = *net->seen/N;
             char buff[256];
@@ -152,6 +256,11 @@ void validate_attention_single(char *datacfg, char *filename, char *weightfile)
     float avg_acc = 0;
     float avg_topk = 0;
     int *indexes = calloc(topk, sizeof(int));
+    int divs = 4;
+    int size = 2;
+    int extra = 0;
+    float *avgs = calloc(classes, sizeof(float));
+    int *inds = calloc(divs*divs, sizeof(int));
 
     for(i = 0; i < m; ++i){
         int class = -1;
@@ -163,14 +272,38 @@ void validate_attention_single(char *datacfg, char *filename, char *weightfile)
             }
         }
         image im = load_image_color(paths[i], 0, 0);
-        image resized = resize_min(im, net->w);
-        image crop = crop_image(resized, (resized.w - net->w)/2, (resized.h - net->h)/2, net->w, net->h);
+        image resized = resize_min(im, net->w*divs/size);
+        image crop = crop_image(resized, (resized.w - net->w*divs/size)/2, (resized.h - net->h*divs/size)/2, net->w*divs/size, net->h*divs/size);
+        image rcrop = resize_image(crop, net->w, net->h);
         //show_image(im, "orig");
         //show_image(crop, "cropped");
         //cvWaitKey(0);
-        float *pred = network_predict(net, crop.data);
+        float *pred = network_predict(net, rcrop.data);
+        //pred[classes + 56] = 0;
+        for(j = 0; j < divs*divs; ++j){
+            printf("%.2f ", pred[classes + j]);
+            if((j+1)%divs == 0) printf("\n");
+        }
+        printf("\n");
+        copy_cpu(classes, pred, 1, avgs, 1);
+        top_k(pred + classes, divs*divs, divs*divs, inds);
+        show_image(crop, "crop");
+        for(j = 0; j < extra; ++j){
+            int index = inds[j];
+            int row = index / divs;
+            int col = index % divs;
+            int y = row * crop.h / divs - (net->h - crop.h/divs)/2;
+            int x = col * crop.w / divs - (net->w - crop.w/divs)/2;
+            printf("%d %d %d %d\n", row, col, y, x);
+            image tile = crop_image(crop, x, y, net->w, net->h);
+            float *pred = network_predict(net, tile.data);
+            axpy_cpu(classes, 1., pred, 1, avgs, 1);
+            show_image(tile, "tile");
+            cvWaitKey(10);
+        }
         if(net->hierarchy) hierarchy_predictions(pred, net->outputs, net->hierarchy, 1, 1);
 
+        if(rcrop.data != resized.data) free_image(rcrop);
         if(resized.data != im.data) free_image(resized);
         free_image(im);
         free_image(crop);
@@ -318,7 +451,7 @@ void run_attention(int argc, char **argv)
     char *filename = (argc > 6) ? argv[6]: 0;
     char *layer_s = (argc > 7) ? argv[7]: 0;
     if(0==strcmp(argv[2], "predict")) predict_attention(data, cfg, weights, filename, top);
-    else if(0==strcmp(argv[2], "train")) train_attention(data, cfg, weights, filename, layer_s, gpus, ngpus, clear);
+    else if(0==strcmp(argv[2], "train")) train_attention(data, cfg, weights, gpus, ngpus, clear);
     else if(0==strcmp(argv[2], "valid")) validate_attention_single(data, cfg, weights);
     else if(0==strcmp(argv[2], "validmulti")) validate_attention_multi(data, cfg, weights);
 }
diff --git a/examples/classifier.c b/examples/classifier.c
index ad80e5f7..8843e548 100644
--- a/examples/classifier.c
+++ b/examples/classifier.c
@@ -447,7 +447,7 @@ void validate_classifier_multi(char *datacfg, char *cfg, char *weights)
         float *pred = calloc(classes, sizeof(float));
         image im = load_image_color(paths[i], 0, 0);
         for(j = 0; j < nscales; ++j){
-            image r = resize_min(im, scales[j]);
+            image r = resize_max(im, scales[j]);
             resize_network(net, r.w, r.h);
             float *p = network_predict(net, r.data);
             if(net->hierarchy) hierarchy_predictions(p, net->outputs, net->hierarchy, 1 , 1);
diff --git a/examples/darknet.c b/examples/darknet.c
index c67f20a1..b89f69ab 100644
--- a/examples/darknet.c
+++ b/examples/darknet.c
@@ -12,6 +12,7 @@ extern void run_coco(int argc, char **argv);
 extern void run_captcha(int argc, char **argv);
 extern void run_nightmare(int argc, char **argv);
 extern void run_classifier(int argc, char **argv);
+extern void run_attention(int argc, char **argv);
 extern void run_regressor(int argc, char **argv);
 extern void run_segmenter(int argc, char **argv);
 extern void run_char_rnn(int argc, char **argv);
@@ -431,6 +432,8 @@ int main(int argc, char **argv)
         predict_classifier("cfg/imagenet1k.data", argv[2], argv[3], argv[4], 5);
     } else if (0 == strcmp(argv[1], "classifier")){
         run_classifier(argc, argv);
+    } else if (0 == strcmp(argv[1], "attention")){
+        run_attention(argc, argv);
     } else if (0 == strcmp(argv[1], "regressor")){
         run_regressor(argc, argv);
     } else if (0 == strcmp(argv[1], "segmenter")){
diff --git a/examples/detector-scipy-opencv.py b/examples/detector-scipy-opencv.py
new file mode 100644
index 00000000..3bfc5913
--- /dev/null
+++ b/examples/detector-scipy-opencv.py
@@ -0,0 +1,56 @@
+# Stupid python path shit.
+# Instead just add darknet.py to somewhere in your python path
+# OK actually that might not be a great idea, idk, work in progress
+# Use at your own risk. or don't, i don't care
+
+from scipy.misc import imread
+import cv2
+
+def array_to_image(arr):
+    arr = arr.transpose(2,0,1)
+    c = arr.shape[0]
+    h = arr.shape[1]
+    w = arr.shape[2]
+    arr = (arr/255.0).flatten()
+    data = dn.c_array(dn.c_float, arr)
+    im = dn.IMAGE(w,h,c,data)
+    return im
+
+def detect2(net, meta, image, thresh=.5, hier_thresh=.5, nms=.45):
+    boxes = dn.make_boxes(net)
+    probs = dn.make_probs(net)
+    num =   dn.num_boxes(net)
+    dn.network_detect(net, image, thresh, hier_thresh, nms, boxes, probs)
+    res = []
+    for j in range(num):
+        for i in range(meta.classes):
+            if probs[j][i] > 0:
+                res.append((meta.names[i], probs[j][i], (boxes[j].x, boxes[j].y, boxes[j].w, boxes[j].h)))
+    res = sorted(res, key=lambda x: -x[1])
+    dn.free_ptrs(dn.cast(probs, dn.POINTER(dn.c_void_p)), num)
+    return res
+
+import sys, os
+sys.path.append(os.path.join(os.getcwd(),'python/'))
+
+import darknet as dn
+
+# Darknet
+net = dn.load_net("cfg/tiny-yolo.cfg", "tiny-yolo.weights", 0)
+meta = dn.load_meta("cfg/coco.data")
+r = dn.detect(net, meta, "data/dog.jpg")
+print r
+
+# scipy
+arr= imread('data/dog.jpg')
+im = array_to_image(arr)
+r = detect2(net, meta, im)
+print r
+
+# OpenCV
+arr = cv2.imread('data/dog.jpg')
+im = array_to_image(arr)
+dn.rgbgr_image(im)
+r = detect2(net, meta, im)
+print r
+
diff --git a/examples/detector.c b/examples/detector.c
index ed6815d5..15575331 100644
--- a/examples/detector.c
+++ b/examples/detector.c
@@ -609,8 +609,8 @@ void test_detector(char *datacfg, char *cfgfile, char *weightfile, char *filenam
         network_predict(net, X);
         printf("%s: Predicted in %f seconds.\n", input, what_time_is_it_now()-time);
         get_region_boxes(l, im.w, im.h, net->w, net->h, thresh, probs, boxes, masks, 0, 0, hier_thresh, 1);
+        //if (nms) do_nms_obj(boxes, probs, l.w*l.h*l.n, l.classes, nms);
         if (nms) do_nms_sort(boxes, probs, l.w*l.h*l.n, l.classes, nms);
-        //else if (nms) do_nms_sort(boxes, probs, l.w*l.h*l.n, l.classes, nms);
         draw_detections(im, l.w*l.h*l.n, thresh, boxes, probs, masks, names, alphabet, l.classes);
         if(outfile){
             save_image(im, outfile);
diff --git a/examples/go.c b/examples/go.c
index f99eebb5..688579dc 100644
--- a/examples/go.c
+++ b/examples/go.c
@@ -1,10 +1,14 @@
 #include "darknet.h"
 
+#include <assert.h>
+#include <math.h>
 #include <unistd.h>
 
 int inverted = 1;
 int noi = 1;
-static const int nind = 2;
+static const int nind = 10;
+int legal_go(float *b, float *ko, int p, int r, int c);
+int check_ko(float *x, float *ko);
 
 typedef struct {
     char **data;
@@ -14,7 +18,7 @@ typedef struct {
 char *fgetgo(FILE *fp)
 {
     if(feof(fp)) return 0;
-    size_t size = 94;
+    size_t size = 96;
     char *line = malloc(size*sizeof(char));
     if(size != fread(line, sizeof(char), size, fp)){
         free(line);
@@ -32,8 +36,8 @@ moves load_go_moves(char *filename)
     FILE *fp = fopen(filename, "rb");
     int count = 0;
     char *line = 0;
-    while((line = fgetgo(fp))){
-        if(count >= m.n){
+    while ((line = fgetgo(fp))) {
+        if (count >= m.n) {
             m.n *= 2;
             m.data = realloc(m.data, m.n*sizeof(char*));
         }
@@ -49,7 +53,7 @@ moves load_go_moves(char *filename)
 void string_to_board(char *s, float *board)
 {
     int i, j;
-    //memset(board, 0, 1*19*19*sizeof(float));
+    memset(board, 0, 2*19*19*sizeof(float));
     int count = 0;
     for(i = 0; i < 91; ++i){
         char c = s[i];
@@ -57,8 +61,7 @@ void string_to_board(char *s, float *board)
             int me = (c >> (2*j)) & 1;
             int you = (c >> (2*j + 1)) & 1;
             if (me) board[count] = 1;
-            else if (you) board[count] = -1;
-            else board[count] = 0;
+            else if (you) board[count + 19*19] = 1;
             ++count;
             if(count >= 19*19) break;
         }
@@ -73,7 +76,7 @@ void board_to_string(char *s, float *board)
     for(i = 0; i < 91; ++i){
         for(j = 0; j < 4; ++j){
             int me = (board[count] == 1);
-            int you = (board[count] == -1);
+            int you = (board[count + 19*19] == 1);
             if (me) s[i] = s[i] | (1<<(2*j));
             if (you) s[i] = s[i] | (1<<(2*j + 1));
             ++count;
@@ -82,29 +85,40 @@ void board_to_string(char *s, float *board)
     }
 }
 
+static int occupied(float *b, int i)
+{
+    if (b[i]) return 1;
+    if (b[i+19*19]) return -1;
+    return 0;
+}
+
 data random_go_moves(moves m, int n)
 {
     data d = {0};
-    d.X = make_matrix(n, 19*19);
-    d.y = make_matrix(n, 19*19+1);
-    int i;
+    d.X = make_matrix(n, 19*19*3);
+    d.y = make_matrix(n, 19*19+2);
+    int i, j;
     for(i = 0; i < n; ++i){
         float *board = d.X.vals[i];
         float *label = d.y.vals[i];
         char *b = m.data[rand()%m.n];
-        int row = b[0];
-        int col = b[1];
+        int player = b[0] - '0';
+        int result = b[1] - '0';
+        int row = b[2];
+        int col = b[3];
+        string_to_board(b+4, board);
+        if(player > 0) for(j = 0; j < 19*19; ++j) board[19*19*2 + j] = 1;
+        label[19*19+1] = (player==result);
         if(row >= 19 || col >= 19){
             label[19*19] = 1;
         } else {
             label[col + 19*row] = 1;
-            string_to_board(b+2, board);
-            if(board[col + 19*row]) printf("hey\n");
+            if(occupied(board, col + 19*row)) printf("hey\n");
         }
 
         int flip = rand()%2;
         int rotate = rand()%4;
-        image in = float_to_image(19, 19, 1, board);
+        image in = float_to_image(19, 19, 3, board);
         image out = float_to_image(19, 19, 1, label);
         if(flip){
             flip_image(in);
@@ -149,18 +163,18 @@ void train_go(char *cfgfile, char *weightfile, char *filename, int *gpus, int ng
     printf("Moves: %d\n", N);
     int epoch = (*net->seen)/N;
     while(get_current_batch(net) < net->max_batches || net->max_batches == 0){
-        clock_t time=clock();
+        double time=what_time_is_it_now();
 
         data train = random_go_moves(m, net->batch*net->subdivisions*ngpus);
-        printf("Loaded: %lf seconds\n", sec(clock()-time));
-        time=clock();
+        printf("Loaded: %lf seconds\n", what_time_is_it_now() - time);
+        time=what_time_is_it_now();
 
         float loss = 0;
 #ifdef GPU
         if(ngpus == 1){
             loss = train_network(net, train);
         } else {
-            loss = train_networks(nets, ngpus, train, 4);
+            loss = train_networks(nets, ngpus, train, 10);
         }
 #else
         loss = train_network(net, train);
@@ -169,7 +183,7 @@ void train_go(char *cfgfile, char *weightfile, char *filename, int *gpus, int ng
 
         if(avg_loss == -1) avg_loss = loss;
         avg_loss = avg_loss*.95 + loss*.05;
-        printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, loss, avg_loss, get_current_rate(net), sec(clock()-time), *net->seen);
+        printf("%ld, %.3f: %f, %f avg, %f rate, %lf seconds, %ld images\n", get_current_batch(net), (float)(*net->seen)/N, loss, avg_loss, get_current_rate(net), what_time_is_it_now()-time, *net->seen);
         if(*net->seen/N > epoch){
             epoch = *net->seen/N;
             char buff[256];
@@ -195,11 +209,11 @@ void train_go(char *cfgfile, char *weightfile, char *filename, int *gpus, int ng
     free(base);
 }
 
-void propagate_liberty(float *board, int *lib, int *visited, int row, int col, int side)
+static void propagate_liberty(float *board, int *lib, int *visited, int row, int col, int side)
 {
     if (row < 0 || row > 18 || col < 0 || col > 18) return;
     int index = row*19 + col;
-    if (board[index] != side) return;
+    if (occupied(board,index) != side) return;
     if (visited[index]) return;
     visited[index] = 1;
     lib[index] += 1;
@@ -210,27 +224,27 @@ void propagate_liberty(float *board, int *lib, int *visited, int row, int col, i
 }
 
 
-int *calculate_liberties(float *board)
+static int *calculate_liberties(float *board)
 {
     int *lib = calloc(19*19, sizeof(int));
-    int visited[361];
+    int visited[19*19];
     int i, j;
     for(j = 0; j < 19; ++j){
         for(i = 0; i < 19; ++i){
             memset(visited, 0, 19*19*sizeof(int));
             int index = j*19 + i;
-            if(board[index] == 0){
-                if ((i > 0)  && board[index - 1]) propagate_liberty(board, lib, visited, j, i-1, board[index-1]);
-                if ((i < 18) && board[index + 1]) propagate_liberty(board, lib, visited, j, i+1, board[index+1]);
-                if ((j > 0)  && board[index - 19]) propagate_liberty(board, lib, visited, j-1, i, board[index-19]);
-                if ((j < 18) && board[index + 19]) propagate_liberty(board, lib, visited, j+1, i, board[index+19]);
+            if(!occupied(board,index)){
+                if ((i > 0)  && occupied(board,index - 1)) propagate_liberty(board, lib, visited, j, i-1, occupied(board,index-1));
+                if ((i < 18) && occupied(board,index + 1)) propagate_liberty(board, lib, visited, j, i+1, occupied(board,index+1));
+                if ((j > 0)  && occupied(board,index - 19)) propagate_liberty(board, lib, visited, j-1, i, occupied(board,index-19));
+                if ((j < 18) && occupied(board,index + 19)) propagate_liberty(board, lib, visited, j+1, i, occupied(board,index+19));
             }
         }
     }
     return lib;
 }
 
-void print_board(FILE *stream, float *board, int swap, int *indexes)
+void print_board(FILE *stream, float *board, int player, int *indexes)
 {
     int i,j,n;
     fprintf(stream, "   ");
@@ -254,20 +268,16 @@ void print_board(FILE *stream, float *board, int swap, int *indexes)
                            else if(n == 3) fprintf(stream, "\uff14");
                            else if(n == 4) fprintf(stream, "\uff15");
                          */
-                        if(n == 0) fprintf(stream, " 1");
-                        else if(n == 1) fprintf(stream, " 2");
-                        else if(n == 2) fprintf(stream, " 3");
-                        else if(n == 3) fprintf(stream, " 4");
-                        else if(n == 4) fprintf(stream, " 5");
+                        fprintf(stream, " %d", n+1);
                     }
                 }
                 if(found) continue;
             }
             //if(board[index]*-swap > 0) fprintf(stream, "\u25C9 ");
             //else if(board[index]*-swap < 0) fprintf(stream, "\u25EF ");
-            if(board[index]*-swap > 0) fprintf(stream, " O");
-            else if(board[index]*-swap < 0) fprintf(stream, " X");
-            else fprintf(stream, "  ");
+            if      (occupied(board, index) == player) fprintf(stream, " X");
+            else if (occupied(board, index) ==-player) fprintf(stream, " O");
+            else fprintf(stream, " .");
         }
         fprintf(stream, "\n");
     }
@@ -277,23 +287,28 @@ void flip_board(float *board)
 {
     int i;
     for(i = 0; i < 19*19; ++i){
-        board[i] = -board[i];
+        float swap = board[i];
+        board[i] = board[i+19*19];
+        board[i+19*19] = swap;
+        board[i+19*19*2] = 1-board[i+19*19*2];
     }
 }
 
-void predict_move(network *net, float *board, float *move, int multi)
+float predict_move2(network *net, float *board, float *move, int multi)
 {
     float *output = network_predict(net, board);
     copy_cpu(19*19+1, output, 1, move, 1);
+    float result = output[19*19 + 1];
     int i;
     if(multi){
-        image bim = float_to_image(19, 19, 1, board);
+        image bim = float_to_image(19, 19, 3, board);
         for(i = 1; i < 8; ++i){
             rotate_image_cw(bim, i);
             if(i >= 4) flip_image(bim);
 
             float *output = network_predict(net, board);
             image oim = float_to_image(19, 19, 1, output);
+            result += output[19*19 + 1];
 
             if(i >= 4) flip_image(oim);
             rotate_image_cw(oim, -i);
@@ -303,19 +318,22 @@ void predict_move(network *net, float *board, float *move, int multi)
             if(i >= 4) flip_image(bim);
             rotate_image_cw(bim, -i);
         }
+        result = result/8;
         scal_cpu(19*19+1, 1./8., move, 1);
     }
     for(i = 0; i < 19*19; ++i){
-        if(board[i]) move[i] = 0;
+        if(board[i] || board[i+19*19]) move[i] = 0;
     }
+    return result;
 }
 
-void remove_connected(float *b, int *lib, int p, int r, int c)
+static void remove_connected(float *b, int *lib, int p, int r, int c)
 {
     if (r < 0 || r >= 19 || c < 0 || c >= 19) return;
-    if (b[r*19 + c] != p) return;
+    if (occupied(b, r*19 + c) != p) return;
     if (lib[r*19 + c] != 1) return;
     b[r*19 + c] = 0;
+    b[19*19 + r*19 + c] = 0;
     remove_connected(b, lib, p, r+1, c);
     remove_connected(b, lib, p, r-1, c);
     remove_connected(b, lib, p, r, c+1);
@@ -326,7 +344,8 @@ void remove_connected(float *b, int *lib, int p, int r, int c)
 void move_go(float *b, int p, int r, int c)
 {
     int *l = calculate_liberties(b);
-    b[r*19 + c] = p;
+    if(p > 0) b[r*19 + c] = 1;
+    else b[19*19 + r*19 + c] = 1;
     remove_connected(b, l, -p, r+1, c);
     remove_connected(b, l, -p, r-1, c);
     remove_connected(b, l, -p, r, c+1);
@@ -334,13 +353,271 @@ void move_go(float *b, int p, int r, int c)
     free(l);
 }
 
-int makes_safe_go(float *b, int *lib, int p, int r, int c){
+int compare_board(float *a, float *b)
+{
+    if(memcmp(a, b, 19*19*3*sizeof(float)) == 0) return 1;
+    return 0;
+}
+
+typedef struct mcts_tree{
+    float *board;
+    struct mcts_tree **children;
+    float *prior;
+    int *visit_count;
+    float *value;
+    float *mean;
+    float *prob;
+    int total_count;
+    float result;
+    int done;
+    int pass;
+} mcts_tree;
+
+void free_mcts(mcts_tree *root)
+{
+    if(!root) return;
+    int i;
+    free(root->board);
+    for(i = 0; i < 19*19+1; ++i){
+        if(root->children[i]) free_mcts(root->children[i]);
+    }
+    free(root->children);
+    free(root->prior);
+    free(root->visit_count);
+    free(root->value);
+    free(root->mean);
+    free(root->prob);
+    free(root);
+}
+
+float *network_predict_rotations(network *net, float *next)
+{
+    int n = net->batch;
+    float *in = calloc(19*19*3*n, sizeof(float));
+    image im = float_to_image(19, 19, 3, next);
+    int i,j;
+    int *inds = random_index_order(0, 8);
+    for(j = 0; j < n; ++j){
+        i = inds[j];
+        rotate_image_cw(im, i);
+        if(i >= 4) flip_image(im);
+        memcpy(in + 19*19*3*j, im.data, 19*19*3*sizeof(float));
+        if(i >= 4) flip_image(im);
+        rotate_image_cw(im, -i);
+    }
+    float *pred = network_predict(net, in);
+    for(j = 0; j < n; ++j){
+        i = inds[j];
+        image im = float_to_image(19, 19, 1, pred + j*(19*19 + 2));
+        if(i >= 4) flip_image(im);
+        rotate_image_cw(im, -i);
+        if(j > 0){
+            axpy_cpu(19*19+2, 1, im.data, 1, pred, 1);
+        }
+    }
+    free(in);
+    free(inds);
+    scal_cpu(19*19+2, 1./n, pred, 1);
+    return pred;
+}
+
+mcts_tree *expand(float *next, float *ko, network *net)
+{
+    mcts_tree *root = calloc(1, sizeof(mcts_tree));
+    root->board = next;
+    root->children = calloc(19*19+1, sizeof(mcts_tree*));
+    root->prior = calloc(19*19 + 1, sizeof(float));
+    root->prob = calloc(19*19 + 1, sizeof(float));
+    root->mean = calloc(19*19 + 1, sizeof(float));
+    root->value = calloc(19*19 + 1, sizeof(float));
+    root->visit_count = calloc(19*19 + 1, sizeof(int));
+    root->total_count = 1;
+    int i;
+    float *pred = network_predict_rotations(net, next);
+    copy_cpu(19*19+1, pred, 1, root->prior, 1);
+    float val = 2*pred[19*19 + 1] - 1;
+    root->result = val;
+    for(i = 0; i < 19*19+1; ++i) {
+        root->visit_count[i] = 0;
+        root->value[i] = 0;
+        root->mean[i] = val;
+        if(i < 19*19 && occupied(next, i)){
+            root->value[i] = -1;
+            root->mean[i] = -1;
+            root->prior[i] = 0;
+        }
+    }
+    //print_board(stderr, next, flip?-1:1, 0);
+    return root;
+}
+
+float *copy_board(float *board)
+{
+    float *next = calloc(19*19*3, sizeof(float));
+    copy_cpu(19*19*3, board, 1, next, 1);
+    return next;
+}
+
+float select_mcts(mcts_tree *root, network *net, float *prev, float cpuct)
+{
+    if(root->done) return -root->result;
+    int i;
+    float max = -1000;
+    int max_i = 0;
+    for(i = 0; i < 19*19+1; ++i){
+        root->prob[i] = root->mean[i] + cpuct*root->prior[i] * sqrt(root->total_count) / (1. + root->visit_count[i]);
+        if(root->prob[i] > max){
+            max = root->prob[i];
+            max_i = i;
+        }
+    }
+    float val;
+    i = max_i;
+    root->visit_count[i]++;
+    root->total_count++;
+    if (root->children[i]) {
+        val = select_mcts(root->children[i], net, root->board, cpuct);
+    } else {
+        if(max_i < 19*19 && !legal_go(root->board, prev, 1, max_i/19, max_i%19)) {
+            root->mean[i]  = -1;
+            root->value[i] = -1;
+            root->prior[i] = 0;
+            --root->total_count;
+            return select_mcts(root, net, prev, cpuct);
+            //printf("Detected ko\n");
+            //getchar();
+        } else {
+            float *next = copy_board(root->board);
+            if (max_i < 19*19) {
+                move_go(next, 1, max_i / 19, max_i % 19);
+            }
+            flip_board(next);
+            root->children[i] = expand(next, root->board, net);
+            val = -root->children[i]->result;
+            if(max_i == 19*19){
+                root->children[i]->pass = 1;
+                if (root->pass){
+                    root->children[i]->done = 1;
+                }
+            }
+        }
+    }
+    root->value[i] += val;
+    root->mean[i] = root->value[i]/root->visit_count[i];
+    return -val;
+}
+
+mcts_tree *run_mcts(mcts_tree *tree, network *net, float *board, float *ko, int player, int n, float cpuct, float secs)
+{
+    int i;
+    double t = what_time_is_it_now();
+    if(player < 0) flip_board(board);
+    if(!tree) tree = expand(copy_board(board), ko, net);
+    assert(compare_board(tree->board, board));
+    for(i = 0; i < n; ++i){
+        if (secs > 0 && (what_time_is_it_now() - t) > secs) break;
+        int max_i = max_int_index(tree->visit_count, 19*19+1);
+        if (tree->visit_count[max_i] >= n) break;
+        select_mcts(tree, net, ko, cpuct);
+    }
+    if(player < 0) flip_board(board);
+    //fprintf(stderr, "%f Seconds\n", what_time_is_it_now() - t);
+    return tree;
+}
+
+mcts_tree *move_mcts(mcts_tree *tree, int index)
+{
+    if(index < 0 || index > 19*19 || !tree || !tree->children[index]) {
+        free_mcts(tree);
+        tree = 0;
+    } else {
+        mcts_tree *swap = tree;
+        tree = tree->children[index];
+        swap->children[index] = 0;
+        free_mcts(swap);
+    }
+    return tree;
+}
+
+typedef struct {
+    float value;
+    float mcts;
+    int row;
+    int col;
+} move;
+
+move pick_move(mcts_tree *tree, float temp, int player)
+{
+    int i;
+    float probs[19*19+1] = {0};
+    move m = {0};
+    double sum = 0;
+    /*
+    for(i = 0; i < 19*19+1; ++i){
+        probs[i] = tree->visit_count[i];
+    }
+    */
+    //softmax(probs, 19*19+1, temp, 1, probs);
+    for(i = 0; i < 19*19+1; ++i){
+        sum += pow(tree->visit_count[i], 1./temp);
+    }
+    for(i = 0; i < 19*19+1; ++i){
+        probs[i] = pow(tree->visit_count[i], 1./temp) / sum;
+    }
+
+    int index = sample_array(probs, 19*19+1);
+    m.row = index / 19;
+    m.col = index % 19;
+    m.value = (tree->result+1.)/2.;
+    m.mcts  = (tree->mean[index]+1.)/2.;
+
+    int indexes[nind];
+    top_k(probs, 19*19+1, nind, indexes);
+    print_board(stderr, tree->board, player, indexes);
+
+    fprintf(stderr, "%d %d, Result: %f, Prior: %f, Prob: %f, Mean Value: %f, Child Result: %f, Visited: %d\n", index/19, index%19, tree->result, tree->prior[index], probs[index], tree->mean[index], (tree->children[index])?tree->children[index]->result:0, tree->visit_count[index]);
+    int ind = max_index(probs, 19*19+1);
+    fprintf(stderr, "%d %d, Result: %f, Prior: %f, Prob: %f, Mean Value: %f, Child Result: %f, Visited: %d\n", ind/19, ind%19, tree->result, tree->prior[ind], probs[ind], tree->mean[ind], (tree->children[ind])?tree->children[ind]->result:0, tree->visit_count[ind]);
+    ind = max_index(tree->prior, 19*19+1);
+    fprintf(stderr, "%d %d, Result: %f, Prior: %f, Prob: %f, Mean Value: %f, Child Result: %f, Visited: %d\n", ind/19, ind%19, tree->result, tree->prior[ind], probs[ind], tree->mean[ind], (tree->children[ind])?tree->children[ind]->result:0, tree->visit_count[ind]);
+    return m;
+}
+
+/*
+   float predict_move(network *net, float *board, float *move, int multi, float *ko, float temp)
+   {
+
+   int i;
+
+   int max_v = 0;
+   int max_i = 0;
+   for(i = 0; i < 19*19+1; ++i){
+   if(root->visit_count[i] > max_v){
+   max_v = root->visit_count[i];
+   max_i = i;
+   }
+   }
+   fprintf(stderr, "%f Seconds\n", what_time_is_it_now() - t);
+   int ind = max_index(root->mean, 19*19+1);
+   fprintf(stderr, "%d %d, Result: %f, Prior: %f, Prob: %f, Mean Value: %f, Child Result: %f, Visited: %d\n", max_i/19, max_i%19, root->result, root->prior[max_i], root->prob[max_i], root->mean[max_i], (root->children[max_i])?root->children[max_i]->result:0, root->visit_count[max_i]);
+   fprintf(stderr, "%d %d, Result: %f, Prior: %f, Prob: %f, Mean Value: %f, Child Result: %f, Visited: %d\n", ind/19, ind%19, root->result, root->prior[ind], root->prob[ind], root->mean[ind], (root->children[ind])?root->children[ind]->result:0, root->visit_count[ind]);
+   ind = max_index(root->prior, 19*19+1);
+   fprintf(stderr, "%d %d, Result: %f, Prior: %f, Prob: %f, Mean Value: %f, Child Result: %f, Visited: %d\n", ind/19, ind%19, root->result, root->prior[ind], root->prob[ind], root->mean[ind], (root->children[ind])?root->children[ind]->result:0, root->visit_count[ind]);
+   if(root->result < -.9 && root->mean[max_i] < -.9) return -1000.f;
+
+   float val = root->result;
+   free_mcts(root);
+   return val;
+   }
+ */
+
+static int makes_safe_go(float *b, int *lib, int p, int r, int c){
     if (r < 0 || r >= 19 || c < 0 || c >= 19) return 0;
-    if (b[r*19 + c] == -p){
+    if (occupied(b,r*19 + c) == -p){
         if (lib[r*19 + c] > 1) return 0;
         else return 1;
     }
-    if (b[r*19 + c] == 0) return 1;
+    if (!occupied(b,r*19 + c)) return 1;
     if (lib[r*19 + c] > 1) return 1;
     return 0;
 }
@@ -357,82 +634,101 @@ int suicide_go(float *b, int p, int r, int c)
     return !safe;
 }
 
-int legal_go(float *b, char *ko, int p, int r, int c)
+int check_ko(float *x, float *ko)
 {
-    if (b[r*19 + c]) return 0;
-    char curr[91];
-    char next[91];
-    board_to_string(curr, b);
-    move_go(b, p, r, c);
-    board_to_string(next, b);
-    string_to_board(curr, b);
-    if(memcmp(next, ko, 91) == 0) return 0;
+    if(!ko) return 0;
+    float curr[19*19*3];
+    copy_cpu(19*19*3, x, 1, curr, 1);
+    if(curr[19*19*2] != ko[19*19*2]) flip_board(curr);
+    if(compare_board(curr, ko)) return 1;
+    return 0;
+}
+
+int legal_go(float *b, float *ko, int p, int r, int c)
+{
+    if (occupied(b, r*19+c)) return 0;
+    float curr[19*19*3];
+    copy_cpu(19*19*3, b, 1, curr, 1);
+    move_go(curr, p, r, c);
+    if(check_ko(curr, ko)) return 0;
+    if(suicide_go(b, p, r, c)) return 0;
     return 1;
 }
 
-int generate_move(network *net, int player, float *board, int multi, float thresh, float temp, char *ko, int print)
-{
-    int i, j;
-    int empty = 1;
-    for(i = 0; i < 19*19; ++i){
-        if (board[i]) {
-            empty = 0;
-            break;
-        }
-    }
-    if(empty) {
-        return 72;
-    }
-    for(i = 0; i < net->n; ++i) net->layers[i].temperature = temp;
-
-    float move[362];
-    if (player < 0) flip_board(board);
-    predict_move(net, board, move, multi);
-    if (player < 0) flip_board(board);
-
-
-    for(i = 0; i < 19; ++i){
-        for(j = 0; j < 19; ++j){
-            if (!legal_go(board, ko, player, i, j)) move[i*19 + j] = 0;
-        }
-    }
-
-    int indexes[nind];
-    top_k(move, 19*19+1, nind, indexes);
-    if(thresh > move[indexes[0]]) thresh = move[indexes[nind-1]];
-
-    for(i = 0; i < 19*19+1; ++i){
-        if (move[i] < thresh) move[i] = 0;
-    }
-
-
-    int max = max_index(move, 19*19+1);
-    int row = max / 19;
-    int col = max % 19;
-    int index = sample_array(move, 19*19+1);
-
-    if(print){
-        top_k(move, 19*19+1, nind, indexes);
-        for(i = 0; i < nind; ++i){
-            if (!move[indexes[i]]) indexes[i] = -1;
-        }
-        print_board(stderr, board, player, indexes);
-        for(i = 0; i < nind; ++i){
-            fprintf(stderr, "%d: %f\n", i+1, move[indexes[i]]);
-        }
-    }
-    if (row == 19) return -1;
-
-    if (suicide_go(board, player, row, col)){
-        return -1; 
-    }
-
-    if (suicide_go(board, player, index/19, index%19)){
-        index = max;
-    }
-    if (index == 19*19) return -1;
-    return index;
+/*
+   move generate_move(mcts_tree *root, network *net, int player, float *board, int multi, float temp, float *ko, int print)
+   {
+   move m = {0};
+//root = run_mcts(tree, network *net, float *board, float *ko, int n, float cpuct)
+int i, j;
+int empty = 1;
+for(i = 0; i < 19*19; ++i){
+if (occupied(board, i)) {
+empty = 0;
+break;
 }
+}
+if(empty) {
+m.value = .5;
+m.mcts = .5;
+m.row = 3;
+m.col = 15;
+return m;
+}
+
+float move[362];
+if (player < 0) flip_board(board);
+float result = predict_move(net, board, move, multi, ko, temp);
+if (player < 0) flip_board(board);
+if(result == -1000.f) return -2;
+
+for(i = 0; i < 19; ++i){
+for(j = 0; j < 19; ++j){
+if (!legal_go(board, ko, player, i, j)) move[i*19 + j] = 0;
+}
+}
+
+int indexes[nind];
+top_k(move, 19*19+1, nind, indexes);
+
+
+int max = max_index(move, 19*19+1);
+int row = max / 19;
+int col = max % 19;
+int index = sample_array(move, 19*19+1);
+
+if(print){
+top_k(move, 19*19+1, nind, indexes);
+for(i = 0; i < nind; ++i){
+if (!move[indexes[i]]) indexes[i] = -1;
+}
+print_board(stderr, board, 1, indexes);
+fprintf(stderr, "%s To Move\n", player > 0 ? "X" : "O");
+fprintf(stderr, "%.2f%% Win Chance\n", (result+1)/2*100);
+for(i = 0; i < nind; ++i){
+int index = indexes[i];
+int row = index / 19;
+int col = index % 19;
+if(row == 19){
+fprintf(stderr, "%d: Pass, %.2f%%\n", i+1, move[index]*100);
+} else {
+fprintf(stderr, "%d: %c %d, %.2f%%\n", i+1, col + 'A' + 1*(col > 7 && noi), (inverted)?19 - row : row+1, move[index]*100);
+}
+}
+}
+if (row == 19) return -1;
+
+if (suicide_go(board, player, row, col)){
+return -1; 
+}
+
+if (suicide_go(board, player, index/19, index%19)){
+index = max;
+}
+if (index == 19*19) return -1;
+return index;
+}
+*/
 
 void valid_go(char *cfgfile, char *weightfile, int multi, char *filename)
 {
@@ -443,22 +739,25 @@ void valid_go(char *cfgfile, char *weightfile, int multi, char *filename)
     set_batch_network(net, 1);
     printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net->learning_rate, net->momentum, net->decay);
 
-    float *board = calloc(19*19, sizeof(float));
-    float *move = calloc(19*19+1, sizeof(float));
-   // moves m = load_go_moves("/home/pjreddie/backup/go.test");
+    float *board = calloc(19*19*3, sizeof(float));
+    float *move = calloc(19*19+2, sizeof(float));
+    // moves m = load_go_moves("/home/pjreddie/backup/go.test");
     moves m = load_go_moves(filename);
 
     int N = m.n;
-    int i;
+    int i,j;
     int correct = 0;
-    for(i = 0; i <N; ++i){
+    for (i = 0; i <N; ++i) {
         char *b = m.data[i];
-        int row = b[0];
-        int col = b[1];
+        int player = b[0] - '0';
+        //int result = b[1] - '0';
+        int row = b[2];
+        int col = b[3];
         int truth = col + 19*row;
-        string_to_board(b+2, board);
-        predict_move(net, board, move, multi);
-        int index = max_index(move, 19*19);
+        string_to_board(b+4, board);
+        if(player > 0) for(j = 0; j < 19*19; ++j) board[19*19*2 + j] = 1;
+        predict_move2(net, board, move, multi);
+        int index = max_index(move, 19*19+1);
         if(index == truth) ++correct;
         printf("%d Accuracy %f\n", i, (float) correct/(i+1));
     }
@@ -473,29 +772,78 @@ int print_game(float *board, FILE *fp)
     fprintf(fp, "clear_board\n");
     for(j = 0; j < 19; ++j){
         for(i = 0; i < 19; ++i){
-            if(board[j*19 + i] == 1) fprintf(fp, "play black %c%d\n", 'A'+i+(i>=8), 19-j);
-            if(board[j*19 + i] == -1) fprintf(fp, "play white %c%d\n", 'A'+i+(i>=8), 19-j);
-            if(board[j*19 + i]) ++count;
+            if(occupied(board,j*19 + i) == 1) fprintf(fp, "play black %c%d\n", 'A'+i+(i>=8), 19-j);
+            if(occupied(board,j*19 + i) == -1) fprintf(fp, "play white %c%d\n", 'A'+i+(i>=8), 19-j);
+            if(occupied(board,j*19 + i)) ++count;
         }
     }
     return count;
 }
 
-void engine_go(char *filename, char *weightfile, int multi)
+
+int stdin_ready()
 {
+    fd_set readfds;
+    FD_ZERO(&readfds);
+
+    struct timeval timeout;
+    timeout.tv_sec = 0;
+    timeout.tv_usec = 0;
+    FD_SET(STDIN_FILENO, &readfds);
+
+    if (select(1, &readfds, NULL, NULL, &timeout)){
+        return 1;
+    }
+    return 0;
+}
+
+mcts_tree *ponder(mcts_tree *tree, network *net, float *b, float *ko, int player, float cpuct)
+{
+    double t = what_time_is_it_now();
+    int count = 0;
+    if (tree) count = tree->total_count;
+    while(!stdin_ready()){
+        if (what_time_is_it_now() - t > 120) break;
+        tree = run_mcts(tree, net, b, ko, player, 100000, cpuct, .1);
+    }
+    fprintf(stderr, "Pondered %d moves...\n", tree->total_count - count);
+    return tree;
+}
+
+void engine_go(char *filename, char *weightfile, int mcts_iters, float secs, float temp, float cpuct, int anon, int resign)
+{
+    mcts_tree *root = 0;
     network *net = load_network(filename, weightfile, 0);
     set_batch_network(net, 1);
     srand(time(0));
-    float *board = calloc(19*19, sizeof(float));
-    char *one = calloc(91, sizeof(char));
-    char *two = calloc(91, sizeof(char));
+    float *board = calloc(19*19*3, sizeof(float));
+    flip_board(board);
+    float *one = calloc(19*19*3, sizeof(float));
+    float *two = calloc(19*19*3, sizeof(float));
+    int ponder_player = 0;
     int passed = 0;
+    int move_num = 0;
+    int main_time = 0;
+    int byo_yomi_time = 0;
+    int byo_yomi_stones = 0;
+    int black_time_left = 0;
+    int black_stones_left = 0;
+    int white_time_left = 0;
+    int white_stones_left = 0;
+    float orig_time = secs;
+    int old_ponder = 0;
     while(1){
+        if(ponder_player){
+            root = ponder(root, net, board, two, ponder_player, cpuct);
+        }
+        old_ponder = ponder_player;
+        ponder_player = 0;
         char buff[256];
         int id = 0;
         int has_id = (scanf("%d", &id) == 1);
         scanf("%s", buff);
         if (feof(stdin)) break;
+        fprintf(stderr, "%s\n", buff);
         char ids[256];
         sprintf(ids, "%d", id);
         //fprintf(stderr, "%s\n", buff);
@@ -503,13 +851,34 @@ void engine_go(char *filename, char *weightfile, int multi)
         if (!strcmp(buff, "protocol_version")){
             printf("=%s 2\n\n", ids);
         } else if (!strcmp(buff, "name")){
-            printf("=%s DarkGo\n\n", ids);
-        } else if (!strcmp(buff, "time_settings") || !strcmp(buff, "time_left")){
-            char *line = fgetl(stdin);
-            free(line);
+            if(anon){
+                printf("=%s The Fool!\n\n", ids);
+            }else{
+                printf("=%s DarkGo\n\n", ids);
+            }
+        } else if (!strcmp(buff, "time_settings")){
+            ponder_player = old_ponder;
+            scanf("%d %d %d", &main_time, &byo_yomi_time, &byo_yomi_stones);
+            printf("=%s \n\n", ids);
+        } else if (!strcmp(buff, "time_left")){
+            ponder_player = old_ponder;
+            char color[256];
+            int time = 0, stones = 0;
+            scanf("%s %d %d", color, &time, &stones);
+            if (color[0] == 'b' || color[0] == 'B'){
+                black_time_left = time;
+                black_stones_left = stones;
+            } else {
+                white_time_left = time;
+                white_stones_left = stones;
+            }
             printf("=%s \n\n", ids);
         } else if (!strcmp(buff, "version")){
-            printf("=%s 1.0. Want more DarkGo? You can find me on OGS, unlimited games, no waiting! https://online-go.com/user/view/434218\n\n", ids);
+            if(anon){
+                printf("=%s :-DDDD\n\n", ids);
+            }else {
+                printf("=%s 1.0. Want more DarkGo? You can find me on OGS, unlimited games, no waiting! https://online-go.com/user/view/434218\n\n", ids);
+            }
         } else if (!strcmp(buff, "known_command")){
             char comm[256];
             scanf("%s", comm);
@@ -541,7 +910,10 @@ void engine_go(char *filename, char *weightfile, int multi)
             if(boardsize != 19){
                 printf("?%s unacceptable size\n\n", ids);
             } else {
-                memset(board, 0, 19*19*sizeof(float));
+                root = move_mcts(root, -1);
+                memset(board, 0, 3*19*19*sizeof(float));
+                flip_board(board);
+                move_num = 0;
                 printf("=%s \n\n", ids);
             }
         } else if (!strcmp(buff, "fixed_handicap")){
@@ -551,10 +923,15 @@ void engine_go(char *filename, char *weightfile, int multi)
             int i;
             for(i = 0; i < handicap; ++i){
                 board[indexes[i]] = 1;   
+                ++move_num;
             }
+            root = move_mcts(root, -1);
         } else if (!strcmp(buff, "clear_board")){
             passed = 0;
-            memset(board, 0, 19*19*sizeof(float));
+            memset(board, 0, 3*19*19*sizeof(float));
+            flip_board(board);
+            move_num = 0;
+            root = move_mcts(root, -1);
             printf("=%s \n\n", ids);
         } else if (!strcmp(buff, "komi")){
             float komi = 0;
@@ -565,6 +942,7 @@ void engine_go(char *filename, char *weightfile, int multi)
             print_board(stdout, board, 1, 0);
             printf("\n");
         } else if (!strcmp(buff, "play") || !strcmp(buff, "black") || !strcmp(buff, "white")){
+            ++move_num;
             char color[256];
             if(!strcmp(buff, "play"))
             {
@@ -584,6 +962,7 @@ void engine_go(char *filename, char *weightfile, int multi)
                 free(line);
                 fflush(stdout);
                 fflush(stderr);
+                root = move_mcts(root, 19*19);
                 continue;
             } else {
                 passed = 0;
@@ -594,15 +973,22 @@ void engine_go(char *filename, char *weightfile, int multi)
             r = 19 - r;
             fprintf(stderr, "move: %d %d\n", r, c);
 
-            char *swap = two;
+            float *swap = two;
             two = one;
             one = swap;
             move_go(board, player, r, c);
-            board_to_string(one, board);
+            copy_cpu(19*19*3, board, 1, one, 1);
+            if(root) fprintf(stderr, "Prior: %f\n", root->prior[r*19 + c]);
+            if(root) fprintf(stderr, "Mean: %f\n", root->mean[r*19 + c]);
+            if(root) fprintf(stderr, "Result: %f\n", root->result);
+            root = move_mcts(root, r*19 + c);
+            if(root) fprintf(stderr, "Visited: %d\n", root->total_count);
+            else fprintf(stderr, "NOT VISITED\n");
 
             printf("=%s \n\n", ids);
             //print_board(stderr, board, 1, 0);
         } else if (!strcmp(buff, "genmove") || !strcmp(buff, "genmove_black") || !strcmp(buff, "genmove_white")){
+            ++move_num;
             int player = 0;
             if(!strcmp(buff, "genmove")){
                 char color[256];
@@ -613,25 +999,41 @@ void engine_go(char *filename, char *weightfile, int multi)
             } else {
                 player = -1;
             }
+            if(player > 0){
+                if(black_time_left <= 30) secs = 2.5;
+                else secs = orig_time;
+            } else {
+                if(white_time_left <= 30) secs = 2.5;
+                else secs = orig_time;
+            }
+            ponder_player = -player;
 
-            int index = generate_move(net, player, board, multi, .4, 1, two, 0);
-            if(passed || index < 0){
+            //tree = generate_move(net, player, board, multi, .1, two, 1);
+            double t = what_time_is_it_now();
+            root = run_mcts(root, net, board, two, player, mcts_iters, cpuct, secs);
+            fprintf(stderr, "%f Seconds\n", what_time_is_it_now() - t);
+            move m = pick_move(root, temp, player);
+            root = move_mcts(root, m.row*19 + m.col);
+
+
+            if(move_num > resign && m.value < .1 && m.mcts < .1){
+                printf("=%s resign\n\n", ids);
+            } else if(m.row == 19){
                 printf("=%s pass\n\n", ids);
                 passed = 0;
             } else {
-                int row = index / 19;
-                int col = index % 19;
+                int row = m.row;
+                int col = m.col;
 
-                char *swap = two;
+                float *swap = two;
                 two = one;
                 one = swap;
 
                 move_go(board, player, row, col);
-                board_to_string(one, board);
+                copy_cpu(19*19*3, board, 1, one, 1);
                 row = 19 - row;
                 if (col >= 8) ++col;
                 printf("=%s %c%d\n\n", ids, 'A' + col, row);
-                //print_board(board, 1, 0);
             }
 
         } else if (!strcmp(buff, "p")){
@@ -661,6 +1063,27 @@ void engine_go(char *filename, char *weightfile, int multi)
             } else {
                 printf("?%s unknown command\n\n", ids);
             }
+        } else if (!strcmp(buff, "kgs-genmove_cleanup")){
+            char type[256];
+            scanf("%s", type);
+            fprintf(stderr, "kgs-genmove_cleanup\n");
+            char *line = fgetl(stdin);
+            free(line);
+            int i;
+            FILE *f = fopen("game.txt", "w");
+            int count = print_game(board, f);
+            fprintf(f, "%s kgs-genmove_cleanup %s\n", ids, type);
+            fclose(f);
+            FILE *p = popen("./gnugo --mode gtp < game.txt", "r");
+            for(i = 0; i < count; ++i){
+                free(fgetl(p));
+                free(fgetl(p));
+            }
+            char *l = 0;
+            while((l = fgetl(p))){
+                printf("%s\n", l);
+                free(l);
+            }
         } else {
             char *line = fgetl(stdin);
             free(line);
@@ -669,19 +1092,22 @@ void engine_go(char *filename, char *weightfile, int multi)
         fflush(stdout);
         fflush(stderr);
     }
+    printf("%d %d %d\n",passed, black_stones_left, white_stones_left);
 }
 
 void test_go(char *cfg, char *weights, int multi)
 {
+    int i;
     network *net = load_network(cfg, weights, 0);
     set_batch_network(net, 1);
     srand(time(0));
-    float *board = calloc(19*19, sizeof(float));
+    float *board = calloc(19*19*3, sizeof(float));
+    flip_board(board);
     float *move = calloc(19*19+1, sizeof(float));
     int color = 1;
     while(1){
-        int i;
-        predict_move(net, board, move, multi);
+        float result = predict_move2(net, board, move, multi);
+        printf("%.2f%% Win Chance\n", (result+1)/2*100);
 
         int indexes[nind];
         int row, col;
@@ -732,14 +1158,24 @@ void test_go(char *cfg, char *weights, int multi)
                 row = (inverted)?19 - row : row-1;
                 col = c - 'A';
                 if (col > 7 && noi) col -= 1;
-                if (num == 3) board[row*19 + col] = (g == 'b') ? color : -color;
+                if (num == 3) {
+                    int mc = (g == 'b') ? 1 : -1;
+                    if (mc == color) {
+                        board[row*19 + col] = 1;
+                    } else {
+                        board[19*19 + row*19 + col] = 1;
+                    }
+                }
             } else if(c == 'c'){
                 char g;
                 int num = sscanf(line, "%c %c %d", &g, &c, &row);
                 row = (inverted)?19 - row : row-1;
                 col = c - 'A';
                 if (col > 7 && noi) col -= 1;
-                if (num == 3) board[row*19 + col] = 0;
+                if (num == 3) {
+                    board[row*19 + col] = 0;
+                    board[19*19 + row*19 + col] = 0;
+                }
             }
         }
         free(line);
@@ -776,8 +1212,10 @@ float score_game(float *board)
 
 void self_go(char *filename, char *weightfile, char *f2, char *w2, int multi)
 {
+    mcts_tree *tree1 = 0;
+    mcts_tree *tree2 = 0;
     network *net = load_network(filename, weightfile, 0);
-    set_batch_network(net, 1);
+    //set_batch_network(net, 1);
 
     network *net2;
     if (f2) {
@@ -792,18 +1230,24 @@ void self_go(char *filename, char *weightfile, char *f2, char *w2, int multi)
     srand(time(0));
     char boards[600][93];
     int count = 0;
-    set_batch_network(net, 1);
-    set_batch_network(net2, 1);
-    float *board = calloc(19*19, sizeof(float));
-    char *one = calloc(91, sizeof(char));
-    char *two = calloc(91, sizeof(char));
+    //set_batch_network(net, 1);
+    //set_batch_network(net2, 1);
+    float *board = calloc(19*19*3, sizeof(float));
+    flip_board(board);
+    float *one = calloc(19*19*3, sizeof(float));
+    float *two = calloc(19*19*3, sizeof(float));
     int done = 0;
     int player = 1;
     int p1 = 0;
     int p2 = 0;
     int total = 0;
+    float temp = .1;
+    int mcts_iters = 500;
+    float cpuct = 5;
     while(1){
         if (done){
+            tree1 = move_mcts(tree1, -1);
+            tree2 = move_mcts(tree2, -1);
             float score = score_game(board);
             if((score > 0) == (total%2==0)) ++p1;
             else ++p2;
@@ -820,25 +1264,42 @@ void self_go(char *filename, char *weightfile, char *f2, char *w2, int multi)
                printf("\n");
                }
              */
-            memset(board, 0, 19*19*sizeof(float));
+            memset(board, 0, 3*19*19*sizeof(float));
+            flip_board(board);
             player = 1;
             done = 0;
             count = 0;
             fflush(stdout);
             fflush(stderr);
         }
-        print_board(stderr, board, 1, 0);
+        //print_board(stderr, board, 1, 0);
         //sleep(1);
+
+        if ((total%2==0) == (player==1)){
+            //mcts_iters = 4500;   
+            cpuct = 5;
+        } else {
+            //mcts_iters = 500;
+            cpuct = 1;
+        }
         network *use = ((total%2==0) == (player==1)) ? net : net2;
-        int index = generate_move(use, player, board, multi, .4, 1, two, 0);
-        if(index < 0){
+        mcts_tree *t = ((total%2==0) == (player==1)) ? tree1 : tree2;
+        t = run_mcts(t, use, board, two, player, mcts_iters, cpuct, 0);
+        move m = pick_move(t, temp, player);
+        if(((total%2==0) == (player==1))) tree1 = t;
+        else tree2 = t;
+
+        tree1 = move_mcts(tree1, m.row*19 + m.col);
+        tree2 = move_mcts(tree2, m.row*19 + m.col);
+
+        if(m.row == 19){
             done = 1;
             continue;
         }
-        int row = index / 19;
-        int col = index % 19;
+        int row = m.row;
+        int col = m.col;
 
-        char *swap = two;
+        float *swap = two;
         two = one;
         one = swap;
 
@@ -850,7 +1311,7 @@ void self_go(char *filename, char *weightfile, char *f2, char *w2, int multi)
         ++count;
 
         move_go(board, player, row, col);
-        board_to_string(one, board);
+        copy_cpu(19*19*3, board, 1, one, 1);
 
         player = -player;
     }
@@ -893,11 +1354,17 @@ void run_go(int argc, char **argv)
     char *c2 = (argc > 5) ? argv[5] : 0;
     char *w2 = (argc > 6) ? argv[6] : 0;
     int multi = find_arg(argc, argv, "-multi");
+    int anon = find_arg(argc, argv, "-anon");
+    int iters = find_int_arg(argc, argv, "-iters", 500);
+    int resign = find_int_arg(argc, argv, "-resign", 175);
+    float cpuct = find_float_arg(argc, argv, "-cpuct", 5);
+    float temp = find_float_arg(argc, argv, "-temp", .1);
+    float time = find_float_arg(argc, argv, "-time", 0);
     if(0==strcmp(argv[2], "train")) train_go(cfg, weights, c2, gpus, ngpus, clear);
     else if(0==strcmp(argv[2], "valid")) valid_go(cfg, weights, multi, c2);
     else if(0==strcmp(argv[2], "self")) self_go(cfg, weights, c2, w2, multi);
     else if(0==strcmp(argv[2], "test")) test_go(cfg, weights, multi);
-    else if(0==strcmp(argv[2], "engine")) engine_go(cfg, weights, multi);
+    else if(0==strcmp(argv[2], "engine")) engine_go(cfg, weights, iters, time, temp, cpuct, anon, resign);
 }
 
 
diff --git a/include/darknet.h b/include/darknet.h
index b3dc89a4..5fa2ec17 100644
--- a/include/darknet.h
+++ b/include/darknet.h
@@ -56,6 +56,10 @@ typedef enum{
     LOGISTIC, RELU, RELIE, LINEAR, RAMP, TANH, PLSE, LEAKY, ELU, LOGGY, STAIR, HARDTAN, LHTAN
 } ACTIVATION;
 
+typedef enum{
+    MULT, ADD, SUB, DIV
+} BINARY_ACTIVATION;
+
 typedef enum {
     CONVOLUTIONAL,
     DECONVOLUTIONAL,
@@ -578,6 +582,8 @@ list *read_data_cfg(char *filename);
 list *read_cfg(char *filename);
 unsigned char *read_file(char *filename);
 data resize_data(data orig, int w, int h);
+data *tile_data(data orig, int divs, int size);
+data select_data(data *orig, int *inds);
 
 void forward_network(network *net);
 void backward_network(network *net);
@@ -588,6 +594,7 @@ void axpy_cpu(int N, float ALPHA, float *X, int INCX, float *Y, int INCY);
 void copy_cpu(int N, float *X, int INCX, float *Y, int INCY);
 void scal_cpu(int N, float ALPHA, float *X, int INCX);
 void normalize_cpu(float *x, float *mean, float *variance, int batch, int filters, int spatial);
+void softmax(float *input, int n, float temp, int stride, float *output);
 
 int best_3d_shift_r(image a, image b, int min, int max);
 #ifdef GPU
@@ -744,12 +751,15 @@ void top_k(float *a, int n, int k, int *index);
 int *read_map(char *filename);
 void error(const char *s);
 int max_index(float *a, int n);
+int max_int_index(int *a, int n);
 int sample_array(float *a, int n);
+int *random_index_order(int min, int max);
 void free_list(list *l);
 float mse_array(float *a, int n);
 float variance_array(float *a, int n);
 float mag_array(float *a, int n);
 float mean_array(float *a, int n);
+float sum_array(float *a, int n);
 void normalize_array(float *a, int n);
 int *read_intlist(char *s, int *n, int d);
 size_t rand_size_t();
diff --git a/python/darknet.py b/python/darknet.py
index c8ec9be5..d6dfd31f 100644
--- a/python/darknet.py
+++ b/python/darknet.py
@@ -31,6 +31,8 @@ class METADATA(Structure):
     _fields_ = [("classes", c_int),
                 ("names", POINTER(c_char_p))]
 
+    
+
 #lib = CDLL("/home/pjreddie/documents/darknet/libdarknet.so", RTLD_GLOBAL)
 lib = CDLL("libdarknet.so", RTLD_GLOBAL)
 lib.network_width.argtypes = [c_void_p]
@@ -42,6 +44,10 @@ predict = lib.network_predict
 predict.argtypes = [c_void_p, POINTER(c_float)]
 predict.restype = POINTER(c_float)
 
+make_image = lib.make_image
+make_image.argtypes = [c_int, c_int, c_int]
+make_image.restype = IMAGE
+
 make_boxes = lib.make_boxes
 make_boxes.argtypes = [c_void_p]
 make_boxes.restype = POINTER(BOX)
@@ -82,6 +88,9 @@ load_image = lib.load_image_color
 load_image.argtypes = [c_char_p, c_int, c_int]
 load_image.restype = IMAGE
 
+rgbgr_image = lib.rgbgr_image
+rgbgr_image.argtypes = [IMAGE]
+
 predict_image = lib.network_predict_image
 predict_image.argtypes = [c_void_p, IMAGE]
 predict_image.restype = POINTER(c_float)
diff --git a/src/activation_kernels.cu b/src/activation_kernels.cu
index 80a849f7..5852eb58 100644
--- a/src/activation_kernels.cu
+++ b/src/activation_kernels.cu
@@ -140,6 +140,41 @@ __device__ float gradient_kernel(float x, ACTIVATION a)
     return 0;
 }
 
+__global__ void binary_gradient_array_kernel(float *x, float *dy, int n, int s, BINARY_ACTIVATION a, float *dx)
+{
+    int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    int i = id % s;
+    int b = id / s;
+    float x1 = x[b*s + i];
+    float x2 = x[b*s + s/2 + i];
+    if(id < n) {
+        float de = dy[id];
+        dx[b*s + i] = x2*de;
+        dx[b*s + s/2 + i] = x1*de; 
+    }
+}
+
+extern "C" void binary_gradient_array_gpu(float *x, float *dx, int n, int size, BINARY_ACTIVATION a, float *y) 
+{
+    binary_gradient_array_kernel<<<cuda_gridsize(n/2), BLOCK>>>(x, dx, n/2, size, a, y);
+    check_error(cudaPeekAtLastError());
+}
+__global__ void binary_activate_array_kernel(float *x, int n, int s, BINARY_ACTIVATION a, float *y)
+{
+    int id = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
+    int i = id % s;
+    int b = id / s;
+    float x1 = x[b*s + i];
+    float x2 = x[b*s + s/2 + i];
+    if(id < n) y[id] = x1*x2;
+}
+
+extern "C" void binary_activate_array_gpu(float *x, int n, int size, BINARY_ACTIVATION a, float *y) 
+{
+    binary_activate_array_kernel<<<cuda_gridsize(n/2), BLOCK>>>(x, n/2, size, a, y);
+    check_error(cudaPeekAtLastError());
+}
+
 __global__ void activate_array_kernel(float *x, int n, ACTIVATION a)
 {
     int i = (blockIdx.x + blockIdx.y*gridDim.x) * blockDim.x + threadIdx.x;
diff --git a/src/data.c b/src/data.c
index cddc7bc1..935e6389 100644
--- a/src/data.c
+++ b/src/data.c
@@ -1172,6 +1172,56 @@ data load_data_regression(char **paths, int n, int m, int min, int max, int size
     return d;
 }
 
+data select_data(data *orig, int *inds)
+{
+    data d = {0};
+    d.shallow = 1;
+    d.w = orig[0].w;
+    d.h = orig[0].h;
+
+    d.X.rows = orig[0].X.rows;
+    d.y.rows = orig[0].X.rows;
+
+    d.X.cols = orig[0].X.cols;
+    d.y.cols = orig[0].y.cols;
+
+    d.X.vals = calloc(orig[0].X.rows, sizeof(float *));
+    d.y.vals = calloc(orig[0].y.rows, sizeof(float *));
+    int i;
+    for(i = 0; i < d.X.rows; ++i){
+        d.X.vals[i] = orig[inds[i]].X.vals[i];
+        d.y.vals[i] = orig[inds[i]].y.vals[i];
+    }
+    return d;
+}
+
+data *tile_data(data orig, int divs, int size)
+{
+    data *ds = calloc(divs*divs, sizeof(data));
+    int i, j;
+    #pragma omp parallel for
+    for(i = 0; i < divs*divs; ++i){
+        data d;
+        d.shallow = 0;
+        d.w = orig.w/divs * size;
+        d.h = orig.h/divs * size;
+        d.X.rows = orig.X.rows;
+        d.X.cols = d.w*d.h*3;
+        d.X.vals = calloc(d.X.rows, sizeof(float*));
+
+        d.y = copy_matrix(orig.y);
+        #pragma omp parallel for
+        for(j = 0; j < orig.X.rows; ++j){
+            int x = (i%divs) * orig.w / divs - (d.w - orig.w/divs)/2;
+            int y = (i/divs) * orig.h / divs - (d.h - orig.h/divs)/2;
+            image im = float_to_image(orig.w, orig.h, 3, orig.X.vals[j]);
+            d.X.vals[j] = crop_image(im, x, y, d.w, d.h).data;
+        }
+        ds[i] = d;
+    }
+    return ds;
+}
+
 data resize_data(data orig, int w, int h)
 {
     data d = {0};
@@ -1181,9 +1231,10 @@ data resize_data(data orig, int w, int h)
     int i;
     d.X.rows = orig.X.rows;
     d.X.cols = w*h*3;
-    d.X.vals = calloc(d.X.rows, sizeof(float));
+    d.X.vals = calloc(d.X.rows, sizeof(float*));
 
     d.y = copy_matrix(orig.y);
+    #pragma omp parallel for
     for(i = 0; i < orig.X.rows; ++i){
         image im = float_to_image(orig.w, orig.h, 3, orig.X.vals[i]);
         d.X.vals[i] = resize_image(im, w, h).data;
@@ -1239,6 +1290,8 @@ data concat_data(data d1, data d2)
     d.shallow = 1;
     d.X = concat_matrix(d1.X, d2.X);
     d.y = concat_matrix(d1.y, d2.y);
+    d.w = d1.w;
+    d.h = d1.h;
     return d;
 }
 
diff --git a/src/utils.c b/src/utils.c
index b469ec55..9f1af1df 100644
--- a/src/utils.c
+++ b/src/utils.c
@@ -91,6 +91,22 @@ void shuffle(void *arr, size_t n, size_t size)
     }
 }
 
+int *random_index_order(int min, int max)
+{
+    int *inds = calloc(max-min, sizeof(int));
+    int i;
+    for(i = min; i < max; ++i){
+        inds[i] = i;
+    }
+    for(i = min; i < max-1; ++i){
+        int swap = inds[i];
+        int index = i + rand()%(max-i);
+        inds[i] = inds[index];
+        inds[index] = swap;
+    }
+    return inds;
+}
+
 void del_arg(int argc, char **argv, int index)
 {
     int i;
@@ -583,6 +599,20 @@ int sample_array(float *a, int n)
     return n-1;
 }
 
+int max_int_index(int *a, int n)
+{
+    if(n <= 0) return -1;
+    int i, max_i = 0;
+    int max = a[0];
+    for(i = 1; i < n; ++i){
+        if(a[i] > max){
+            max = a[i];
+            max_i = i;
+        }
+    }
+    return max_i;
+}
+
 int max_index(float *a, int n)
 {
     if(n <= 0) return -1;
diff --git a/src/utils.h b/src/utils.h
index 4e467075..b0db7abf 100644
--- a/src/utils.h
+++ b/src/utils.h
@@ -44,7 +44,6 @@ int constrain_int(int a, int min, int max);
 float rand_uniform(float min, float max);
 float rand_scale(float s);
 int rand_int(int min, int max);
-float sum_array(float *a, int n);
 void mean_arrays(float **a, int n, int els, float *avg);
 float dist_array(float *a, float *b, int n, int sub);
 float **one_hot_encode(float *a, int n, int k);

From 1a63695c44d56a71a18dda97617bd68cb67800d5 Mon Sep 17 00:00:00 2001
From: Joseph Redmon <pjreddie@gmail.com>
Date: Tue, 7 Nov 2017 16:12:00 -0800
Subject: [PATCH 2/3] that attention stuff doesn't even work :_(

---
 examples/attention.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/attention.c b/examples/attention.c
index 1afd8489..cd1e579d 100644
--- a/examples/attention.c
+++ b/examples/attention.c
@@ -299,7 +299,7 @@ void validate_attention_single(char *datacfg, char *filename, char *weightfile)
             float *pred = network_predict(net, tile.data);
             axpy_cpu(classes, 1., pred, 1, avgs, 1);
             show_image(tile, "tile");
-            cvWaitKey(10);
+            //cvWaitKey(10);
         }
         if(net->hierarchy) hierarchy_predictions(pred, net->outputs, net->hierarchy, 1, 1);
 

From 16686cec576580489ab3c7c78183e6efeafae780 Mon Sep 17 00:00:00 2001
From: Joseph Redmon <pjreddie@gmail.com>
Date: Tue, 7 Nov 2017 16:22:09 -0800
Subject: [PATCH 3/3] SEE I TOTALLY LISTEN TO MY COMMUNITY SOMETIMES I'M
 LOOKING AT YOU ISSUE \#291

---
 examples/detector.py | 1 +
 python/darknet.py    | 3 +++
 src/cuda.c           | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/examples/detector.py b/examples/detector.py
index aa565d61..5b639a3a 100644
--- a/examples/detector.py
+++ b/examples/detector.py
@@ -8,6 +8,7 @@ sys.path.append(os.path.join(os.getcwd(),'python/'))
 
 import darknet as dn
 
+dn.set_gpu(0)
 net = dn.load_net("cfg/tiny-yolo.cfg", "tiny-yolo.weights", 0)
 meta = dn.load_meta("cfg/coco.data")
 r = dn.detect(net, meta, "data/dog.jpg")
diff --git a/python/darknet.py b/python/darknet.py
index d6dfd31f..f9aab1ee 100644
--- a/python/darknet.py
+++ b/python/darknet.py
@@ -44,6 +44,9 @@ predict = lib.network_predict
 predict.argtypes = [c_void_p, POINTER(c_float)]
 predict.restype = POINTER(c_float)
 
+set_gpu = lib.cuda_set_device
+set_gpu.argtypes = [c_int]
+
 make_image = lib.make_image
 make_image.argtypes = [c_int, c_int, c_int]
 make_image.restype = IMAGE
diff --git a/src/cuda.c b/src/cuda.c
index b5c0c329..48aba6e4 100644
--- a/src/cuda.c
+++ b/src/cuda.c
@@ -172,5 +172,7 @@ float cuda_mag_array(float *x_gpu, size_t n)
     free(temp);
     return m;
 }
+#else
+void cuda_set_device(int n){}
 
 #endif