yolo has dataset file as the first parameter.

README.md

@@ -6,3 +6,10 @@ Darknet is an open source neural network framework written in C and CUDA. It is
 For more information see the [Darknet project website](http://pjreddie.com/darknet).
 
 For questions or issues please use the [Google Group](https://groups.google.com/forum/#!forum/darknet).
+
+#Extension#
+yolo now has dataset as the first parameter
+
+```Shell
+./darknet yolo train cfg/voc0712.dataset cfg/tiny-yolo.cfg  model/darknet.conv.weights
+```

cfg/ilsvrc2012.dataset

@@ -0,0 +1,11 @@
+classes=1000
+#train  = /home/labuser/CNN/data/imagenet/train.list
+##valid  = /home/labuser/CNN/data/imagenet/val.list	# no classifier in file path
+#valid  = data/inet.val.list
+train  = /data/imagenet/train_data.list
+valid  = /data/imagenet/inet.val.list
+backup = /home/labuser/CNN/backup
+labels = data/ilsvrc2012.labels.list
+names  = data/ilsvrc2012.shortnames.list
+top=5
+

cfg/voc0712.dataset

@@ -0,0 +1,7 @@
+classes=20
+train  = /data/voc/train.txt
+valid  = /data/voc/2007_val.txt
+backup = /home/labuser/CNN/backup
+labels = data/voc.labels.list
+names  = data/voc.shortnames.list
+

src/data.c

@@ -242,7 +242,7 @@ void fill_truth_swag(char *path, float *truth, int classes, int flip, float dx,
     free(boxes);
 }
 
-void fill_truth_region(char *path, float *truth, int classes, int num_boxes, int flip, float dx, float dy, float sx, float sy)
+void fill_truth_region(char *path, float *truth, char **labels, int classes, int num_boxes, int flip, float dx, float dy, float sx, float sy)
 {
     char labelpath[4096];
     find_replace(path, "images", "labels", labelpath);
@@ -258,7 +258,7 @@ void fill_truth_region(char *path, float *truth, int classes, int num_boxes, int
     correct_boxes(boxes, count, dx, dy, sx, sy, flip);
     float x,y,w,h;
     int id;
-    int i;
+    int i, j;
 
     for (i = 0; i < count; ++i) {
         x =  boxes[i].x;
@@ -267,6 +267,12 @@ void fill_truth_region(char *path, float *truth, int classes, int num_boxes, int
         h =  boxes[i].h;
         id = boxes[i].id;
 
+        // check id in labels
+        for (j = 0; j < classes; ++j) {
+            if (id == atoi(labels[j])) break;
+        }
+        if (j == classes) continue;
+
         if (w < .005 || h < .005) continue;
 
         int col = (int)(x*num_boxes);
@@ -279,7 +285,7 @@ void fill_truth_region(char *path, float *truth, int classes, int num_boxes, int
         if (truth[index]) continue;
         truth[index++] = 1;
 
-        if (id < classes) truth[index+id] = 1;
+        truth[index+j] = 1;
         index += classes;
 
         truth[index++] = x;
@@ -488,7 +494,7 @@ void free_data(data d)
     }
 }
 
-data load_data_region(int n, char **paths, int m, int w, int h, int size, int classes, float jitter, float hue, float saturation, float exposure)
+data load_data_region(int n, char **paths, int m, int w, int h, int size, char **labels, int classes, float jitter, float hue, float saturation, float exposure)
 {
     char **random_paths = get_random_paths(paths, n, m);
     int i;
@@ -533,7 +539,7 @@ data load_data_region(int n, char **paths, int m, int w, int h, int size, int cl
         random_distort_image(sized, hue, saturation, exposure);
         d.X.vals[i] = sized.data;
 
-        fill_truth_region(random_paths[i], d.y.vals[i], classes, size, flip, dx, dy, 1./sx, 1./sy);
+        fill_truth_region(random_paths[i], d.y.vals[i], labels, classes, size, flip, dx, dy, 1./sx, 1./sy);
 
         free_image(orig);
         free_image(cropped);
@@ -729,7 +735,7 @@ void *load_thread(void *ptr)
     } else if (a.type == WRITING_DATA){
         *a.d = load_data_writing(a.paths, a.n, a.m, a.w, a.h, a.out_w, a.out_h);
     } else if (a.type == REGION_DATA){
-        *a.d = load_data_region(a.n, a.paths, a.m, a.w, a.h, a.num_boxes, a.classes, a.jitter, a.hue, a.saturation, a.exposure);
+        *a.d = load_data_region(a.n, a.paths, a.m, a.w, a.h, a.num_boxes, a.labels, a.classes, a.jitter, a.hue, a.saturation, a.exposure);
     } else if (a.type == DETECTION_DATA){
         *a.d = load_data_detection(a.n, a.paths, a.m, a.w, a.h, a.num_boxes, a.classes, a.jitter, a.hue, a.saturation, a.exposure);
     } else if (a.type == SWAG_DATA){

src/yolo.c

@@ -3,19 +3,16 @@
 #include "cost_layer.h"
 #include "utils.h"
 #include "parser.h"
+#include "option_list.h"
 #include "box.h"
 #include "demo.h"
 
 #ifdef OPENCV
 #include "opencv2/highgui/highgui_c.h"
 #endif
 
-char *voc_names[] = {"aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"};
-
-void train_yolo(char *cfgfile, char *weightfile)
+void train_yolo(char *datacfg, char *cfgfile, char *weightfile)
 {
-    char *train_images = "/data/voc/train.txt";
-    char *backup_directory = "/home/pjreddie/backup/";
     srand(time(0));
     char *base = basecfg(cfgfile);
     printf("%s\n", base);
@@ -26,19 +23,24 @@ void train_yolo(char *cfgfile, char *weightfile)
     }
     printf("Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
     int imgs = net.batch*net.subdivisions;
-    int i = *net.seen/imgs;
-    data train, buffer;
+    list *options = read_data_cfg(datacfg);
 
+    char *backup_directory = option_find_str(options, "backup", "/backup/");
+    char *label_list = option_find_str(options, "labels", "data/voc.labels.list");
+    char *train_list = option_find_str(options, "train", "data/voc.train.list");
+    int classes = option_find_int(options, "classes", 20);
 
     layer l = net.layers[net.n - 1];
 
     int side = l.side;
-    int classes = l.classes;
     float jitter = l.jitter;
+    assert(classes == l.classes);
 
-    list *plist = get_paths(train_images);
-    //int N = plist->size;
+    char **labels = get_labels(label_list);
+    list *plist = get_paths(train_list);
+    int N = plist->size;
     char **paths = (char **)list_to_array(plist);
+    data train, buffer;
 
     load_args args = {0};
     args.w = net.w;
@@ -50,6 +52,7 @@ void train_yolo(char *cfgfile, char *weightfile)
     args.jitter = jitter;
     args.num_boxes = side;
     args.d = &buffer;
+    args.labels = labels;
     args.type = REGION_DATA;
 
     args.angle = net.angle;
@@ -59,9 +62,8 @@ void train_yolo(char *cfgfile, char *weightfile)
 
     pthread_t load_thread = load_data_in_thread(args);
     clock_t time;
-    //while(i*imgs < N*120){
-    while(get_current_batch(net) < net.max_batches){
-        i += 1;
+    int epoch = (*net.seen)/N;
+    while(get_current_batch(net) < net.max_batches || net.max_batches == 0){
         time=clock();
         pthread_join(load_thread, 0);
         train = buffer;
@@ -74,10 +76,16 @@ void train_yolo(char *cfgfile, char *weightfile)
         if (avg_loss < 0) avg_loss = loss;
         avg_loss = avg_loss*.9 + loss*.1;
 
-        printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), i*imgs);
-        if(i%1000==0 || (i < 1000 && i%100 == 0)){
+        printf("%d: %f, %f avg, %f rate, %lf seconds, %d images\n", get_current_batch(net), loss, avg_loss, get_current_rate(net), sec(clock()-time), *net.seen);
+        if(*net.seen/N > epoch){
+            epoch = *net.seen/N;
+            char buff[256];
+            sprintf(buff, "%s/%s_%d.weights", backup_directory, base, epoch);
+            save_weights(net, buff);
+        }
+        if(get_current_batch(net)%100 == 0){
             char buff[256];
-            sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
+            sprintf(buff, "%s/%s.backup",backup_directory,base);
             save_weights(net, buff);
         }
         free_data(train);
@@ -108,8 +116,10 @@ void print_yolo_detections(FILE **fps, char *id, box *boxes, float **probs, int
     }
 }
 
-void validate_yolo(char *cfgfile, char *weightfile)
+void validate_yolo(char *datacfg, char *cfgfile, char *weightfile)
 {
+    char *base = "results/comp4_det_test_";
+
     network net = parse_network_cfg(cfgfile);
     if(weightfile){
         load_weights(&net, weightfile);
@@ -118,20 +128,25 @@ void validate_yolo(char *cfgfile, char *weightfile)
     fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
     srand(time(0));
 
-    char *base = "results/comp4_det_test_";
-    //list *plist = get_paths("data/voc.2007.test");
-    list *plist = get_paths("/home/pjreddie/data/voc/2007_test.txt");
-    //list *plist = get_paths("data/voc.2012.test");
+    list *options = read_data_cfg(datacfg);
+
+    char *name_list = option_find_str(options, "names", "data/voc.shortnames.list");
+    char *valid_list = option_find_str(options, "valid", "data/voc.2007.valid");
+    int classes = option_find_int(options, "classes", 20);
+
+    char **names = get_labels(name_list);
+    list *plist = get_paths(valid_list);
+
     char **paths = (char **)list_to_array(plist);
 
     layer l = net.layers[net.n-1];
-    int classes = l.classes;
+    assert(classes == l.classes);
 
     int j;
     FILE **fps = calloc(classes, sizeof(FILE *));
     for(j = 0; j < classes; ++j){
         char buff[1024];
-        snprintf(buff, 1024, "%s%s.txt", base, voc_names[j]);
+        snprintf(buff, 1024, "%s%s.txt", base, names[j]);
         fps[j] = fopen(buff, "w");
     }
     box *boxes = calloc(l.side*l.side*l.n, sizeof(box));
@@ -196,8 +211,10 @@ void validate_yolo(char *cfgfile, char *weightfile)
     fprintf(stderr, "Total Detection Time: %f Seconds\n", (double)(time(0) - start));
 }
 
-void validate_yolo_recall(char *cfgfile, char *weightfile)
+void validate_yolo_recall(char *datacfg, char *cfgfile, char *weightfile)
 {
+    char *base = "results/comp4_det_test_";
+
     network net = parse_network_cfg(cfgfile);
     if(weightfile){
         load_weights(&net, weightfile);
@@ -206,19 +223,26 @@ void validate_yolo_recall(char *cfgfile, char *weightfile)
     fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
     srand(time(0));
 
-    char *base = "results/comp4_det_test_";
-    list *plist = get_paths("data/voc.2007.test");
+    list *options = read_data_cfg(datacfg);
+
+    char *name_list = option_find_str(options, "names", "data/voc.shortnames.list");
+    char *test_list = option_find_str(options, "test", "data/voc.2007.test");
+    int classes = option_find_int(options, "classes", 20);
+
+    char **names = get_labels(name_list);
+    list *plist = get_paths(test_list);
+
     char **paths = (char **)list_to_array(plist);
 
     layer l = net.layers[net.n-1];
-    int classes = l.classes;
+    assert(classes == l.classes);
     int side = l.side;
 
     int j, k;
     FILE **fps = calloc(classes, sizeof(FILE *));
     for(j = 0; j < classes; ++j){
         char buff[1024];
-        snprintf(buff, 1024, "%s%s.txt", base, voc_names[j]);
+        snprintf(buff, 1024, "%s%s.txt", base, names[j]);
         fps[j] = fopen(buff, "w");
     }
     box *boxes = calloc(side*side*l.n, sizeof(box));
@@ -282,13 +306,16 @@ void validate_yolo_recall(char *cfgfile, char *weightfile)
     }
 }
 
-void test_yolo(char *cfgfile, char *weightfile, char *filename, float thresh)
+void test_yolo(char *datacfg, char *cfgfile, char *weightfile, char *filename, float thresh)
 {
     image **alphabet = load_alphabet();
     network net = parse_network_cfg(cfgfile);
     if(weightfile){
         load_weights(&net, weightfile);
     }
+    list *options = read_data_cfg(datacfg);
+    char *name_list = option_find_str(options, "names", "data/voc.shortnames.list");
+    char **names = get_labels(name_list);
     detection_layer l = net.layers[net.n-1];
     set_batch_network(&net, 1);
     srand(2222222);
@@ -318,8 +345,8 @@ void test_yolo(char *cfgfile, char *weightfile, char *filename, float thresh)
         printf("%s: Predicted in %f seconds.\n", input, sec(clock()-time));
         get_detection_boxes(l, 1, 1, thresh, probs, boxes, 0);
         if (nms) do_nms_sort(boxes, probs, l.side*l.side*l.n, l.classes, nms);
-        //draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, voc_names, alphabet, 20);
-        draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, voc_names, alphabet, 20);
+        //draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, names, alphabet, l.classes);
+        draw_detections(im, l.side*l.side*l.n, thresh, boxes, probs, names, alphabet, l.classes);
         save_image(im, "predictions");
         show_image(im, "predictions");
 
@@ -344,12 +371,19 @@ void run_yolo(int argc, char **argv)
         return;
     }
 
-    char *cfg = argv[3];
-    char *weights = (argc > 4) ? argv[4] : 0;
-    char *filename = (argc > 5) ? argv[5]: 0;
-    if(0==strcmp(argv[2], "test")) test_yolo(cfg, weights, filename, thresh);
-    else if(0==strcmp(argv[2], "train")) train_yolo(cfg, weights);
-    else if(0==strcmp(argv[2], "valid")) validate_yolo(cfg, weights);
-    else if(0==strcmp(argv[2], "recall")) validate_yolo_recall(cfg, weights);
-    else if(0==strcmp(argv[2], "demo")) demo(cfg, weights, thresh, cam_index, filename, voc_names, 20, frame_skip, prefix, .5);
+    char *data = argv[3];
+    char *cfg = argv[4];
+    char *weights = (argc > 5) ? argv[5] : 0;
+    char *filename = (argc > 6) ? argv[6]: 0;
+    if(0==strcmp(argv[2], "test")) test_yolo(data, cfg, weights, filename, thresh);
+    else if(0==strcmp(argv[2], "train")) train_yolo(data, cfg, weights);
+    else if(0==strcmp(argv[2], "valid")) validate_yolo(data, cfg, weights);
+    else if(0==strcmp(argv[2], "recall")) validate_yolo_recall(data, cfg, weights);
+    else if(0==strcmp(argv[2], "demo")){
+        list *options = read_data_cfg(data);
+        char *name_list = option_find_str(options, "names", "data/voc.shortnames.list");
+        char **names = get_labels(name_list);
+        int classes = option_find_int(options, "classes", 20);
+        demo(cfg, weights, thresh, cam_index, filename, names, classes, frame_skip, prefix, .5);
+    }
 }