Added batch to col2im, padding option

Makefile

@@ -23,19 +23,21 @@ CFLAGS= $(COMMON) $(OPTS)
 LDFLAGS+=`pkg-config --libs opencv` -lm
 VPATH=./src/
 EXEC=cnn
+OBJDIR=./obj/
 
-OBJ=network.o image.o tests.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o data.o matrix.o softmax_layer.o mini_blas.o convolutional_layer.o gemm.o normalization_layer.o opencl.o im2col.o col2im.o axpy.o
+OBJ=network.o image.o cnn.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o data.o matrix.o softmax_layer.o mini_blas.o convolutional_layer.o gemm.o normalization_layer.o opencl.o im2col.o col2im.o axpy.o
+OBJS = $(addprefix $(OBJDIR), $(OBJ))
 
 all: $(EXEC)
 
-$(EXEC): $(OBJ)
+$(EXEC): $(OBJS)
 	$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@
 
-%.o: %.c 
+$(OBJDIR)%.o: %.c 
 	$(CC) $(CFLAGS) -c $< -o $@
 
 .PHONY: clean
 
 clean:
-	rm -rf $(OBJ) $(EXEC)
+	rm -rf $(OBJS) $(EXEC)
 

src/activations.cl

@@ -2,6 +2,12 @@ typedef enum{
     SIGMOID, RELU, LINEAR, RAMP, TANH
 }ACTIVATION;
 
+float linear_activate(float x){return x;}
+float sigmoid_activate(float x){return 1./(1. + exp(-x));}
+float relu_activate(float x){return x*(x>0);}
+float ramp_activate(float x){return x*(x>0)+.1*x;}
+float tanh_activate(float x){return (exp(2*x)-1)/(exp(2*x)+1);}
+
 float activate(float x, ACTIVATION a, float dropout)
 {
     //if((float)rand()/RAND_MAX < dropout) return 0;

src/tests.c → src/cnn.c

@@ -52,7 +52,7 @@ void test_convolve_matrix()
 	int i;
 	clock_t start = clock(), end;
 	for(i = 0; i < 1000; ++i){
-		im2col_cpu(dog.data,  1, dog.c,  dog.h,  dog.w,  size,  stride, matrix);
+		im2col_cpu(dog.data,  1, dog.c,  dog.h,  dog.w,  size,  stride, 0, matrix);
 		gemm(0,0,n,mw,mh,1,filters,mh,matrix,mw,1,edge.data,mw);
 	}
 	end = clock();
@@ -76,7 +76,7 @@ void verify_convolutional_layer()
 	int size = 3;
 	float eps = .00000001;
 	image test = make_random_image(5,5, 1);
-	convolutional_layer layer = *make_convolutional_layer(1,test.h,test.w,test.c, n, size, stride, RELU);
+	convolutional_layer layer = *make_convolutional_layer(1,test.h,test.w,test.c, n, size, stride, 0, RELU);
 	image out = get_convolutional_image(layer);
 	float **jacobian = calloc(test.h*test.w*test.c, sizeof(float));
 
@@ -301,30 +301,34 @@ void test_vince()
 void test_nist()
 {
 	srand(444444);
-	srand(888888);
+	srand(222222);
 	network net = parse_network_cfg("cfg/nist.cfg");
 	data train = load_categorical_data_csv("data/mnist/mnist_train.csv", 0, 10);
 	data test = load_categorical_data_csv("data/mnist/mnist_test.csv",0,10);
 	normalize_data_rows(train);
 	normalize_data_rows(test);
 	//randomize_data(train);
 	int count = 0;
-	float lr = .00005;
+	float lr = .000075;
 	float momentum = .9;
 	float decay = 0.0001;
 	decay = 0;
 	//clock_t start = clock(), end;
-	int batch = 10000;
-	while(++count <= 10000){
-		float loss = train_network_sgd(net, train, batch, lr, momentum, decay);
+	int iters = 100;
+	while(++count <= 10){
+		clock_t start = clock(), end;
+		float loss = train_network_sgd(net, train, iters, lr, momentum, decay);
+		end = clock();
 		float test_acc = network_accuracy(net, test);
-		printf("%3d %5f %5f\n",count, loss, test_acc);
+		printf("%d: %f %f, Time: %lf seconds, LR: %f, Momentum: %f, Decay: %f\n", count, loss, test_acc,(float)(end-start)/CLOCKS_PER_SEC, lr, momentum, decay);
+
 		//printf("%5d Training Loss: %lf, Params: %f %f %f, ",count*1000, loss, lr, momentum, decay);
 		//end = clock();
 		//printf("Time: %lf seconds\n", (float)(end-start)/CLOCKS_PER_SEC);
 		//start=end;
 		//lr *= .5;
 	}
+	//save_network(net, "cfg/nist_basic_trained.cfg");
 }
 
 void test_ensemble()
@@ -431,7 +435,7 @@ void test_im2row()
 	float *matrix = calloc(msize, sizeof(float));
 	int i;
 	for(i = 0; i < 1000; ++i){
-		im2col_cpu(test.data, 1, c,  h,  w,  size,  stride, matrix);
+		im2col_cpu(test.data, 1, c,  h,  w,  size,  stride, 0, matrix);
 		//image render = float_to_image(mh, mw, mc, matrix);
 	}
 }
@@ -442,34 +446,36 @@ void flip_network()
 	save_network(net, "cfg/voc_imagenet_rev.cfg");
 }
 
-void train_VOC()
+void tune_VOC()
 {
 	network net = parse_network_cfg("cfg/voc_start.cfg");
 	srand(2222222);
 	int i = 20;
 	char *labels[] = {"aeroplane","bicycle","bird","boat","bottle","bus","car","cat","chair","cow","diningtable","dog","horse","motorbike","person","pottedplant","sheep","sofa","train","tvmonitor"};
-	float lr = .00001;
+	float lr = .000005;
 	float momentum = .9;
-	float decay = 0.01;
+	float decay = 0.0001;
 	while(i++ < 1000 || 1){
-		data train = load_data_image_pathfile_random("images/VOC2012/val_paths.txt", 1000, labels, 20, 300, 400);
+		data train = load_data_image_pathfile_random("/home/pjreddie/VOC2012/trainval_paths.txt", 10, labels, 20, 256, 256);
 
-		image im = float_to_image(300, 400, 3,train.X.vals[0]);
+		image im = float_to_image(256, 256, 3,train.X.vals[0]);
 		show_image(im, "input");
 		visualize_network(net);
 		cvWaitKey(100);
 
-		normalize_data_rows(train);
+		translate_data_rows(train, -144);
 		clock_t start = clock(), end;
-		float loss = train_network_sgd(net, train, 1000, lr, momentum, decay);
+		float loss = train_network_sgd(net, train, 10, lr, momentum, decay);
 		end = clock();
 		printf("%d: %f, Time: %lf seconds, LR: %f, Momentum: %f, Decay: %f\n", i, loss, (float)(end-start)/CLOCKS_PER_SEC, lr, momentum, decay);
 		free_data(train);
+        /*
 		if(i%10==0){
 			char buff[256];
-			sprintf(buff, "cfg/voc_clean_ramp_%d.cfg", i);
+			sprintf(buff, "/home/pjreddie/voc_cfg/voc_ramp_%d.cfg", i);
 			save_network(net, buff);
 		}
+        */
 		//lr *= .99;
 	}
 }
@@ -778,7 +784,7 @@ int main(int argc, char *argv[])
 	//test_cifar10();
 	//test_vince();
 	//test_full();
-	//train_VOC();
+	//tune_VOC();
 	//features_VOC_image(argv[1], argv[2], argv[3], 0);
 	//features_VOC_image(argv[1], argv[2], argv[3], 1);
 	//features_VOC_image_size(argv[1], atoi(argv[2]), atoi(argv[3]));

src/col2im.c

@@ -0,0 +1,47 @@
+inline void col2im_set_pixel(float *im, int height, int width, int channels,
+                        int row, int col, int channel, int pad, float val)
+{
+    row -= pad;
+    col -= pad;
+
+    if (row < 0 || col < 0 ||
+        row >= height || col >= width) return;
+    im[col + width*(row + channel*height)] = val;
+}
+//This one might be too, can't remember.
+void col2im_cpu(float* data_col,
+        const int batch, const int channels, const int height, const int width,
+        const int ksize, const int stride, int pad, float* data_im) 
+{
+    int c,h,w,b;
+    int height_col = (height - ksize) / stride + 1;
+    int width_col = (width - ksize) / stride + 1;
+    if (pad){
+        height_col = 1 + (height-1) / stride;
+        width_col = 1 + (width-1) / stride;
+        pad = ksize/2;
+    }
+    int channels_col = channels * ksize * ksize;
+    int im_size = height*width*channels;
+    int col_size = height_col*width_col*channels_col;
+    for (b = 0; b < batch; ++b) {
+        for (c = 0; c < channels_col; ++c) {
+            int w_offset = c % ksize;
+            int h_offset = (c / ksize) % ksize;
+            int c_im = c / ksize / ksize;
+            for (h = 0; h < height_col; ++h) {
+                for (w = 0; w < width_col; ++w) {
+                    int im_row = h_offset + h * stride;
+                    int im_col = w_offset + w * stride;
+                    double val = data_col[(c * height_col + h) * width_col + w];
+                    col2im_set_pixel(data_im, height, width, channels,
+                                    im_row, im_col, c_im, pad, val);
+                }
+            }
+        }
+        data_im += im_size;
+        data_col+= col_size;
+    }
+}
+
+

src/connected_layer.c

@@ -57,8 +57,11 @@ void update_connected_layer(connected_layer layer, float step, float momentum, f
 
 void forward_connected_layer(connected_layer layer, float *input, int train)
 {
+    int i;
     if(!train) layer.dropout = 0;
-    memcpy(layer.output, layer.biases, layer.outputs*sizeof(float));
+    for(i = 0; i < layer.batch; ++i){
+        memcpy(layer.output+i*layer.outputs, layer.biases, layer.outputs*sizeof(float));
+    }
     int m = layer.batch;
     int k = layer.inputs;
     int n = layer.outputs;
@@ -82,16 +85,16 @@ void backward_connected_layer(connected_layer layer, float *input, float *delta)
     float *a = input;
     float *b = layer.delta;
     float *c = layer.weight_updates;
-    gemm(0,0,m,n,k,1,a,k,b,n,1,c,n);
+    gemm(1,0,m,n,k,1,a,k,b,n,1,c,n);
 
-    m = layer.inputs;
+    m = layer.batch;
     k = layer.outputs;
-    n = layer.batch;
+    n = layer.inputs;
 
-    a = layer.weights;
-    b = layer.delta;
+    a = layer.delta;
+    b = layer.weights;
     c = delta;
 
-    if(c) gemm(0,0,m,n,k,1,a,k,b,n,0,c,n);
+    if(c) gemm(0,1,m,n,k,1,a,k,b,k,0,c,n);
 }
 

src/convolutional_layer.c

@@ -5,12 +5,18 @@
 
 int convolutional_out_height(convolutional_layer layer)
 {
-    return (layer.h-layer.size)/layer.stride + 1;
+    int h = layer.h;
+    if (!layer.pad) h -= layer.size;
+    else h -= 1;
+    return h/layer.stride + 1;
 }
 
 int convolutional_out_width(convolutional_layer layer)
 {
-    return (layer.w-layer.size)/layer.stride + 1;
+    int w = layer.w;
+    if (!layer.pad) w -= layer.size;
+    else w -= 1;
+    return w/layer.stride + 1;
 }
 
 image get_convolutional_image(convolutional_layer layer)
@@ -31,7 +37,7 @@ image get_convolutional_delta(convolutional_layer layer)
     return float_to_image(h,w,c,layer.delta);
 }
 
-convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, int n, int size, int stride, ACTIVATION activation)
+convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, int n, int size, int stride, int pad, ACTIVATION activation)
 {
     int i;
     size = 2*(size/2)+1; //HA! And you thought you'd use an even sized filter...
@@ -43,6 +49,7 @@ convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, in
     layer->batch = batch;
     layer->stride = stride;
     layer->size = size;
+    layer->pad = pad;
 
     layer->filters = calloc(c*n*size*size, sizeof(float));
     layer->filter_updates = calloc(c*n*size*size, sizeof(float));
@@ -64,6 +71,17 @@ convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, in
     layer->output = calloc(layer->batch*out_h * out_w * n, sizeof(float));
     layer->delta  = calloc(layer->batch*out_h * out_w * n, sizeof(float));
     #ifdef GPU
+    layer->filters_cl = cl_make_array(layer->filters, c*n*size*size);
+    layer->filter_updates_cl = cl_make_array(layer->filter_updates, c*n*size*size);
+    layer->filter_momentum_cl = cl_make_array(layer->filter_momentum, c*n*size*size);
+
+    layer->biases_cl = cl_make_array(layer->biases, n);
+    layer->bias_updates_cl = cl_make_array(layer->bias_updates, n);
+    layer->bias_momentum_cl = cl_make_array(layer->bias_momentum, n);
+
+    layer->col_image_cl = cl_make_array(layer->col_image, layer->batch*out_h*out_w*size*size*c);
+    layer->delta_cl = cl_make_array(layer->delta, layer->batch*out_h*out_w*n);
+    layer->output_cl = cl_make_array(layer->output, layer->batch*out_h*out_w*n);
     #endif
     layer->activation = activation;
 
@@ -91,12 +109,14 @@ void resize_convolutional_layer(convolutional_layer *layer, int h, int w, int c)
 
 void bias_output(const convolutional_layer layer)
 {
-    int i,j;
+    int i,j,b;
     int out_h = convolutional_out_height(layer);
     int out_w = convolutional_out_width(layer);
-    for(i = 0; i < layer.n; ++i){
-        for(j = 0; j < out_h*out_w; ++j){
-            layer.output[i*out_h*out_w + j] = layer.biases[i];
+    for(b = 0; b < layer.batch; ++b){
+        for(i = 0; i < layer.n; ++i){
+            for(j = 0; j < out_h*out_w; ++j){
+                layer.output[(b*layer.n + i)*out_h*out_w + j] = layer.biases[i];
+            }
         }
     }
 }
@@ -114,7 +134,7 @@ void forward_convolutional_layer(const convolutional_layer layer, float *in)
     float *b = layer.col_image;
     float *c = layer.output;
     im2col_cpu(in,layer.batch, layer.c, layer.h, layer.w, 
-        layer.size, layer.stride, b);
+        layer.size, layer.stride, layer.pad, b);
     bias_output(layer);
     gemm(0,0,m,n,k,1,a,k,b,n,1,c,n);
     activate_array(layer.output, m*n, layer.activation, 0.);
@@ -169,7 +189,6 @@ void backward_convolutional_layer(convolutional_layer layer, float *delta)
     gemm(0,1,m,n,k,1,a,k,b,k,1,c,n);
 
     if(delta){
-        int i;
         m = layer.size*layer.size*layer.c;
         k = layer.n;
         n = convolutional_out_height(layer)*
@@ -183,9 +202,7 @@ void backward_convolutional_layer(convolutional_layer layer, float *delta)
         gemm(1,0,m,n,k,1,a,m,b,n,0,c,n);
 
         memset(delta, 0, layer.batch*layer.h*layer.w*layer.c*sizeof(float));
-        for(i = 0; i < layer.batch; ++i){
-            col2im_cpu(c+i*n/layer.batch,  layer.c,  layer.h,  layer.w,  layer.size,  layer.stride, delta+i*n/layer.batch);
-        }
+        col2im_cpu(c, layer.batch,  layer.c,  layer.h,  layer.w,  layer.size,  layer.stride, layer.pad, delta);
     }
 }
 

src/convolutional_layer.h

@@ -14,6 +14,7 @@ typedef struct {
     int n;
     int size;
     int stride;
+    int pad;
     float *filters;
     float *filter_updates;
     float *filter_momentum;
@@ -47,7 +48,7 @@ typedef struct {
 void forward_convolutional_layer_gpu(convolutional_layer layer, cl_mem in);
 #endif
 
-convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, int n, int size, int stride, ACTIVATION activation);
+convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, int n, int size, int stride, int pad, ACTIVATION activation);
 void resize_convolutional_layer(convolutional_layer *layer, int h, int w, int c);
 void forward_convolutional_layer(const convolutional_layer layer, float *in);
 void update_convolutional_layer(convolutional_layer layer, float step, float momentum, float decay);

src/data.c

@@ -166,6 +166,14 @@ void scale_data_rows(data d, float s)
     }
 }
 
+void translate_data_rows(data d, float s)
+{
+    int i;
+    for(i = 0; i < d.X.rows; ++i){
+        translate_array(d.X.vals[i], d.X.cols, s);
+    }
+}
+
 void normalize_data_rows(data d)
 {
     int i;

src/data.h

@@ -22,6 +22,7 @@ list *get_paths(char *filename);
 data load_categorical_data_csv(char *filename, int target, int k);
 void normalize_data_rows(data d);
 void scale_data_rows(data d, float s);
+void translate_data_rows(data d, float s);
 void randomize_data(data d);
 data *split_data(data d, int part, int total);