From 2db9fbef2bd7d35a547d0018a9850f6b249c524f Mon Sep 17 00:00:00 2001
From: Joseph Redmon <pjreddie@gmail.com>
Date: Wed, 13 Nov 2013 18:50:38 +0000
Subject: [PATCH] Parsing, image loading, lots of stuff
---
.gitignore | 1
test_random_filter.cfg | 29 +
Makefile | 6
src/list.c | 90 +++
src/parser.h | 7
src/convolutional_layer.h | 24
src/image.c | 29 +
test.cfg | 37 +
src/tests.c | 277 ++++-----
src/image.h | 3
src/maxpool_layer.c | 33
src/utils.c | 147 +++++
src/network.c | 154 +++--
src/matrix.c | 106 +++
src/maxpool_layer.h | 7
src/matrix.h | 17
src/utils.h | 18
src/network.h | 9
src/connected_layer.c | 49
random_filter_finish.cfg | 8
src/connected_layer.h | 15
src/data.c | 108 ++++
src/data.h | 18
src/option_list.h | 12
src/convolutional_layer.c | 99 ++
src/option_list.c | 68 ++
src/parser.c | 168 ++++++
src/list.h | 26
test_parser.cfg | 8
29 files changed, 1,295 insertions(+), 278 deletions(-)
diff --git a/.gitignore b/.gitignore
index 9153fbb..7913d67 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@
opencv/
convnet/
decaf/
+submission/
cnn
# OS Generated #
diff --git a/Makefile b/Makefile
index bf7cfd3..3140af5 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,10 @@
CC=gcc
-CFLAGS=-Wall `pkg-config --cflags opencv` -O3 -flto -ffast-math
-CFLAGS=-Wall `pkg-config --cflags opencv` -O0 -g
+CFLAGS=-Wall `pkg-config --cflags opencv` -O3 -ffast-math -flto -march=native
+#CFLAGS=-Wall `pkg-config --cflags opencv` -O0 -g
LDFLAGS=`pkg-config --libs opencv` -lm
VPATH=./src/
-OBJ=network.o image.o tests.o convolutional_layer.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o
+OBJ=network.o image.o tests.o convolutional_layer.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o data.o matrix.o
all: cnn
diff --git a/random_filter_finish.cfg b/random_filter_finish.cfg
new file mode 100644
index 0000000..68ddda1
--- /dev/null
+++ b/random_filter_finish.cfg
@@ -0,0 +1,8 @@
+[conn]
+input = 1690
+output = 20
+activation=relu
+
+[conn]
+output = 1
+activation=relu
diff --git a/src/connected_layer.c b/src/connected_layer.c
index 9fafc38..d77a10c 100644
--- a/src/connected_layer.c
+++ b/src/connected_layer.c
@@ -1,27 +1,32 @@
#include "connected_layer.h"
#include <math.h>
+#include <stdio.h>
#include <stdlib.h>
#include <string.h>
connected_layer *make_connected_layer(int inputs, int outputs, ACTIVATION activator)
{
+ printf("Connected Layer: %d inputs, %d outputs\n", inputs, outputs);
int i;
connected_layer *layer = calloc(1, sizeof(connected_layer));
layer->inputs = inputs;
layer->outputs = outputs;
layer->output = calloc(outputs, sizeof(double*));
+ layer->delta = calloc(outputs, sizeof(double*));
layer->weight_updates = calloc(inputs*outputs, sizeof(double));
+ layer->weight_momentum = calloc(inputs*outputs, sizeof(double));
layer->weights = calloc(inputs*outputs, sizeof(double));
for(i = 0; i < inputs*outputs; ++i)
- layer->weights[i] = .5 - (double)rand()/RAND_MAX;
+ layer->weights[i] = .01*(.5 - (double)rand()/RAND_MAX);
layer->bias_updates = calloc(outputs, sizeof(double));
+ layer->bias_momentum = calloc(outputs, sizeof(double));
layer->biases = calloc(outputs, sizeof(double));
for(i = 0; i < outputs; ++i)
- layer->biases[i] = (double)rand()/RAND_MAX;
+ layer->biases[i] = 1;
if(activator == SIGMOID){
layer->activation = sigmoid_activation;
@@ -37,7 +42,7 @@
return layer;
}
-void run_connected_layer(double *input, connected_layer layer)
+void forward_connected_layer(connected_layer layer, double *input)
{
int i, j;
for(i = 0; i < layer.outputs; ++i){
@@ -49,48 +54,44 @@
}
}
-void learn_connected_layer(double *input, connected_layer layer)
+void learn_connected_layer(connected_layer layer, double *input)
{
- calculate_update_connected_layer(input, layer);
- backpropagate_connected_layer(input, layer);
+ int i, j;
+ for(i = 0; i < layer.outputs; ++i){
+ layer.bias_updates[i] += layer.delta[i];
+ for(j = 0; j < layer.inputs; ++j){
+ layer.weight_updates[i*layer.inputs + j] += layer.delta[i]*input[j];
+ }
+ }
}
-void update_connected_layer(connected_layer layer, double step)
+void update_connected_layer(connected_layer layer, double step, double momentum, double decay)
{
int i,j;
for(i = 0; i < layer.outputs; ++i){
- layer.biases[i] += step*layer.bias_updates[i];
+ layer.bias_momentum[i] = step*(layer.bias_updates[i] - decay*layer.biases[i]) + momentum*layer.bias_momentum[i];
+ layer.biases[i] += layer.bias_momentum[i];
for(j = 0; j < layer.inputs; ++j){
int index = i*layer.inputs+j;
- layer.weights[index] += step*layer.weight_updates[index];
+ layer.weight_momentum[index] = step*(layer.weight_updates[index] - decay*layer.weights[index]) + momentum*layer.weight_momentum[index];
+ layer.weights[index] += layer.weight_momentum[index];
}
}
memset(layer.bias_updates, 0, layer.outputs*sizeof(double));
memset(layer.weight_updates, 0, layer.outputs*layer.inputs*sizeof(double));
}
-void calculate_update_connected_layer(double *input, connected_layer layer)
-{
- int i, j;
- for(i = 0; i < layer.outputs; ++i){
- layer.bias_updates[i] += layer.output[i];
- for(j = 0; j < layer.inputs; ++j){
- layer.weight_updates[i*layer.inputs + j] += layer.output[i]*input[j];
- }
- }
-}
-
-void backpropagate_connected_layer(double *input, connected_layer layer)
+void backward_connected_layer(connected_layer layer, double *input, double *delta)
{
int i, j;
for(j = 0; j < layer.inputs; ++j){
double grad = layer.gradient(input[j]);
- input[j] = 0;
+ delta[j] = 0;
for(i = 0; i < layer.outputs; ++i){
- input[j] += layer.output[i]*layer.weights[i*layer.inputs + j];
+ delta[j] += layer.delta[i]*layer.weights[i*layer.inputs + j];
}
- input[j] *= grad;
+ delta[j] *= grad;
}
}
diff --git a/src/connected_layer.h b/src/connected_layer.h
index eaea306..86815cb 100644
--- a/src/connected_layer.h
+++ b/src/connected_layer.h
@@ -8,9 +8,15 @@
int outputs;
double *weights;
double *biases;
+
double *weight_updates;
double *bias_updates;
+
+ double *weight_momentum;
+ double *bias_momentum;
+
double *output;
+ double *delta;
double (* activation)();
double (* gradient)();
@@ -18,12 +24,11 @@
connected_layer *make_connected_layer(int inputs, int outputs, ACTIVATION activator);
-void run_connected_layer(double *input, connected_layer layer);
-void learn_connected_layer(double *input, connected_layer layer);
-void update_connected_layer(connected_layer layer, double step);
+void forward_connected_layer(connected_layer layer, double *input);
+void backward_connected_layer(connected_layer layer, double *input, double *delta);
+void learn_connected_layer(connected_layer layer, double *input);
+void update_connected_layer(connected_layer layer, double step, double momentum, double decay);
-void backpropagate_connected_layer(double *input, connected_layer layer);
-void calculate_update_connected_layer(double *input, connected_layer layer);
#endif
diff --git a/src/convolutional_layer.c b/src/convolutional_layer.c
index 7478158..d4aff73 100644
--- a/src/convolutional_layer.c
+++ b/src/convolutional_layer.c
@@ -1,52 +1,93 @@
#include "convolutional_layer.h"
+#include <stdio.h>
-double convolution_activation(double x)
+image get_convolutional_image(convolutional_layer layer)
{
- return x*(x>0);
+ int h = (layer.h-1)/layer.stride + 1;
+ int w = (layer.w-1)/layer.stride + 1;
+ int c = layer.n;
+ return double_to_image(h,w,c,layer.output);
}
-double convolution_gradient(double x)
+image get_convolutional_delta(convolutional_layer layer)
{
- return (x>=0);
+ int h = (layer.h-1)/layer.stride + 1;
+ int w = (layer.w-1)/layer.stride + 1;
+ int c = layer.n;
+ return double_to_image(h,w,c,layer.delta);
}
-convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int size, int stride)
+convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int size, int stride, ACTIVATION activator)
{
+ printf("Convolutional Layer: %d x %d x %d image, %d filters\n", h,w,c,n);
int i;
convolutional_layer *layer = calloc(1, sizeof(convolutional_layer));
+ layer->h = h;
+ layer->w = w;
+ layer->c = c;
layer->n = n;
layer->stride = stride;
layer->kernels = calloc(n, sizeof(image));
layer->kernel_updates = calloc(n, sizeof(image));
+ layer->biases = calloc(n, sizeof(double));
+ layer->bias_updates = calloc(n, sizeof(double));
for(i = 0; i < n; ++i){
+ layer->biases[i] = .005;
layer->kernels[i] = make_random_kernel(size, c);
layer->kernel_updates[i] = make_random_kernel(size, c);
}
- layer->output = make_image((h-1)/stride+1, (w-1)/stride+1, n);
+ layer->output = calloc(((h-1)/stride+1) * ((w-1)/stride+1) * n, sizeof(double));
+ layer->delta = calloc(((h-1)/stride+1) * ((w-1)/stride+1) * n, sizeof(double));
layer->upsampled = make_image(h,w,n);
+
+ if(activator == SIGMOID){
+ layer->activation = sigmoid_activation;
+ layer->gradient = sigmoid_gradient;
+ }else if(activator == RELU){
+ layer->activation = relu_activation;
+ layer->gradient = relu_gradient;
+ }else if(activator == IDENTITY){
+ layer->activation = identity_activation;
+ layer->gradient = identity_gradient;
+ }
return layer;
}
-void run_convolutional_layer(const image input, const convolutional_layer layer)
+void forward_convolutional_layer(const convolutional_layer layer, double *in)
{
- int i;
+ image input = double_to_image(layer.h, layer.w, layer.c, in);
+ image output = get_convolutional_image(layer);
+ int i,j;
for(i = 0; i < layer.n; ++i){
- convolve(input, layer.kernels[i], layer.stride, i, layer.output);
+ convolve(input, layer.kernels[i], layer.stride, i, output);
}
- for(i = 0; i < layer.output.h*layer.output.w*layer.output.c; ++i){
- layer.output.data[i] = convolution_activation(layer.output.data[i]);
+ for(i = 0; i < output.c; ++i){
+ for(j = 0; j < output.h*output.w; ++j){
+ int index = i*output.h*output.w + j;
+ output.data[index] += layer.biases[i];
+ output.data[index] = layer.activation(output.data[index]);
+ }
}
}
-void backpropagate_convolutional_layer(image input, convolutional_layer layer)
+void backward_convolutional_layer(convolutional_layer layer, double *input, double *delta)
{
int i;
- zero_image(input);
+
+ image in_image = double_to_image(layer.h, layer.w, layer.c, input);
+ image in_delta = double_to_image(layer.h, layer.w, layer.c, delta);
+ image out_delta = get_convolutional_delta(layer);
+ zero_image(in_delta);
+
for(i = 0; i < layer.n; ++i){
- back_convolve(input, layer.kernels[i], layer.stride, i, layer.output);
+ back_convolve(in_delta, layer.kernels[i], layer.stride, i, out_delta);
+ }
+ for(i = 0; i < layer.h*layer.w*layer.c; ++i){
+ in_delta.data[i] *= layer.gradient(in_image.data[i]);
}
}
+/*
void backpropagate_convolutional_layer_convolve(image input, convolutional_layer layer)
{
int i,j;
@@ -66,25 +107,26 @@
rotate_image(layer.kernels[i]);
}
}
+*/
-void learn_convolutional_layer(image input, convolutional_layer layer)
+void learn_convolutional_layer(convolutional_layer layer, double *input)
{
int i;
+ image in_image = double_to_image(layer.h, layer.w, layer.c, input);
+ image out_delta = get_convolutional_delta(layer);
for(i = 0; i < layer.n; ++i){
- kernel_update(input, layer.kernel_updates[i], layer.stride, i, layer.output);
+ kernel_update(in_image, layer.kernel_updates[i], layer.stride, i, out_delta);
+ layer.bias_updates[i] += avg_image_layer(out_delta, i);
}
- image old_input = copy_image(input);
- backpropagate_convolutional_layer(input, layer);
- for(i = 0; i < input.h*input.w*input.c; ++i){
- input.data[i] *= convolution_gradient(old_input.data[i]);
- }
- free_image(old_input);
}
void update_convolutional_layer(convolutional_layer layer, double step)
{
+ return;
int i,j;
for(i = 0; i < layer.n; ++i){
+ layer.biases[i] += step*layer.bias_updates[i];
+ layer.bias_updates[i] = 0;
int pixels = layer.kernels[i].h*layer.kernels[i].w*layer.kernels[i].c;
for(j = 0; j < pixels; ++j){
layer.kernels[i].data[j] += step*layer.kernel_updates[i].data[j];
@@ -93,3 +135,16 @@
}
}
+void visualize_convolutional_layer(convolutional_layer layer)
+{
+ int i;
+ char buff[256];
+ //image vis = make_image(layer.n*layer.size, layer.size*layer.kernels[0].c, 3);
+ for(i = 0; i < layer.n; ++i){
+ image k = layer.kernels[i];
+ sprintf(buff, "Kernel %d", i);
+ if(k.c <= 3) show_image(k, buff);
+ else show_image_layers(k, buff);
+ }
+}
+
diff --git a/src/convolutional_layer.h b/src/convolutional_layer.h
index 75be04b..ab414ec 100644
--- a/src/convolutional_layer.h
+++ b/src/convolutional_layer.h
@@ -2,22 +2,36 @@
#define CONVOLUTIONAL_LAYER_H
#include "image.h"
+#include "activations.h"
typedef struct {
+ int h,w,c;
int n;
int stride;
image *kernels;
image *kernel_updates;
+ double *biases;
+ double *bias_updates;
image upsampled;
- image output;
+ double *delta;
+ double *output;
+
+ double (* activation)();
+ double (* gradient)();
} convolutional_layer;
-convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int size, int stride);
-void run_convolutional_layer(const image input, const convolutional_layer layer);
-void learn_convolutional_layer(image input, convolutional_layer layer);
+convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int size, int stride, ACTIVATION activator);
+void forward_convolutional_layer(const convolutional_layer layer, double *in);
+void backward_convolutional_layer(convolutional_layer layer, double *input, double *delta);
+void learn_convolutional_layer(convolutional_layer layer, double *input);
+
void update_convolutional_layer(convolutional_layer layer, double step);
-void backpropagate_convolutional_layer(image input, convolutional_layer layer);
+
void backpropagate_convolutional_layer_convolve(image input, convolutional_layer layer);
+void visualize_convolutional_layer(convolutional_layer layer);
+
+image get_convolutional_image(convolutional_layer layer);
+image get_convolutional_delta(convolutional_layer layer);
#endif
diff --git a/src/data.c b/src/data.c
new file mode 100644
index 0000000..7ef0d80
--- /dev/null
+++ b/src/data.c
@@ -0,0 +1,108 @@
+#include "data.h"
+#include "list.h"
+#include "utils.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+batch make_batch(int n, int k)
+{
+ batch b;
+ b.n = n;
+ if(k < 3) k = 1;
+ b.images = calloc(n, sizeof(image));
+ b.truth = calloc(n, sizeof(double *));
+ int i;
+ for(i =0 ; i < n; ++i) b.truth[i] = calloc(k, sizeof(double));
+ return b;
+}
+
+list *get_paths(char *filename)
+{
+ char *path;
+ FILE *file = fopen(filename, "r");
+ list *lines = make_list();
+ while((path=fgetl(file))){
+ list_insert(lines, path);
+ }
+ fclose(file);
+ return lines;
+}
+
+int get_truth(char *path)
+{
+ if(strstr(path, "dog")) return 1;
+ return 0;
+}
+
+batch load_list(list *paths)
+{
+ char *path;
+ batch data = make_batch(paths->size, 2);
+ node *n = paths->front;
+ int i;
+ for(i = 0; i < data.n; ++i){
+ path = (char *)n->val;
+ data.images[i] = load_image(path);
+ data.truth[i][0] = get_truth(path);
+ n = n->next;
+ }
+ return data;
+}
+
+batch get_all_data(char *filename)
+{
+ list *paths = get_paths(filename);
+ batch b = load_list(paths);
+ free_list_contents(paths);
+ free_list(paths);
+ return b;
+}
+
+void free_batch(batch b)
+{
+ int i;
+ for(i = 0; i < b.n; ++i){
+ free_image(b.images[i]);
+ free(b.truth[i]);
+ }
+ free(b.images);
+ free(b.truth);
+}
+
+batch get_batch(char *filename, int curr, int total)
+{
+ list *plist = get_paths(filename);
+ char **paths = (char **)list_to_array(plist);
+ int i;
+ int start = curr*plist->size/total;
+ int end = (curr+1)*plist->size/total;
+ batch b = make_batch(end-start, 2);
+ for(i = start; i < end; ++i){
+ b.images[i-start] = load_image(paths[i]);
+ b.truth[i-start][0] = get_truth(paths[i]);
+ }
+ free_list_contents(plist);
+ free_list(plist);
+ free(paths);
+ return b;
+}
+
+batch random_batch(char *filename, int n)
+{
+ list *plist = get_paths(filename);
+ char **paths = (char **)list_to_array(plist);
+ int i;
+ batch b = make_batch(n, 2);
+ for(i = 0; i < n; ++i){
+ int index = rand()%plist->size;
+ b.images[i] = load_image(paths[index]);
+ normalize_image(b.images[i]);
+ b.truth[i][0] = get_truth(paths[index]);
+ }
+ free_list_contents(plist);
+ free_list(plist);
+ free(paths);
+ return b;
+}
diff --git a/src/data.h b/src/data.h
new file mode 100644
index 0000000..fbcb144
--- /dev/null
+++ b/src/data.h
@@ -0,0 +1,18 @@
+#ifndef DATA_H
+#define DATA_H
+
+#include "image.h"
+
+typedef struct{
+ int n;
+ image *images;
+ double **truth;
+} batch;
+
+batch get_all_data(char *filename);
+batch random_batch(char *filename, int n);
+batch get_batch(char *filename, int curr, int total);
+void free_batch(batch b);
+
+
+#endif
diff --git a/src/image.c b/src/image.c
index a1aa8a7..a509d32 100644
--- a/src/image.c
+++ b/src/image.c
@@ -34,6 +34,18 @@
p.data[i+j*p.h*p.w] = (p.data[i+j*p.h*p.w] - min[j])/(max[j]-min[j]);
}
}
+ free(min);
+ free(max);
+}
+
+double avg_image_layer(image m, int l)
+{
+ int i;
+ double sum = 0;
+ for(i = 0; i < m.h*m.w; ++i){
+ sum += m.data[l*m.h*m.w + i];
+ }
+ return sum/(m.h*m.w);
}
void threshold_image(image p, double t)
@@ -95,16 +107,29 @@
}
}
-image make_image(int h, int w, int c)
+image make_empty_image(int h, int w, int c)
{
image out;
out.h = h;
out.w = w;
out.c = c;
+ return out;
+}
+
+image make_image(int h, int w, int c)
+{
+ image out = make_empty_image(h,w,c);
out.data = calloc(h*w*c, sizeof(double));
return out;
}
+image double_to_image(int h, int w, int c, double *data)
+{
+ image out = make_empty_image(h,w,c);
+ out.data = data;
+ return out;
+}
+
void zero_image(image m)
{
memset(m.data, 0, m.h*m.w*m.c*sizeof(double));
@@ -132,7 +157,7 @@
image out = make_image(h,w,c);
int i;
for(i = 0; i < h*w*c; ++i){
- out.data[i] = .5-(double)rand()/RAND_MAX;
+ out.data[i] = (.5-(double)rand()/RAND_MAX);
}
return out;
}
diff --git a/src/image.h b/src/image.h
index e2fe8c0..3117ded 100644
--- a/src/image.h
+++ b/src/image.h
@@ -15,13 +15,16 @@
void zero_image(image m);
void rotate_image(image m);
void subtract_image(image a, image b);
+double avg_image_layer(image m, int l);
void show_image(image p, char *name);
void show_image_layers(image p, char *name);
image make_image(int h, int w, int c);
+image make_empty_image(int h, int w, int c);
image make_random_image(int h, int w, int c);
image make_random_kernel(int size, int c);
+image double_to_image(int h, int w, int c, double *data);
image copy_image(image p);
image load_image(char *filename);
diff --git a/src/list.c b/src/list.c
new file mode 100644
index 0000000..948d960
--- /dev/null
+++ b/src/list.c
@@ -0,0 +1,90 @@
+#include <stdlib.h>
+#include <string.h>
+#include "list.h"
+
+list *make_list()
+{
+ list *l = malloc(sizeof(list));
+ l->size = 0;
+ l->front = 0;
+ l->back = 0;
+ return l;
+}
+
+void transfer_node(list *s, list *d, node *n)
+{
+ node *prev, *next;
+ prev = n->prev;
+ next = n->next;
+ if(prev) prev->next = next;
+ if(next) next->prev = prev;
+ --s->size;
+ if(s->front == n) s->front = next;
+ if(s->back == n) s->back = prev;
+}
+
+void *list_pop(list *l){
+ if(!l->back) return 0;
+ node *b = l->back;
+ void *val = b->val;
+ l->back = b->prev;
+ if(l->back) l->back->next = 0;
+ free(b);
+ --l->size;
+
+ return val;
+}
+
+void list_insert(list *l, void *val)
+{
+ node *new = malloc(sizeof(node));
+ new->val = val;
+ new->next = 0;
+
+ if(!l->back){
+ l->front = new;
+ new->prev = 0;
+ }else{
+ l->back->next = new;
+ new->prev = l->back;
+ }
+ l->back = new;
+ ++l->size;
+}
+
+void free_node(node *n)
+{
+ node *next;
+ while(n) {
+ next = n->next;
+ free(n);
+ n = next;
+ }
+}
+
+void free_list(list *l)
+{
+ free_node(l->front);
+ free(l);
+}
+
+void free_list_contents(list *l)
+{
+ node *n = l->front;
+ while(n){
+ free(n->val);
+ n = n->next;
+ }
+}
+
+void **list_to_array(list *l)
+{
+ void **a = calloc(l->size, sizeof(void*));
+ int count = 0;
+ node *n = l->front;
+ while(n){
+ a[count++] = n->val;
+ n = n->next;
+ }
+ return a;
+}
diff --git a/src/list.h b/src/list.h
new file mode 100644
index 0000000..fb818c2
--- /dev/null
+++ b/src/list.h
@@ -0,0 +1,26 @@
+#ifndef LIST_H
+#define LIST_H
+
+typedef struct node{
+ void *val;
+ struct node *next;
+ struct node *prev;
+} node;
+
+typedef struct list{
+ int size;
+ node *front;
+ node *back;
+} list;
+
+list *make_list();
+int list_find(list *l, void *val);
+
+void list_insert(list *, void *);
+
+void **list_to_array(list *l);
+
+void free_list(list *l);
+void free_list_contents(list *l);
+
+#endif
diff --git a/src/matrix.c b/src/matrix.c
new file mode 100644
index 0000000..562a364
--- /dev/null
+++ b/src/matrix.c
@@ -0,0 +1,106 @@
+#include "matrix.h"
+#include "utils.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+#include <math.h>
+
+void free_matrix(matrix m)
+{
+ int i;
+ for(i = 0; i < m.rows; ++i) free(m.vals[i]);
+ free(m.vals);
+}
+
+matrix make_matrix(int rows, int cols)
+{
+ matrix m;
+ m.rows = rows;
+ m.cols = cols;
+ m.vals = calloc(m.rows, sizeof(double *));
+ int i;
+ for(i = 0; i < m.rows; ++i) m.vals[i] = calloc(m.cols, sizeof(double));
+ return m;
+}
+
+matrix hold_out_matrix(matrix *m, int n)
+{
+ int i;
+ matrix h;
+ h.rows = n;
+ h.cols = m->cols;
+ h.vals = calloc(h.rows, sizeof(double *));
+ for(i = 0; i < n; ++i){
+ int index = rand()%m->rows;
+ h.vals[i] = m->vals[index];
+ m->vals[index] = m->vals[--(m->rows)];
+ }
+ return h;
+}
+
+double *pop_column(matrix *m, int c)
+{
+ double *col = calloc(m->rows, sizeof(double));
+ int i, j;
+ for(i = 0; i < m->rows; ++i){
+ col[i] = m->vals[i][c];
+ for(j = c; j < m->cols-1; ++j){
+ m->vals[i][j] = m->vals[i][j+1];
+ }
+ }
+ --m->cols;
+ return col;
+}
+
+matrix csv_to_matrix(char *filename)
+{
+ FILE *fp = fopen(filename, "r");
+ if(!fp) file_error(filename);
+
+ matrix m;
+ m.cols = -1;
+
+ char *line;
+
+ int n = 0;
+ int size = 1024;
+ m.vals = calloc(size, sizeof(double*));
+ while((line = fgetl(fp))){
+ if(m.cols == -1) m.cols = count_fields(line);
+ if(n == size){
+ size *= 2;
+ m.vals = realloc(m.vals, size*sizeof(double*));
+ }
+ m.vals[n] = parse_fields(line, m.cols);
+ free(line);
+ ++n;
+ }
+ m.vals = realloc(m.vals, n*sizeof(double*));
+ m.rows = n;
+ return m;
+}
+
+void print_matrix(matrix m)
+{
+ int i, j;
+ printf("%d X %d Matrix:\n",m.rows, m.cols);
+ printf(" __");
+ for(j = 0; j < 16*m.cols-1; ++j) printf(" ");
+ printf("__ \n");
+
+ printf("| ");
+ for(j = 0; j < 16*m.cols-1; ++j) printf(" ");
+ printf(" |\n");
+
+ for(i = 0; i < m.rows; ++i){
+ printf("| ");
+ for(j = 0; j < m.cols; ++j){
+ printf("%15.7f ", m.vals[i][j]);
+ }
+ printf(" |\n");
+ }
+ printf("|__");
+ for(j = 0; j < 16*m.cols-1; ++j) printf(" ");
+ printf("__|\n");
+}
diff --git a/src/matrix.h b/src/matrix.h
new file mode 100644
index 0000000..182135a
--- /dev/null
+++ b/src/matrix.h
@@ -0,0 +1,17 @@
+#ifndef MATRIX_H
+#define MATRIX_H
+typedef struct matrix{
+ int rows, cols;
+ double **vals;
+} matrix;
+
+matrix make_matrix(int rows, int cols);
+void free_matrix(matrix m);
+void print_matrix(matrix m);
+
+matrix csv_to_matrix(char *filename);
+matrix hold_out_matrix(matrix *m, int n);
+
+double *pop_column(matrix *m, int c);
+
+#endif
diff --git a/src/maxpool_layer.c b/src/maxpool_layer.c
index 6f7d2a2..f58a22f 100644
--- a/src/maxpool_layer.c
+++ b/src/maxpool_layer.c
@@ -1,24 +1,41 @@
#include "maxpool_layer.h"
+#include <stdio.h>
+
+image get_maxpool_image(maxpool_layer layer)
+{
+ int h = (layer.h-1)/layer.stride + 1;
+ int w = (layer.w-1)/layer.stride + 1;
+ int c = layer.c;
+ return double_to_image(h,w,c,layer.output);
+}
maxpool_layer *make_maxpool_layer(int h, int w, int c, int stride)
{
+ printf("Maxpool Layer: %d x %d x %d image, %d stride\n", h,w,c,stride);
maxpool_layer *layer = calloc(1, sizeof(maxpool_layer));
+ layer->h = h;
+ layer->w = w;
+ layer->c = c;
layer->stride = stride;
- layer->output = make_image((h-1)/stride+1, (w-1)/stride+1, c);
+ layer->output = calloc(((h-1)/stride+1) * ((w-1)/stride+1) * c, sizeof(double));
+ layer->delta = calloc(((h-1)/stride+1) * ((w-1)/stride+1) * c, sizeof(double));
return layer;
}
-void run_maxpool_layer(const image input, const maxpool_layer layer)
+void forward_maxpool_layer(const maxpool_layer layer, double *in)
{
+ image input = double_to_image(layer.h, layer.w, layer.c, in);
+ image output = get_maxpool_image(layer);
int i,j,k;
- for(i = 0; i < layer.output.h*layer.output.w*layer.output.c; ++i) layer.output.data[i] = -DBL_MAX;
- for(i = 0; i < input.h; ++i){
- for(j = 0; j < input.w; ++j){
- for(k = 0; k < input.c; ++k){
+ for(i = 0; i < output.h*output.w*output.c; ++i) output.data[i] = -DBL_MAX;
+ for(k = 0; k < input.c; ++k){
+ for(i = 0; i < input.h; ++i){
+ for(j = 0; j < input.w; ++j){
double val = get_pixel(input, i, j, k);
- double cur = get_pixel(layer.output, i/layer.stride, j/layer.stride, k);
- if(val > cur) set_pixel(layer.output, i/layer.stride, j/layer.stride, k, val);
+ double cur = get_pixel(output, i/layer.stride, j/layer.stride, k);
+ if(val > cur) set_pixel(output, i/layer.stride, j/layer.stride, k, val);
}
}
}
}
+
diff --git a/src/maxpool_layer.h b/src/maxpool_layer.h
index 4d7726d..04fb4b4 100644
--- a/src/maxpool_layer.h
+++ b/src/maxpool_layer.h
@@ -4,12 +4,15 @@
#include "image.h"
typedef struct {
+ int h,w,c;
int stride;
- image output;
+ double *delta;
+ double *output;
} maxpool_layer;
+image get_maxpool_image(maxpool_layer layer);
maxpool_layer *make_maxpool_layer(int h, int w, int c, int stride);
-void run_maxpool_layer(const image input, const maxpool_layer layer);
+void forward_maxpool_layer(const maxpool_layer layer, double *in);
#endif
diff --git a/src/network.c b/src/network.c
index 53184d9..a77d607 100644
--- a/src/network.c
+++ b/src/network.c
@@ -1,5 +1,7 @@
+#include <stdio.h>
#include "network.h"
#include "image.h"
+#include "data.h"
#include "connected_layer.h"
#include "convolutional_layer.h"
@@ -14,27 +16,24 @@
return net;
}
-void run_network(image input, network net)
+void forward_network(network net, double *input)
{
int i;
- double *input_d = input.data;
for(i = 0; i < net.n; ++i){
if(net.types[i] == CONVOLUTIONAL){
convolutional_layer layer = *(convolutional_layer *)net.layers[i];
- run_convolutional_layer(input, layer);
+ forward_convolutional_layer(layer, input);
input = layer.output;
- input_d = layer.output.data;
}
else if(net.types[i] == CONNECTED){
connected_layer layer = *(connected_layer *)net.layers[i];
- run_connected_layer(input_d, layer);
- input_d = layer.output;
+ forward_connected_layer(layer, input);
+ input = layer.output;
}
else if(net.types[i] == MAXPOOL){
maxpool_layer layer = *(maxpool_layer *)net.layers[i];
- run_maxpool_layer(input, layer);
+ forward_maxpool_layer(layer, input);
input = layer.output;
- input_d = layer.output.data;
}
}
}
@@ -52,74 +51,112 @@
}
else if(net.types[i] == CONNECTED){
connected_layer layer = *(connected_layer *)net.layers[i];
- update_connected_layer(layer, step);
+ update_connected_layer(layer, step, .3, 0);
}
}
}
-void learn_network(image input, network net)
+double *get_network_output_layer(network net, int i)
+{
+ if(net.types[i] == CONVOLUTIONAL){
+ convolutional_layer layer = *(convolutional_layer *)net.layers[i];
+ return layer.output;
+ } else if(net.types[i] == MAXPOOL){
+ maxpool_layer layer = *(maxpool_layer *)net.layers[i];
+ return layer.output;
+ } else if(net.types[i] == CONNECTED){
+ connected_layer layer = *(connected_layer *)net.layers[i];
+ return layer.output;
+ }
+ return 0;
+}
+double *get_network_output(network net)
+{
+ return get_network_output_layer(net, net.n-1);
+}
+
+double *get_network_delta_layer(network net, int i)
+{
+ if(net.types[i] == CONVOLUTIONAL){
+ convolutional_layer layer = *(convolutional_layer *)net.layers[i];
+ return layer.delta;
+ } else if(net.types[i] == MAXPOOL){
+ maxpool_layer layer = *(maxpool_layer *)net.layers[i];
+ return layer.delta;
+ } else if(net.types[i] == CONNECTED){
+ connected_layer layer = *(connected_layer *)net.layers[i];
+ return layer.delta;
+ }
+ return 0;
+}
+
+double *get_network_delta(network net)
+{
+ return get_network_delta_layer(net, net.n-1);
+}
+
+void learn_network(network net, double *input)
{
int i;
- image prev;
- double *prev_p;
+ double *prev_input;
+ double *prev_delta;
for(i = net.n-1; i >= 0; --i){
if(i == 0){
- prev = input;
- prev_p = prev.data;
- } else if(net.types[i-1] == CONVOLUTIONAL){
- convolutional_layer layer = *(convolutional_layer *)net.layers[i-1];
- prev = layer.output;
- prev_p = prev.data;
- } else if(net.types[i-1] == MAXPOOL){
- maxpool_layer layer = *(maxpool_layer *)net.layers[i-1];
- prev = layer.output;
- prev_p = prev.data;
- } else if(net.types[i-1] == CONNECTED){
- connected_layer layer = *(connected_layer *)net.layers[i-1];
- prev_p = layer.output;
+ prev_input = input;
+ prev_delta = 0;
+ }else{
+ prev_input = get_network_output_layer(net, i-1);
+ prev_delta = get_network_delta_layer(net, i-1);
}
-
if(net.types[i] == CONVOLUTIONAL){
convolutional_layer layer = *(convolutional_layer *)net.layers[i];
- learn_convolutional_layer(prev, layer);
+ learn_convolutional_layer(layer, prev_input);
+ if(i != 0) backward_convolutional_layer(layer, prev_input, prev_delta);
}
else if(net.types[i] == MAXPOOL){
//maxpool_layer layer = *(maxpool_layer *)net.layers[i];
}
else if(net.types[i] == CONNECTED){
connected_layer layer = *(connected_layer *)net.layers[i];
- learn_connected_layer(prev_p, layer);
+ learn_connected_layer(layer, prev_input);
+ if(i != 0) backward_connected_layer(layer, prev_input, prev_delta);
}
}
}
-
-double *get_network_output_layer(network net, int i)
+void train_network_batch(network net, batch b)
{
- if(net.types[i] == CONVOLUTIONAL){
- convolutional_layer layer = *(convolutional_layer *)net.layers[i];
- return layer.output.data;
+ int i,j;
+ int k = get_network_output_size(net);
+ int correct = 0;
+ for(i = 0; i < b.n; ++i){
+ forward_network(net, b.images[i].data);
+ image o = get_network_image(net);
+ double *output = get_network_output(net);
+ double *delta = get_network_delta(net);
+ for(j = 0; j < k; ++j){
+ //printf("%f %f\n", b.truth[i][j], output[j]);
+ delta[j] = b.truth[i][j]-output[j];
+ if(fabs(delta[j]) < .5) ++correct;
+ //printf("%f\n", output[j]);
+ }
+ learn_network(net, b.images[i].data);
+ update_network(net, .00001);
}
- else if(net.types[i] == MAXPOOL){
- maxpool_layer layer = *(maxpool_layer *)net.layers[i];
- return layer.output.data;
- }
- else if(net.types[i] == CONNECTED){
- connected_layer layer = *(connected_layer *)net.layers[i];
- return layer.output;
- }
- return 0;
+ printf("Accuracy: %f\n", (double)correct/b.n);
}
int get_network_output_size_layer(network net, int i)
{
if(net.types[i] == CONVOLUTIONAL){
convolutional_layer layer = *(convolutional_layer *)net.layers[i];
- return layer.output.h*layer.output.w*layer.output.c;
+ image output = get_convolutional_image(layer);
+ return output.h*output.w*output.c;
}
else if(net.types[i] == MAXPOOL){
maxpool_layer layer = *(maxpool_layer *)net.layers[i];
- return layer.output.h*layer.output.w*layer.output.c;
+ image output = get_maxpool_image(layer);
+ return output.h*output.w*output.c;
}
else if(net.types[i] == CONNECTED){
connected_layer layer = *(connected_layer *)net.layers[i];
@@ -128,21 +165,21 @@
return 0;
}
-double *get_network_output(network net)
+int get_network_output_size(network net)
{
int i = net.n-1;
- return get_network_output_layer(net, i);
+ return get_network_output_size_layer(net, i);
}
image get_network_image_layer(network net, int i)
{
if(net.types[i] == CONVOLUTIONAL){
convolutional_layer layer = *(convolutional_layer *)net.layers[i];
- return layer.output;
+ return get_convolutional_image(layer);
}
else if(net.types[i] == MAXPOOL){
maxpool_layer layer = *(maxpool_layer *)net.layers[i];
- return layer.output;
+ return get_maxpool_image(layer);
}
return make_image(0,0,0);
}
@@ -151,15 +188,20 @@
{
int i;
for(i = net.n-1; i >= 0; --i){
- if(net.types[i] == CONVOLUTIONAL){
- convolutional_layer layer = *(convolutional_layer *)net.layers[i];
- return layer.output;
- }
- else if(net.types[i] == MAXPOOL){
- maxpool_layer layer = *(maxpool_layer *)net.layers[i];
- return layer.output;
- }
+ image m = get_network_image_layer(net, i);
+ if(m.h != 0) return m;
}
return make_image(1,1,1);
}
+void visualize_network(network net)
+{
+ int i;
+ for(i = 0; i < 1; ++i){
+ if(net.types[i] == CONVOLUTIONAL){
+ convolutional_layer layer = *(convolutional_layer *)net.layers[i];
+ visualize_convolutional_layer(layer);
+ }
+ }
+}
+
diff --git a/src/network.h b/src/network.h
index ad2b1dc..10fa6c5 100644
--- a/src/network.h
+++ b/src/network.h
@@ -3,6 +3,7 @@
#define NETWORK_H
#include "image.h"
+#include "data.h"
typedef enum {
CONVOLUTIONAL,
@@ -17,12 +18,16 @@
} network;
network make_network(int n);
-void run_network(image input, network net);
-void learn_network(image input, network net);
+void forward_network(network net, double *input);
+void learn_network(network net, double *input);
void update_network(network net, double step);
+void train_network_batch(network net, batch b);
double *get_network_output(network net);
double *get_network_output_layer(network net, int i);
+double *get_network_delta_layer(network net, int i);
+double *get_network_delta(network net);
int get_network_output_size_layer(network net, int i);
+int get_network_output_size(network net);
image get_network_image(network net);
image get_network_image_layer(network net, int i);
diff --git a/src/option_list.c b/src/option_list.c
new file mode 100644
index 0000000..1b32ebb
--- /dev/null
+++ b/src/option_list.c
@@ -0,0 +1,68 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "option_list.h"
+
+typedef struct{
+ char *key;
+ char *val;
+ int used;
+} kvp;
+
+void option_insert(list *l, char *key, char *val)
+{
+ kvp *p = malloc(sizeof(kvp));
+ p->key = key;
+ p->val = val;
+ p->used = 0;
+ list_insert(l, p);
+}
+
+void option_unused(list *l)
+{
+ node *n = l->front;
+ while(n){
+ kvp *p = (kvp *)n->val;
+ if(!p->used){
+ fprintf(stderr, "Unused field: '%s = %s'\n", p->key, p->val);
+ }
+ n = n->next;
+ }
+}
+
+char *option_find(list *l, char *key)
+{
+ node *n = l->front;
+ while(n){
+ kvp *p = (kvp *)n->val;
+ if(strcmp(p->key, key) == 0){
+ p->used = 1;
+ return p->val;
+ }
+ n = n->next;
+ }
+ return 0;
+}
+char *option_find_str(list *l, char *key, char *def)
+{
+ char *v = option_find(l, key);
+ if(v) return v;
+ fprintf(stderr, "%s: Using default '%s'\n", key, def);
+ return def;
+}
+
+int option_find_int(list *l, char *key, int def)
+{
+ char *v = option_find(l, key);
+ if(v) return atoi(v);
+ fprintf(stderr, "%s: Using default '%d'\n", key, def);
+ return def;
+}
+
+double option_find_double(list *l, char *key, double def)
+{
+ char *v = option_find(l, key);
+ if(v) return atof(v);
+ fprintf(stderr, "%s: Using default '%lf'\n", key, def);
+ return def;
+}
diff --git a/src/option_list.h b/src/option_list.h
new file mode 100644
index 0000000..0270465
--- /dev/null
+++ b/src/option_list.h
@@ -0,0 +1,12 @@
+#ifndef OPTION_LIST_H
+#define OPTION_LIST_H
+#include "list.h"
+
+void option_insert(list *l, char *key, char *val);
+char *option_find(list *l, char *key);
+char *option_find_str(list *l, char *key, char *def);
+int option_find_int(list *l, char *key, int def);
+double option_find_double(list *l, char *key, double def);
+void option_unused(list *l);
+
+#endif
diff --git a/src/parser.c b/src/parser.c
new file mode 100644
index 0000000..7541620
--- /dev/null
+++ b/src/parser.c
@@ -0,0 +1,168 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "parser.h"
+#include "activations.h"
+#include "convolutional_layer.h"
+#include "connected_layer.h"
+#include "maxpool_layer.h"
+#include "list.h"
+#include "option_list.h"
+#include "utils.h"
+
+typedef struct{
+ char *type;
+ list *options;
+}section;
+
+int is_convolutional(section *s);
+int is_connected(section *s);
+int is_maxpool(section *s);
+list *read_cfg(char *filename);
+
+
+network parse_network_cfg(char *filename)
+{
+ list *sections = read_cfg(filename);
+ network net = make_network(sections->size);
+
+ node *n = sections->front;
+ int count = 0;
+ while(n){
+ section *s = (section *)n->val;
+ list *options = s->options;
+ if(is_convolutional(s)){
+ int h,w,c;
+ int n = option_find_int(options, "filters",1);
+ int size = option_find_int(options, "size",1);
+ int stride = option_find_int(options, "stride",1);
+ char *activation_s = option_find_str(options, "activation", "sigmoid");
+ ACTIVATION activation = get_activation(activation_s);
+ if(count == 0){
+ h = option_find_int(options, "height",1);
+ w = option_find_int(options, "width",1);
+ c = option_find_int(options, "channels",1);
+ }else{
+ image m = get_network_image_layer(net, count-1);
+ h = m.h;
+ w = m.w;
+ c = m.c;
+ if(h == 0) error("Layer before convolutional layer must output image.");
+ }
+ convolutional_layer *layer = make_convolutional_layer(h,w,c,n,size,stride, activation);
+ net.types[count] = CONVOLUTIONAL;
+ net.layers[count] = layer;
+ option_unused(options);
+ }
+ else if(is_connected(s)){
+ int input;
+ int output = option_find_int(options, "output",1);
+ char *activation_s = option_find_str(options, "activation", "sigmoid");
+ ACTIVATION activation = get_activation(activation_s);
+ if(count == 0){
+ input = option_find_int(options, "input",1);
+ }else{
+ input = get_network_output_size_layer(net, count-1);
+ }
+ connected_layer *layer = make_connected_layer(input, output, activation);
+ net.types[count] = CONNECTED;
+ net.layers[count] = layer;
+ option_unused(options);
+ }else if(is_maxpool(s)){
+ int h,w,c;
+ int stride = option_find_int(options, "stride",1);
+ //char *activation_s = option_find_str(options, "activation", "sigmoid");
+ if(count == 0){
+ h = option_find_int(options, "height",1);
+ w = option_find_int(options, "width",1);
+ c = option_find_int(options, "channels",1);
+ }else{
+ image m = get_network_image_layer(net, count-1);
+ h = m.h;
+ w = m.w;
+ c = m.c;
+ if(h == 0) error("Layer before convolutional layer must output image.");
+ }
+ maxpool_layer *layer = make_maxpool_layer(h,w,c,stride);
+ net.types[count] = MAXPOOL;
+ net.layers[count] = layer;
+ option_unused(options);
+ }else{
+ fprintf(stderr, "Type not recognized: %s\n", s->type);
+ }
+ ++count;
+ n = n->next;
+ }
+ return net;
+}
+
+int is_convolutional(section *s)
+{
+ return (strcmp(s->type, "[conv]")==0
+ || strcmp(s->type, "[convolutional]")==0);
+}
+int is_connected(section *s)
+{
+ return (strcmp(s->type, "[conn]")==0
+ || strcmp(s->type, "[connected]")==0);
+}
+int is_maxpool(section *s)
+{
+ return (strcmp(s->type, "[max]")==0
+ || strcmp(s->type, "[maxpool]")==0);
+}
+
+int read_option(char *s, list *options)
+{
+ int i;
+ int len = strlen(s);
+ char *val = 0;
+ for(i = 0; i < len; ++i){
+ if(s[i] == '='){
+ s[i] = '\0';
+ val = s+i+1;
+ break;
+ }
+ }
+ if(i == len-1) return 0;
+ char *key = s;
+ option_insert(options, key, val);
+ return 1;
+}
+
+list *read_cfg(char *filename)
+{
+ FILE *file = fopen(filename, "r");
+ if(file == 0) file_error(filename);
+ char *line;
+ int nu = 0;
+ list *sections = make_list();
+ section *current = 0;
+ while((line=fgetl(file)) != 0){
+ ++ nu;
+ strip(line);
+ switch(line[0]){
+ case '[':
+ current = malloc(sizeof(section));
+ list_insert(sections, current);
+ current->options = make_list();
+ current->type = line;
+ break;
+ case '\0':
+ case '#':
+ case ';':
+ free(line);
+ break;
+ default:
+ if(!read_option(line, current->options)){
+ printf("Config file error line %d, could parse: %s\n", nu, line);
+ free(line);
+ }
+ break;
+ }
+ }
+ fclose(file);
+ return sections;
+}
+
diff --git a/src/parser.h b/src/parser.h
new file mode 100644
index 0000000..878baa3
--- /dev/null
+++ b/src/parser.h
@@ -0,0 +1,7 @@
+#ifndef PARSER_H
+#define PARSER_H
+#include "network.h"
+
+network parse_network_cfg(char *filename);
+
+#endif
diff --git a/src/tests.c b/src/tests.c
index 0e639be..65811e9 100644
--- a/src/tests.c
+++ b/src/tests.c
@@ -4,6 +4,8 @@
#include "network.h"
#include "image.h"
#include "parser.h"
+#include "data.h"
+#include "matrix.h"
#include <time.h>
#include <stdlib.h>
@@ -40,18 +42,19 @@
int n = 3;
int stride = 1;
int size = 3;
- convolutional_layer layer = *make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride);
+ convolutional_layer layer = *make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride, RELU);
char buff[256];
for(i = 0; i < n; ++i) {
sprintf(buff, "Kernel %d", i);
show_image(layer.kernels[i], buff);
}
- run_convolutional_layer(dog, layer);
+ forward_convolutional_layer(layer, dog.data);
- maxpool_layer mlayer = *make_maxpool_layer(layer.output.h, layer.output.w, layer.output.c, 2);
- run_maxpool_layer(layer.output,mlayer);
+ image output = get_convolutional_image(layer);
+ maxpool_layer mlayer = *make_maxpool_layer(output.h, output.w, output.c, 2);
+ forward_maxpool_layer(mlayer, layer.output);
- show_image_layers(mlayer.output, "Test Maxpool Layer");
+ show_image_layers(get_maxpool_image(mlayer), "Test Maxpool Layer");
}
void test_load()
@@ -90,168 +93,144 @@
show_image(random, "Test Rotate Random");
}
-void test_network()
-{
- network net;
- net.n = 11;
- net.layers = calloc(net.n, sizeof(void *));
- net.types = calloc(net.n, sizeof(LAYER_TYPE));
- net.types[0] = CONVOLUTIONAL;
- net.types[1] = MAXPOOL;
- net.types[2] = CONVOLUTIONAL;
- net.types[3] = MAXPOOL;
- net.types[4] = CONVOLUTIONAL;
- net.types[5] = CONVOLUTIONAL;
- net.types[6] = CONVOLUTIONAL;
- net.types[7] = MAXPOOL;
- net.types[8] = CONNECTED;
- net.types[9] = CONNECTED;
- net.types[10] = CONNECTED;
-
- image dog = load_image("test_hinton.jpg");
-
- int n = 48;
- int stride = 4;
- int size = 11;
- convolutional_layer cl = *make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride);
- maxpool_layer ml = *make_maxpool_layer(cl.output.h, cl.output.w, cl.output.c, 2);
-
- n = 128;
- size = 5;
- stride = 1;
- convolutional_layer cl2 = *make_convolutional_layer(ml.output.h, ml.output.w, ml.output.c, n, size, stride);
- maxpool_layer ml2 = *make_maxpool_layer(cl2.output.h, cl2.output.w, cl2.output.c, 2);
-
- n = 192;
- size = 3;
- convolutional_layer cl3 = *make_convolutional_layer(ml2.output.h, ml2.output.w, ml2.output.c, n, size, stride);
- convolutional_layer cl4 = *make_convolutional_layer(cl3.output.h, cl3.output.w, cl3.output.c, n, size, stride);
- n = 128;
- convolutional_layer cl5 = *make_convolutional_layer(cl4.output.h, cl4.output.w, cl4.output.c, n, size, stride);
- maxpool_layer ml3 = *make_maxpool_layer(cl5.output.h, cl5.output.w, cl5.output.c, 4);
- connected_layer nl = *make_connected_layer(ml3.output.h*ml3.output.w*ml3.output.c, 4096, RELU);
- connected_layer nl2 = *make_connected_layer(4096, 4096, RELU);
- connected_layer nl3 = *make_connected_layer(4096, 1000, RELU);
-
- net.layers[0] = &cl;
- net.layers[1] = &ml;
- net.layers[2] = &cl2;
- net.layers[3] = &ml2;
- net.layers[4] = &cl3;
- net.layers[5] = &cl4;
- net.layers[6] = &cl5;
- net.layers[7] = &ml3;
- net.layers[8] = &nl;
- net.layers[9] = &nl2;
- net.layers[10] = &nl3;
-
- int i;
- clock_t start = clock(), end;
- for(i = 0; i < 10; ++i){
- run_network(dog, net);
- rotate_image(dog);
- }
- end = clock();
- printf("Ran %lf second per iteration\n", (double)(end-start)/CLOCKS_PER_SEC/10);
-
- show_image_layers(get_network_image(net), "Test Network Layer");
-}
-
-void test_backpropagate()
-{
- int n = 3;
- int size = 4;
- int stride = 10;
- image dog = load_image("dog.jpg");
- show_image(dog, "Test Backpropagate Input");
- image dog_copy = copy_image(dog);
- convolutional_layer cl = *make_convolutional_layer(dog.h, dog.w, dog.c, n, size, stride);
- run_convolutional_layer(dog, cl);
- show_image(cl.output, "Test Backpropagate Output");
- int i;
- clock_t start = clock(), end;
- for(i = 0; i < 100; ++i){
- backpropagate_convolutional_layer(dog_copy, cl);
- }
- end = clock();
- printf("Backpropagate: %lf seconds\n", (double)(end-start)/CLOCKS_PER_SEC);
- start = clock();
- for(i = 0; i < 100; ++i){
- backpropagate_convolutional_layer_convolve(dog, cl);
- }
- end = clock();
- printf("Backpropagate Using Convolutions: %lf seconds\n", (double)(end-start)/CLOCKS_PER_SEC);
- show_image(dog_copy, "Test Backpropagate 1");
- show_image(dog, "Test Backpropagate 2");
- subtract_image(dog, dog_copy);
- show_image(dog, "Test Backpropagate Difference");
-}
-
-void test_ann()
-{
- network net;
- net.n = 3;
- net.layers = calloc(net.n, sizeof(void *));
- net.types = calloc(net.n, sizeof(LAYER_TYPE));
- net.types[0] = CONNECTED;
- net.types[1] = CONNECTED;
- net.types[2] = CONNECTED;
-
- connected_layer nl = *make_connected_layer(1, 20, RELU);
- connected_layer nl2 = *make_connected_layer(20, 20, RELU);
- connected_layer nl3 = *make_connected_layer(20, 1, RELU);
-
- net.layers[0] = &nl;
- net.layers[1] = &nl2;
- net.layers[2] = &nl3;
-
- image t = make_image(1,1,1);
- int count = 0;
-
- double avgerr = 0;
- while(1){
- double v = ((double)rand()/RAND_MAX);
- double truth = v*v;
- set_pixel(t,0,0,0,v);
- run_network(t, net);
- double *out = get_network_output(net);
- double err = pow((out[0]-truth),2.);
- avgerr = .99 * avgerr + .01 * err;
- //if(++count % 100000 == 0) printf("%f\n", avgerr);
- if(++count % 100000 == 0) printf("%f %f :%f AVG %f \n", truth, out[0], err, avgerr);
- out[0] = truth - out[0];
- learn_network(t, net);
- update_network(net, .001);
- }
-
-}
-
void test_parser()
{
- network net = parse_network_cfg("test.cfg");
- image t = make_image(1,1,1);
+ network net = parse_network_cfg("test_parser.cfg");
+ double input[1];
int count = 0;
double avgerr = 0;
while(1){
double v = ((double)rand()/RAND_MAX);
double truth = v*v;
- set_pixel(t,0,0,0,v);
- run_network(t, net);
+ input[0] = v;
+ forward_network(net, input);
double *out = get_network_output(net);
+ double *delta = get_network_delta(net);
double err = pow((out[0]-truth),2.);
avgerr = .99 * avgerr + .01 * err;
//if(++count % 100000 == 0) printf("%f\n", avgerr);
- if(++count % 100000 == 0) printf("%f %f :%f AVG %f \n", truth, out[0], err, avgerr);
- out[0] = truth - out[0];
- learn_network(t, net);
+ if(++count % 1000000 == 0) printf("%f %f :%f AVG %f \n", truth, out[0], err, avgerr);
+ delta[0] = truth - out[0];
+ learn_network(net, input);
update_network(net, .001);
}
}
+void test_data()
+{
+ batch train = random_batch("train_paths.txt", 101);
+ show_image(train.images[0], "Test Data Loading");
+ show_image(train.images[100], "Test Data Loading");
+ show_image(train.images[10], "Test Data Loading");
+ free_batch(train);
+}
+
+void test_train()
+{
+ network net = parse_network_cfg("test.cfg");
+ srand(0);
+ //visualize_network(net);
+ int i = 1000;
+ //while(1){
+ while(i > 0){
+ batch train = random_batch("train_paths.txt", 100);
+ train_network_batch(net, train);
+ //show_image_layers(get_network_image(net), "hey");
+ //visualize_network(net);
+ //cvWaitKey(0);
+ free_batch(train);
+ --i;
+ }
+ //}
+}
+
+double error_network(network net, matrix m, double *truth)
+{
+ int i;
+ int correct = 0;
+ for(i = 0; i < m.rows; ++i){
+ forward_network(net, m.vals[i]);
+ double *out = get_network_output(net);
+ double err = truth[i] - out[0];
+ if(fabs(err) < .5) ++correct;
+ }
+ return (double)correct/m.rows;
+}
+
+void classify_random_filters()
+{
+ network net = parse_network_cfg("random_filter_finish.cfg");
+ matrix m = csv_to_matrix("train.csv");
+ matrix ho = hold_out_matrix(&m, 2500);
+ double *truth = pop_column(&m, 0);
+ double *ho_truth = pop_column(&ho, 0);
+ int i;
+ clock_t start = clock(), end;
+ int count = 0;
+ while(++count <= 300){
+ for(i = 0; i < m.rows; ++i){
+ int index = rand()%m.rows;
+ //image p = double_to_image(1690,1,1,m.vals[index]);
+ //normalize_image(p);
+ forward_network(net, m.vals[index]);
+ double *out = get_network_output(net);
+ double *delta = get_network_delta(net);
+ //printf("%f\n", out[0]);
+ delta[0] = truth[index] - out[0];
+ // printf("%f\n", delta[0]);
+ //printf("%f %f\n", truth[index], out[0]);
+ learn_network(net, m.vals[index]);
+ update_network(net, .000005);
+ }
+ double test_acc = error_network(net, m, truth);
+ double valid_acc = error_network(net, ho, ho_truth);
+ printf("%f, %f\n", test_acc, valid_acc);
+ fprintf(stderr, "%5d: %f Valid: %f\n",count, test_acc, valid_acc);
+ //if(valid_acc > .70) break;
+ }
+ end = clock();
+ FILE *fp = fopen("submission/out.txt", "w");
+ matrix test = csv_to_matrix("test.csv");
+ truth = pop_column(&test, 0);
+ for(i = 0; i < test.rows; ++i){
+ forward_network(net, test.vals[i]);
+ double *out = get_network_output(net);
+ if(fabs(out[0]) < .5) fprintf(fp, "0\n");
+ else fprintf(fp, "1\n");
+ }
+ fclose(fp);
+ printf("Neural Net Learning: %lf seconds\n", (double)(end-start)/CLOCKS_PER_SEC);
+}
+
+void test_random_filters()
+{
+ FILE *file = fopen("test.csv", "w");
+ int i,j,k;
+ srand(0);
+ network net = parse_network_cfg("test_random_filter.cfg");
+ for(i = 0; i < 100; ++i){
+ printf("%d\n", i);
+ batch part = get_batch("test_paths.txt", i, 100);
+ for(j = 0; j < part.n; ++j){
+ forward_network(net, part.images[j].data);
+ double *out = get_network_output(net);
+ fprintf(file, "%f", part.truth[j][0]);
+ for(k = 0; k < get_network_output_size(net); ++k){
+ fprintf(file, ",%f", out[k]);
+ }
+ fprintf(file, "\n");
+ }
+ free_batch(part);
+ }
+}
+
int main()
{
- test_parser();
+ //classify_random_filters();
+ //test_random_filters();
+ test_train();
+ //test_parser();
//test_backpropagate();
//test_ann();
//test_convolve();
diff --git a/src/utils.c b/src/utils.c
new file mode 100644
index 0000000..9848d08
--- /dev/null
+++ b/src/utils.c
@@ -0,0 +1,147 @@
+#include "utils.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+void error(char *s)
+{
+ fprintf(stderr, "Error: %s\n", s);
+ exit(0);
+}
+
+void malloc_error()
+{
+ fprintf(stderr, "Malloc error\n");
+ exit(-1);
+}
+
+void file_error(char *s)
+{
+ fprintf(stderr, "Couldn't open file: %s\n", s);
+ exit(0);
+}
+
+list *split_str(char *s, char delim)
+{
+ int i;
+ int len = strlen(s);
+ list *l = make_list();
+ list_insert(l, s);
+ for(i = 0; i < len; ++i){
+ if(s[i] == delim){
+ s[i] = '\0';
+ list_insert(l, &(s[i+1]));
+ }
+ }
+ return l;
+}
+
+void strip(char *s)
+{
+ int i;
+ int len = strlen(s);
+ int offset = 0;
+ for(i = 0; i < len; ++i){
+ char c = s[i];
+ if(c==' '||c=='\t'||c=='\n') ++offset;
+ else s[i-offset] = c;
+ }
+ s[len-offset] = '\0';
+}
+
+void strip_char(char *s, char bad)
+{
+ int i;
+ int len = strlen(s);
+ int offset = 0;
+ for(i = 0; i < len; ++i){
+ char c = s[i];
+ if(c==bad) ++offset;
+ else s[i-offset] = c;
+ }
+ s[len-offset] = '\0';
+}
+
+char *fgetl(FILE *fp)
+{
+ if(feof(fp)) return 0;
+ int size = 512;
+ char *line = malloc(size*sizeof(char));
+ if(!fgets(line, size, fp)){
+ free(line);
+ return 0;
+ }
+
+ int curr = strlen(line);
+
+ while(line[curr-1]!='\n'){
+ size *= 2;
+ line = realloc(line, size*sizeof(char));
+ if(!line) malloc_error();
+ fgets(&line[curr], size-curr, fp);
+ curr = strlen(line);
+ }
+ line[curr-1] = '\0';
+
+ return line;
+}
+
+char *copy_string(char *s)
+{
+ char *copy = malloc(strlen(s)+1);
+ strncpy(copy, s, strlen(s)+1);
+ return copy;
+}
+
+list *parse_csv_line(char *line)
+{
+ list *l = make_list();
+ char *c, *p;
+ int in = 0;
+ for(c = line, p = line; *c != '\0'; ++c){
+ if(*c == '"') in = !in;
+ else if(*c == ',' && !in){
+ *c = '\0';
+ list_insert(l, copy_string(p));
+ p = c+1;
+ }
+ }
+ list_insert(l, copy_string(p));
+ return l;
+}
+
+int count_fields(char *line)
+{
+ int count = 0;
+ int done = 0;
+ char *c;
+ for(c = line; !done; ++c){
+ done = (*c == '\0');
+ if(*c == ',' || done) ++count;
+ }
+ return count;
+}
+
+double *parse_fields(char *line, int n)
+{
+ double *field = calloc(n, sizeof(double));
+ char *c, *p, *end;
+ int count = 0;
+ int done = 0;
+ for(c = line, p = line; !done; ++c){
+ done = (*c == '\0');
+ if(*c == ',' || done){
+ *c = '\0';
+ field[count] = strtod(p, &end);
+ if(p == c) field[count] = nan("");
+ if(end != c && (end != c-1 || *end != '\r')) field[count] = nan(""); //DOS file formats!
+ p = c+1;
+ ++count;
+ }
+ }
+ return field;
+}
+
+
+
diff --git a/src/utils.h b/src/utils.h
new file mode 100644
index 0000000..87ef428
--- /dev/null
+++ b/src/utils.h
@@ -0,0 +1,18 @@
+#ifndef UTILS_H
+#define UTILS_H
+#include <stdio.h>
+#include "list.h"
+
+void error(char *s);
+void malloc_error();
+void file_error(char *s);
+void strip(char *s);
+void strip_char(char *s, char bad);
+list *split_str(char *s, char delim);
+char *fgetl(FILE *fp);
+list *parse_csv_line(char *line);
+char *copy_string(char *s);
+int count_fields(char *line);
+double *parse_fields(char *line, int n);
+#endif
+
diff --git a/test.cfg b/test.cfg
new file mode 100644
index 0000000..84b6c82
--- /dev/null
+++ b/test.cfg
@@ -0,0 +1,37 @@
+[conv]
+width=200
+height=200
+channels=3
+filters=10
+size=3
+stride=2
+activation=relu
+
+[maxpool]
+stride=2
+
+[conv]
+filters=10
+size=10
+stride=2
+activation=relu
+
+[maxpool]
+stride=2
+
+[conv]
+filters=10
+size=10
+stride=2
+activation=relu
+
+[maxpool]
+stride=2
+
+[conn]
+output = 10
+activation=relu
+
+[conn]
+output = 1
+activation=relu
diff --git a/test_parser.cfg b/test_parser.cfg
new file mode 100644
index 0000000..788d71a
--- /dev/null
+++ b/test_parser.cfg
@@ -0,0 +1,8 @@
+[conn]
+input=1
+output = 20
+activation=sigmoid
+
+[conn]
+output = 1
+activation=sigmoid
diff --git a/test_random_filter.cfg b/test_random_filter.cfg
new file mode 100644
index 0000000..bfd7f0c
--- /dev/null
+++ b/test_random_filter.cfg
@@ -0,0 +1,29 @@
+[conv]
+width=200
+height=200
+channels=3
+filters=10
+size=15
+stride=2
+activation=relu
+
+[maxpool]
+stride=2
+
+[conv]
+filters=10
+size=5
+stride=1
+activation=relu
+
+[maxpool]
+stride=2
+
+[conv]
+filters=10
+size=3
+stride=1
+activation=relu
+
+[maxpool]
+stride=2
--
Gitblit v1.10.0