From c7b10ceadb1a78e7480d281444a31ae2a7dc1b05 Mon Sep 17 00:00:00 2001
From: Joseph Redmon <pjreddie@gmail.com>
Date: Fri, 06 May 2016 23:25:16 +0000
Subject: [PATCH] so much need to commit
---
src/rnn.c | 103 +++++++++++++++++++++++++++++++++++++--------------
1 files changed, 74 insertions(+), 29 deletions(-)
diff --git a/src/rnn.c b/src/rnn.c
index 30fa4bd..b72fafc 100644
--- a/src/rnn.c
+++ b/src/rnn.c
@@ -1,6 +1,7 @@
#include "network.h"
#include "cost_layer.h"
#include "utils.h"
+#include "blas.h"
#include "parser.h"
#ifdef OPENCV
@@ -12,29 +13,26 @@
float *y;
} float_pair;
-float_pair get_rnn_data(unsigned char *text, int characters, int len, int batch, int steps)
+float_pair get_rnn_data(unsigned char *text, size_t *offsets, int characters, size_t len, int batch, int steps)
{
float *x = calloc(batch * steps * characters, sizeof(float));
float *y = calloc(batch * steps * characters, sizeof(float));
int i,j;
for(i = 0; i < batch; ++i){
- int index = rand() %(len - steps - 1);
- /*
- int done = 1;
- while(!done){
- index = rand() %(len - steps - 1);
- while(index < len-steps-1 && text[index++] != '\n');
- if (index < len-steps-1) done = 1;
- }
- */
for(j = 0; j < steps; ++j){
- x[(j*batch + i)*characters + text[index + j]] = 1;
- y[(j*batch + i)*characters + text[index + j + 1]] = 1;
+ unsigned char curr = text[(offsets[i])%len];
+ unsigned char next = text[(offsets[i] + 1)%len];
- if(text[index+j] > 255 || text[index+j] <= 0 || text[index+j+1] > 255 || text[index+j+1] <= 0){
- text[index+j+2] = 0;
- printf("%d %d %d %d %d\n", index, j, len, (int)text[index+j], (int)text[index+j+1]);
+ x[(j*batch + i)*characters + curr] = 1;
+ y[(j*batch + i)*characters + next] = 1;
+
+ offsets[i] = (offsets[i] + 1) % len;
+
+ if(curr > 255 || curr <= 0 || next > 255 || next <= 0){
+ /*text[(index+j+2)%len] = 0;
+ printf("%ld %d %d %d %d\n", index, j, len, (int)text[index+j], (int)text[index+j+1]);
printf("%s", text+index);
+ */
error("Bad char");
}
}
@@ -45,8 +43,23 @@
return p;
}
-void train_char_rnn(char *cfgfile, char *weightfile, char *filename)
+void reset_rnn_state(network net, int b)
{
+ int i;
+ for (i = 0; i < net.n; ++i) {
+ layer l = net.layers[i];
+ #ifdef GPU
+ if(l.state_gpu){
+ fill_ongpu(l.outputs, 0, l.state_gpu + l.outputs*b, 1);
+ }
+ #endif
+ }
+}
+
+void train_char_rnn(char *cfgfile, char *weightfile, char *filename, int clear)
+{
+ srand(time(0));
+ data_seed = time(0);
FILE *fp = fopen(filename, "rb");
fseek(fp, 0, SEEK_END);
@@ -58,8 +71,6 @@
fclose(fp);
char *backup_directory = "/home/pjreddie/backup/";
- srand(time(0));
- data_seed = time(0);
char *base = basecfg(cfgfile);
fprintf(stderr, "%s\n", base);
float avg_loss = -1;
@@ -67,18 +78,26 @@
if(weightfile){
load_weights(&net, weightfile);
}
+
int inputs = get_network_input_size(net);
fprintf(stderr, "Learning Rate: %g, Momentum: %g, Decay: %g\n", net.learning_rate, net.momentum, net.decay);
int batch = net.batch;
int steps = net.time_steps;
- //*net.seen = 0;
+ if(clear) *net.seen = 0;
int i = (*net.seen)/net.batch;
+ int streams = batch/steps;
+ size_t *offsets = calloc(streams, sizeof(size_t));
+ int j;
+ for(j = 0; j < streams; ++j){
+ offsets[j] = rand_size_t()%size;
+ }
+
clock_t time;
while(get_current_batch(net) < net.max_batches){
i += 1;
time=clock();
- float_pair p = get_rnn_data(text, inputs, size, batch/steps, steps);
+ float_pair p = get_rnn_data(text, offsets, inputs, size, streams, steps);
float loss = train_network_datum(net, p.x, p.y) / (batch);
free(p.x);
@@ -86,7 +105,18 @@
if (avg_loss < 0) avg_loss = loss;
avg_loss = avg_loss*.9 + loss*.1;
- fprintf(stderr, "%d: %f, %f avg, %f rate, %lf seconds\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time));
+ int chars = get_current_batch(net)*batch;
+ fprintf(stderr, "%d: %f, %f avg, %f rate, %lf seconds, %f epochs\n", i, loss, avg_loss, get_current_rate(net), sec(clock()-time), (float) chars/size);
+
+ for(j = 0; j < streams; ++j){
+ //printf("%d\n", j);
+ if(rand()%10 == 0){
+ //fprintf(stderr, "Reset\n");
+ offsets[j] = rand_size_t()%size;
+ reset_rnn_state(net, j);
+ }
+ }
+
if(i%100==0){
char buff[256];
sprintf(buff, "%s/%s_%d.weights", backup_directory, base, i);
@@ -120,6 +150,15 @@
unsigned char c;
int len = strlen(seed);
float *input = calloc(inputs, sizeof(float));
+
+/*
+ fill_cpu(inputs, 0, input, 1);
+ for(i = 0; i < 10; ++i){
+ network_predict(net, input);
+ }
+ fill_cpu(inputs, 0, input, 1);
+ */
+
for(i = 0; i < len-1; ++i){
c = seed[i];
input[(int)c] = 1;
@@ -130,16 +169,16 @@
c = seed[len-1];
for(i = 0; i < num; ++i){
printf("%c", c);
- float r = rand_uniform(0,1);
- float sum = 0;
input[(int)c] = 1;
float *out = network_predict(net, input);
input[(int)c] = 0;
- for(j = 0; j < inputs; ++j){
- sum += out[j];
- if(sum > r) break;
+ for(j = 32; j < 127; ++j){
+ //printf("%d %c %f\n",j, j, out[j]);
}
- c = j;
+ for(j = 0; j < inputs; ++j){
+ //if (out[j] < .0001) out[j] = 0;
+ }
+ c = sample_array(out, inputs);
}
printf("\n");
}
@@ -158,11 +197,16 @@
int count = 0;
int c;
float *input = calloc(inputs, sizeof(float));
+ int i;
+ for(i = 0; i < 100; ++i){
+ network_predict(net, input);
+ }
float sum = 0;
c = getc(stdin);
float log2 = log(2);
while(c != EOF){
int next = getc(stdin);
+ if(next < 0 || next >= 255) error("Out of range character");
if(next == EOF) break;
++count;
input[c] = 1;
@@ -170,8 +214,8 @@
input[c] = 0;
sum += log(out[next])/log2;
c = next;
+ printf("%d Perplexity: %f\n", count, pow(2, -sum/count));
}
- printf("Perplexity: %f\n", pow(2, -sum/count));
}
@@ -186,10 +230,11 @@
int len = find_int_arg(argc, argv, "-len", 1000);
float temp = find_float_arg(argc, argv, "-temp", .7);
int rseed = find_int_arg(argc, argv, "-srand", time(0));
+ int clear = find_arg(argc, argv, "-clear");
char *cfg = argv[3];
char *weights = (argc > 4) ? argv[4] : 0;
- if(0==strcmp(argv[2], "train")) train_char_rnn(cfg, weights, filename);
+ if(0==strcmp(argv[2], "train")) train_char_rnn(cfg, weights, filename, clear);
else if(0==strcmp(argv[2], "valid")) valid_char_rnn(cfg, weights);
else if(0==strcmp(argv[2], "generate")) test_char_rnn(cfg, weights, len, seed, temp, rseed);
}
--
Gitblit v1.10.0