From 76e258520edb50e8bb897ba15aa9467579e70a6a Mon Sep 17 00:00:00 2001
From: AlexeyAB <alexeyab84@gmail.com>
Date: Wed, 20 Jun 2018 10:28:25 +0000
Subject: [PATCH] Minor fix
---
src/convolutional_layer.c | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
1 files changed, 52 insertions(+), 3 deletions(-)
diff --git a/src/convolutional_layer.c b/src/convolutional_layer.c
index fb606ae..90f9551 100644
--- a/src/convolutional_layer.c
+++ b/src/convolutional_layer.c
@@ -177,6 +177,9 @@
// batch norm
cudnnSetTensor4dDescriptor(l->normTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, 1, l->out_c, 1, 1);
+ cudnnSetTensor4dDescriptor(l->normDstTensorDesc, CUDNN_TENSOR_NCHW, CUDNN_DATA_FLOAT, l->batch, l->out_c, l->out_h, l->out_w);
+
+ cudnnSetTensor4dDescriptor(l->normDstTensorDescF16, CUDNN_TENSOR_NCHW, data_type, l->batch, l->out_c, l->out_h, l->out_w);
#if(CUDNN_MAJOR >= 6)
cudnnSetConvolution2dDescriptor(l->convDesc, l->pad, l->pad, l->stride, l->stride, 1, 1, CUDNN_CROSS_CORRELATION, CUDNN_DATA_FLOAT); // cudnn >= 6.0
#else
@@ -190,6 +193,7 @@
forward_algo = CUDNN_CONVOLUTION_FWD_NO_WORKSPACE;
backward_algo = CUDNN_CONVOLUTION_BWD_DATA_NO_WORKSPACE;
backward_filter = CUDNN_CONVOLUTION_BWD_FILTER_NO_WORKSPACE;
+ printf(" CUDNN-slow ");
}
cudnnGetConvolutionForwardAlgorithm(cudnn_handle(),
@@ -216,6 +220,38 @@
backward_filter,
0,
&l->bf_algo);
+
+ if (data_type == CUDNN_DATA_HALF)
+ {
+ // HALF-16 if(data_type == CUDNN_DATA_HALF)
+ l->fw_algo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM;
+ l->bd_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_1;
+ l->bf_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1;
+
+ // FLOAT-32 if(data_type == CUDNN_DATA_FLOAT)
+ //l->fw_algo = CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED;
+ //l->bd_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED;
+ //l->bf_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED;
+
+ int fw = 0, bd = 0, bf = 0;
+ if (l->fw_algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM) fw = 1;
+ //printf("Tensor Cores - Forward enabled: l->fw_algo = CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM \n");
+ if (l->fw_algo == CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED) fw = 2;
+ //printf("Tensor Cores - Forward enabled: l->fw_algo = CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED \n");
+
+ if (l->bd_algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1) bd = 1;
+ //printf("Tensor Cores - Backward-data enabled: l->bd_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 \n");
+ if (l->bd_algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED) bd = 2;
+ //printf("Tensor Cores - Backward-data enabled: l->bd_algo = CUDNN_CONVOLUTION_BWD_DATA_ALGO_WINOGRAD_NONFUSED \n");
+
+ if (l->bf_algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1) bf = 1;
+ //printf("Tensor Cores - Backward-filter enabled: l->bf_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 \n");
+ if (l->bf_algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED) bf = 2;
+ //printf("Tensor Cores - Backward-filter enabled: l->bf_algo = CUDNN_CONVOLUTION_BWD_FILTER_ALGO_WINOGRAD_NONFUSED \n");
+
+ if (fw == 2 && bd == 2 && bf == 2) printf("TF ");
+ else if (fw == 1 && bd == 1 && bf == 1) printf("TH ");
+ }
}
#endif
#endif
@@ -343,7 +379,9 @@
l.x_gpu = cuda_make_array(l.output, l.batch*out_h*out_w*n);
l.x_norm_gpu = cuda_make_array(l.output, l.batch*out_h*out_w*n);
}
-#ifdef CUDNN
+#ifdef CUDNN
+ cudnnCreateTensorDescriptor(&l.normDstTensorDesc);
+ cudnnCreateTensorDescriptor(&l.normDstTensorDescF16);
cudnnCreateTensorDescriptor(&l.normTensorDesc);
cudnnCreateTensorDescriptor(&l.srcTensorDesc);
cudnnCreateTensorDescriptor(&l.dstTensorDesc);
@@ -359,7 +397,9 @@
l.workspace_size = get_workspace_size(l);
l.activation = activation;
- fprintf(stderr, "conv %5d %2d x%2d /%2d %4d x%4d x%4d -> %4d x%4d x%4d\n", n, size, size, stride, w, h, c, l.out_w, l.out_h, l.out_c);
+ //fprintf(stderr, "conv %5d %2d x%2d /%2d %4d x%4d x%4d -> %4d x%4d x%4d\n", n, size, size, stride, w, h, c, l.out_w, l.out_h, l.out_c);
+ l.bflops = (2.0 * l.n * l.size*l.size*l.c * l.out_h*l.out_w) / 1000000000.;
+ fprintf(stderr, "conv %5d %2d x%2d /%2d %4d x%4d x%4d -> %4d x%4d x%4d %5.3f BF\n", n, size, size, stride, w, h, c, l.out_w, l.out_h, l.out_c, l.bflops);
return l;
}
@@ -425,6 +465,10 @@
l->x_norm = realloc(l->x_norm, l->batch*l->outputs*sizeof(float));
}
+ if (l->xnor) {
+ //l->binary_input = realloc(l->inputs*l->batch, sizeof(float));
+ }
+
#ifdef GPU
if (old_w < w || old_h < h) {
cuda_free(l->delta_gpu);
@@ -440,6 +484,11 @@
l->x_gpu = cuda_make_array(l->output, l->batch*l->outputs);
l->x_norm_gpu = cuda_make_array(l->output, l->batch*l->outputs);
}
+
+ if (l->xnor) {
+ cuda_free(l->binary_input_gpu);
+ l->binary_input_gpu = cuda_make_array(0, l->inputs*l->batch);
+ }
}
#ifdef CUDNN
cudnn_convolutional_setup(l, cudnn_fastest);
@@ -453,7 +502,7 @@
size_t total_byte;
check_error(cudaMemGetInfo(&free_byte, &total_byte));
if (l->workspace_size > free_byte || l->workspace_size >= total_byte / 2) {
- printf(" used slow CUDNN algo without Workspace! \n");
+ printf(" used slow CUDNN algo without Workspace! Need memory: %zu, available: %zu\n", l->workspace_size, (free_byte < total_byte/2) ? free_byte : total_byte/2);
cudnn_convolutional_setup(l, cudnn_smallest);
l->workspace_size = get_workspace_size(*l);
}
--
Gitblit v1.10.0