From 352ae7e65b6a74bcd768aa88b866a44c713284c8 Mon Sep 17 00:00:00 2001
From: Joseph Redmon <pjreddie@gmail.com>
Date: Wed, 26 Oct 2016 15:35:44 +0000
Subject: [PATCH] ADAM
---
src/convolutional_kernels.cu | 21 +++++++++++++++++----
1 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/src/convolutional_kernels.cu b/src/convolutional_kernels.cu
index b8d6478..76a3fb3 100644
--- a/src/convolutional_kernels.cu
+++ b/src/convolutional_kernels.cu
@@ -233,7 +233,6 @@
void update_convolutional_layer_gpu(convolutional_layer layer, int batch, float learning_rate, float momentum, float decay)
{
int size = layer.size*layer.size*layer.c*layer.n;
-
axpy_ongpu(layer.n, learning_rate/batch, layer.bias_updates_gpu, 1, layer.biases_gpu, 1);
scal_ongpu(layer.n, momentum, layer.bias_updates_gpu, 1);
@@ -242,9 +241,23 @@
scal_ongpu(layer.n, momentum, layer.scale_updates_gpu, 1);
}
- axpy_ongpu(size, -decay*batch, layer.weights_gpu, 1, layer.weight_updates_gpu, 1);
- axpy_ongpu(size, learning_rate/batch, layer.weight_updates_gpu, 1, layer.weights_gpu, 1);
- scal_ongpu(size, momentum, layer.weight_updates_gpu, 1);
+ if(layer.adam){
+ scal_ongpu(size, layer.B1, layer.m_gpu, 1);
+ scal_ongpu(size, layer.B2, layer.v_gpu, 1);
+
+ axpy_ongpu(size, -decay*batch, layer.weights_gpu, 1, layer.weight_updates_gpu, 1);
+
+ axpy_ongpu(size, -(1-layer.B1), layer.weight_updates_gpu, 1, layer.m_gpu, 1);
+ mul_ongpu(size, layer.weight_updates_gpu, 1, layer.weight_updates_gpu, 1);
+ axpy_ongpu(size, (1-layer.B2), layer.weight_updates_gpu, 1, layer.v_gpu, 1);
+
+ adam_gpu(size, layer.weights_gpu, layer.m_gpu, layer.v_gpu, layer.B1, layer.B2, learning_rate/batch, layer.eps, layer.t+1);
+ fill_ongpu(size, 0, layer.weight_updates_gpu, 1);
+ }else{
+ axpy_ongpu(size, -decay*batch, layer.weights_gpu, 1, layer.weight_updates_gpu, 1);
+ axpy_ongpu(size, learning_rate/batch, layer.weight_updates_gpu, 1, layer.weights_gpu, 1);
+ scal_ongpu(size, momentum, layer.weight_updates_gpu, 1);
+ }
}
--
Gitblit v1.10.0