学习率自适应

DNN/flags.py

@@ -8,12 +8,14 @@ def parse_args(check=True):
  parser.add_argument('--keep_prob', type=float, default=config.keep_prob)
  parser.add_argument('--logfrequency', type=int, default=config.logfrequency)
  parser.add_argument('--Max_step', type=int, default=config.Max_step)
- parser.add_argument('--Max_epoch', type=int, default=config.Max_epoch)
+ parser.add_argument('--Max_epoch', type=int,  default=config.Max_epoch)
  parser.add_argument('--embed_dim', type=int, default=config.embed_dim)
  parser.add_argument('--learning_rate', type=float, default=config.learning_rate)
+ parser.add_argument('--decay_steps', type=int, default=config.decay_steps)
+ parser.add_argument('--decay_rate', type=float, default=config.decay_rate)
  parser.add_argument('--oridata_dim', type=int, default=config.oridata_dim)
- parser.add_argument('--valid_switch', type=int, default=config.valid_switch)
- parser.add_argument('--model_flag', type=str, default=config.model_flag)
+ parser.add_argument('--valid_switch', type=int,  default=config.valid_switch)
+ parser.add_argument('--model_flag', type=str,  default=config.model_flag)
  # 路径和文件配置
  parser.add_argument('--encod_train_path', type=str, default=config.encod_train_path)
  parser.add_argument('--encod_vaild_path', type=str, default=config.encod_vaild_path)

DNN/model.py

@@ -11,7 +11,7 @@
 
 # 实现模型的构建以及损失函数交叉熵的计算
 class Model():
- def __init__(self, learning_rate, oridata_dim, embed_max, embed_dim=128):
+ def __init__(self, learning_rate, oridata_dim, embed_max, embed_dim=128, decay_steps=5000, decay_rate=0.96):
  self.learning_rate = learning_rate
  self.layers = [6, 6, 6]
  self.oridata_dim = oridata_dim
@@ -20,6 +20,8 @@ def __init__(self, learning_rate, oridata_dim, embed_max, embed_dim=128):
  self.growth = 128
  self.outunits = 2048
  self.outrate = 0.5
+ self.decay_steps = decay_steps
+ self.decay_rate = decay_rate
 
  def embeding(self, embed_inputs):
  with tf.name_scope("embedding"):
@@ -183,7 +185,8 @@ def build(self):
  #self.loss = self.log_loss
  self.loss = tf.reduce_mean(self.log_loss)
 
- optimizer = tf.train.FtrlOptimizer(self.learning_rate)
+ step_learning_rate = tf.train.exponential_decay(self.learning_rate, self.global_step, self.decay_steps, self.decay_rate, staircase=True, name='step_learning_rate')
+ optimizer = tf.train.FtrlOptimizer(step_learning_rate)
  self.gradients = optimizer.compute_gradients(self.loss)
  self.train_step = optimizer.apply_gradients(self.gradients, global_step=self.global_step)
 

DNN/model2.py

@@ -3,12 +3,14 @@
 slim = tf.contrib.slim
 
 class Model():
- def __init__(self, learning_rate,oridata_dim,embed_max,embed_dim=128):
+ def __init__(self, learning_rate, oridata_dim, embed_max, embed_dim=128, decay_steps=5000, decay_rate=0.96):
  self.learning_rate = learning_rate
  self.outunits = [1024, 128, 8]
  self.oridata_dim = oridata_dim
  self.embed_dim = int(embed_dim)
  self.embed_max = int(embed_max)
+ self.decay_steps = decay_steps
+ self.decay_rate = decay_rate
 
  def embeding(self,embed_inputs):
  with tf.name_scope("embedding"):
@@ -81,11 +83,10 @@ def build(self):
  self.log_loss = tf.losses.log_loss(labels=self.label,predictions=self.end_points['prediction'])
  self.loss = tf.reduce_mean(self.log_loss)
 
- #返回logloss异常的数据索引
-
+ step_learning_rate = tf.train.exponential_decay(self.learning_rate, self.global_step, self.decay_steps, self.decay_rate, staircase=True, name='step_learning_rate')
  #self.train_step = tf.train.AdamOptimizer(self.learning_rate).minimize(self.loss,global_step=self.global_step)
  #self.optimizer = train_op.apply_gradients()
- optimizer = tf.train.FtrlOptimizer(self.learning_rate)
+ optimizer = tf.train.FtrlOptimizer(step_learning_rate)
  #结果证明使用adam优化器，在这个问题上效果比Ftrl优化器差很多
  #optimizer = tf.train.AdamOptimizer(self.learning_rate)
  self.gradients = optimizer.compute_gradients(self.loss)

DNN/train.py

@@ -47,9 +47,9 @@ def train_model(batch_size=FLAGS.batch_size):
 
  #构建网络模型
  if FLAGS.model_flag == 'model':
- dnnmodel = model.Model(learning_rate=FLAGS.learning_rate, oridata_dim=categorial_data.shape[1], embed_max=embed_max, embed_dim=FLAGS.embed_dim )
+ dnnmodel = model.Model(learning_rate=FLAGS.learning_rate, oridata_dim=categorial_data.shape[1], embed_max=embed_max, embed_dim=FLAGS.embed_dim, decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate )
  elif FLAGS.model_flag == 'model2':
- dnnmodel = model2.Model(learning_rate=FLAGS.learning_rate, oridata_dim=categorial_data.shape[1], embed_max=embed_max, embed_dim=FLAGS.embed_dim)
+ dnnmodel = model2.Model(learning_rate=FLAGS.learning_rate, oridata_dim=categorial_data.shape[1], embed_max=embed_max, embed_dim=FLAGS.embed_dim, decay_steps=FLAGS.decay_steps, decay_rate=FLAGS.decay_rate)
  dnnmodel.build()
 
  #如果没有checkpoint文件则需要对所有变量进行初始化
@@ -77,6 +77,7 @@ def train_model(batch_size=FLAGS.batch_size):
 
  #会按照配置内容来进行最大step之内的训练,到达Max_step自动停止训练
  global_step = 0
+ epoch = 1
  while 1 == 1:
  # 使用训练数据进行模型训练
  batches = utils.genbatch(inputs, lables, batch_size=FLAGS.batch_size)