抑制过拟合的方法之Dropout(随机删除神经元)

        在抑制过拟合的方法中&＃xff0c;我们前面有讲到一个方法&＃xff1a;抑制过拟合的方法之权值衰减 ,在某种程度上能够很好的抑制过拟合&＃xff0c;如果神经网络的模型很复杂&＃xff0c;只用权值衰减就难以应对了&＃xff0c;这样的情况下&＃xff0c;我们一般选择Dropout方法&＃xff0c;也就是在训练的过程中&＃xff0c;随机选出隐藏层的神经元&＃xff0c;然后将其删除&＃xff0c;被删除的神经元不再进行信号的传递。代码在权值衰减中有出现&＃xff0c;layers.py里面&＃xff0c;摘录出来

class Dropout:&＃39;&＃39;&＃39;随机删除神经元self.mask&＃xff1a;保存的是False和True的数组&＃xff0c;False的值为0是删除的数据&＃39;&＃39;&＃39;def __init__(self,dropout_ratio&＃61;0.5):self.dropout_ratio&＃61;dropout_ratioself.mask&＃61;Nonedef forward(self,x,train_flg&＃61;True):if train_flg:self.mask&＃61;np.random.rand(*x.shape)>self.dropout_ratioreturn x*self.maskelse:return x*(1.0-self.dropout_ratio)def backward(self,dout):return dout*self.mask

随机删除的意思是指每次正向传播时&＃xff0c;self.mask中都会以False的形式保存要删除的神经元。

np.random.rand(2,3) 随机生成[0,1)形状为(2,3)的数组
np.random.rand(2,3)>0.5 把大于0.5的值设为True,其余为False&＃xff08;而不是删除一半的意思&＃xff0c;因为数据是随机的&＃xff09;
x * self.mask 结果就是False为0&＃xff0c;True还是x原来的值

        正向传播时传递了信号的神经元&＃xff0c;反向传播时按照原样传递信号&＃xff0c;正向传播时没有传递信号的神经元&＃xff0c;反向传播时信号将停在那里。
        现在我们来比较使用Dropout和不使用Dropout的情况&＃xff0c;还是基于MNIST数据集来测试
训练类&＃xff08;common.trainer.py&＃xff09;

import numpy as np
from common.optimizer import *class Trainer:&＃39;&＃39;&＃39;把前面用来训练的代码做一个类&＃39;&＃39;&＃39;def __init__(self,network,x_train,t_train,x_test,t_test,epochs&＃61;20,mini_batch_size&＃61;100,optimizer&＃61;&＃39;SGD&＃39;,optimizer_param&＃61;{&＃39;lr&＃39;:0.01},evaluate_sample_num_per_epoch&＃61;None,verbose&＃61;True):self.network&＃61;networkself.verbose&＃61;verbose#是否打印数据(调试或查看)self.x_train&＃61;x_trainself.t_train&＃61;t_trainself.x_test&＃61;x_testself.t_test&＃61;t_testself.epochs&＃61;epochsself.batch_size&＃61;mini_batch_sizeself.evaluate_sample_num_per_epoch&＃61;evaluate_sample_num_per_epochoptimizer_dict&＃61;{&＃39;sgd&＃39;:SGD,&＃39;momentum&＃39;:Momentum,&＃39;nesterov&＃39;:Nesterov,&＃39;adagrad&＃39;:AdaGrad,&＃39;rmsprop&＃39;:RMSprop,&＃39;adam&＃39;:Adam}self.optimizer&＃61;optimizer_dict[optimizer.lower()](**optimizer_param)self.train_size&＃61;x_train.shape[0]self.iter_per_epoch&＃61;max(self.train_size/mini_batch_size,1)self.max_iter&＃61;int(epochs*self.iter_per_epoch)self.current_iter&＃61;0self.current_epoch&＃61;0self.train_loss_list&＃61;[]self.train_acc_list&＃61;[]self.test_acc_list&＃61;[]def train_step(self):batch_mask&＃61;np.random.choice(self.train_size,self.batch_size)x_batch&＃61;self.x_train[batch_mask]t_batch&＃61;self.t_train[batch_mask]grads&＃61;self.network.gradient(x_batch,t_batch)self.optimizer.update(self.network.params,grads)loss&＃61;self.network.loss(x_batch,t_batch)self.train_loss_list.append(loss)if self.verbose:print(&＃39;训练损失值:&＃39;&＃43;str(loss))if self.current_iter%self.iter_per_epoch&＃61;&＃61;0:self.current_epoch&＃43;&＃61;1x_train_sample,t_train_sample&＃61;self.x_train,self.t_trainx_test_sample,t_test_sample&＃61;self.x_test,self.t_testif not self.evaluate_sample_num_per_epoch is None:t&＃61;self.evaluate_sample_num_per_epochx_train_sample,t_train_sample&＃61;self.x_test[:t],self.t_test[:t] train_acc&＃61;self.network.accuracy(x_train_sample,t_train_sample)test_acc&＃61;self.network.accuracy(x_test_sample,t_test_sample) self.train_acc_list.append(train_acc)self.test_acc_list.append(test_acc)if self.verbose:print(&＃39;epoch:&＃39;&＃43;str(self.current_epoch)&＃43;&＃39;,train acc:&＃39;&＃43;str(train_acc)&＃43;&＃39; | test acc:&＃39;&＃43;str(test_acc))self.current_iter&＃43;&＃61;1def train(self):for i in range(self.max_iter):self.train_step()test_acc&＃61;self.network.accuracy(self.x_test,self.t_test)if self.verbose:print(&＃39;最终测试的正确率:&＃39;&＃43;str(format(test_acc,&＃39;.2%&＃39;)))

import numpy as np
import matplotlib.pyplot as plt
from dataset.mnist import load_mnist
from common.multi_layer_net_extend import MultiLayerNetExtend
from common.trainer import Trainer(x_train,t_train),(x_test,t_test)&＃61;load_mnist(normalize&＃61;True)
#截取少量数据&＃xff0c;让它再现过拟合
x_train&＃61;x_train[:300]#(300,784)
t_train&＃61;t_train[:300]#构建7层神经网络(6个隐藏层)
epochsNum&＃61;300
network&＃61;MultiLayerNetExtend(inputSize&＃61;784,hiddenSizeList&＃61;[100,100,100,100,100,100],outputSize&＃61;10,use_dropout&＃61;True,dropout_ration&＃61;0.2)
trainer&＃61;Trainer(network,x_train,t_train,x_test,t_test,epochs&＃61;epochsNum,mini_batch_size&＃61;100,optimizer&＃61;&＃39;sgd&＃39;,optimizer_param&＃61;{&＃39;lr&＃39;:0.01},verbose&＃61;True)
trainer.train()#画图
train_acc_list,test_acc_list&＃61;trainer.train_acc_list,trainer.test_acc_list
x&＃61;np.arange(len(train_acc_list))
plt.plot(x,train_acc_list,marker&＃61;&＃39;s&＃39;,label&＃61;&＃39;train&＃39;,markevery&＃61;10)
plt.plot(x,test_acc_list,marker&＃61;&＃39;d&＃39;,label&＃61;&＃39;test&＃39;,markevery&＃61;10)
plt.xlabel(&＃39;epochs&＃39;)
plt.ylabel(&＃39;accuracy&＃39;)
plt.ylim(0,1.0)
plt.legend(loc&＃61;&＃39;lower right&＃39;)
plt.show()


        不使用Dropout的情况&＃xff0c;修改成use_dropout&＃61;False&＃xff0c;我们会发现下图中&＃xff0c;train数据过拟合了&＃xff0c;所以很多时候我们都会优选Dropout来抑制过拟合。

其中需要用到的多层神经网络扩展版本&＃xff08;支持Dropout&＃xff09;multi_layer_net_extend.py

基于MNIST数据集的Batch Normalization(批标准化层)https://blog.csdn.net/weixin_41896770/article/details/121557928