1、dropout
dropout 是指在深度学习网络的训练过程中，按照一定的概率将一部分神经网络单元暂时从网络中丢弃，相当于从原始的网络中找到一个更瘦的网络，这篇博客中讲的非常详细
2、tensorflow实现
用dropout：
import tensorflow as tf
import numpy as np x_data=np.linspace(-1.,1.,300)[:, np.newaxis]
noise=np.random.normal(0,0.05,x_data.shape)
y_data=np.square(x_data)+noise def add_layer(input,in_size,out_size,activation_function=None,dropout_function=False):
Weights=tf.Variable(tf.random_normal([in_size,out_size]))
biases=tf.Variable(tf.zeros([1,out_size])+0.1)
Wx_plus_b=tf.matmul(input,Weights)+biases if dropout_function==True:
Wx_plus_b=tf.nn.dropout(Wx_plus_b,keep_prob=0.5)
else:
pass if activation_function is None:
outputs=Wx_plus_b
else:
outputs=activation_function(Wx_plus_b)
return outputs xs = tf.placeholder(tf.float32, [None, 1])
ys = tf.placeholder(tf.float32, [None, 1]) layer1=add_layer(xs,1,10,activation_function=tf.nn.relu,dropout_function=True)
predict=add_layer(layer1,10,1) loss=tf.reduce_mean(tf.reduce_sum(tf.square(ys-predict),reduction_indices=[1]))
train_step=tf.train.GradientDescentOptimizer(0.1).minimize(loss) init=tf.global_variables_initializer()
sess=tf.Session()
sess.run(init) for i in range(100):
sess.run(train_step,feed_dict={xs: x_data,ys: y_data})
if i%5==0:
print(sess.run(loss,feed_dict={xs:x_data,ys:y_data}))
输出结果：
1.87953
0.294975
0.173229
0.122831
0.0859246
0.0712419
0.0839021
0.0572673
0.0676766
0.0584741
0.0556422
0.0576011
0.0534098
0.0472549
0.0528496
0.0427771
0.0485248
0.0502765
0.0454531
0.0516633
不用dropout：
import tensorflow as tf
import numpy as np x_data=np.linspace(-1.,1.,300)[:, np.newaxis]
noise=np.random.normal(0,0.05,x_data.shape)
y_data=np.square(x_data)+noise def add_layer(input,in_size,out_size,activation_function=None,dropout_function=False):
Weights=tf.Variable(tf.random_normal([in_size,out_size]))
biases=tf.Variable(tf.zeros([1,out_size])+0.1)
Wx_plus_b=tf.matmul(input,Weights)+biases if dropout_function==True:
Wx_plus_b=tf.nn.dropout(Wx_plus_b,keep_prob=1)
else:
pass if activation_function is None:
outputs=Wx_plus_b
else:
outputs=activation_function(Wx_plus_b)
return outputs xs = tf.placeholder(tf.float32, [None, 1])
ys = tf.placeholder(tf.float32, [None, 1]) layer1=add_layer(xs,1,10,activation_function=tf.nn.relu)
predict=add_layer(layer1,10,1) loss=tf.reduce_mean(tf.reduce_sum(tf.square(ys-predict),reduction_indices=[1]))
train_step=tf.train.GradientDescentOptimizer(0.1).minimize(loss) init=tf.global_variables_initializer()
sess=tf.Session()
sess.run(init) for i in range(100):
sess.run(train_step,feed_dict={xs: x_data,ys: y_data})
if i%5==0:
print(sess.run(loss,feed_dict={xs:x_data,ys:y_data}))
输出结果：
0.256632
0.0532672
0.0364148
0.0264385
0.0202409
0.0164314
0.0141813
0.0129079
0.0121878
0.0117478
0.0114399
0.0111906
0.0109715
0.0107702
0.0105785
0.010393
0.0102144
0.0100426
0.00987562
0.00971407
这里举例 用过dropout后，loss变得更大，是因为我这里所用数据较少，用更多的训练集去测试，dropout会变现的很好！