BP神经网络是深度学习的重要基础，它是深度学习的重要前行算法之一，因此理解BP神经网络原理以及实现技巧非常有必要。接下来，我们对原理和实现展开讨论。
1.原理
有空再慢慢补上，请先参考老外一篇不错的文章：A Step by Step Backpropagation Example
激活函数参考：深度学习常用激活函数之— Sigmoid & ReLU & Softmax
浅显易懂的初始化：CS231n课程笔记翻译：神经网络笔记 2
有效的Trick：神经网络训练中的Tricks之高效BP（反向传播算法）
通过简单演示BPNN的计算过程：一文弄懂神经网络中的反向传播法——BackPropagation
2.实现----Batch随机梯度法
这里实现了层数可定义的BP神经网络，可通过参数net_struct进行定义网络结果，如定义只有输出层，没有隐藏层的网络结构，激活函数为”sigmoid",学习率，可如下定义
net_struct = [[10,"sigmoid",0.01]] # 网络结构
如定义一层隐藏层为100个神经元，再接一层隐藏层为50个神经元，输出层为10个神经元的网络结构，如下
net_struct = [[100,"sigmoid",0.01],[50,"sigmoid",0.01],[10,"sigmoid",0.01]] # 网络结构
码农最爱的实现如下：
# # encoding=utf8
'''
Created on 2017-7-3 @author: Administrator
'''
import random
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split as ttsplit class LossFun:
def __init__(self, lf_type="least_square"):
self.name = "loss function"
self.type = lf_type def cal(self, t, z):
loss = 0
if self.type == "least_square":
loss = self.least_square(t, z)
return loss def cal_deriv(self, t, z):
delta = 0
if self.type == "least_square":
delta = self.least_square_deriv(t, z)
return delta def least_square(self, t, z):
zsize = z.shape
sample_num = zsize[1]
return np.sum(0.5 * (t - z) * (t - z) * t) / sample_num def least_square_deriv(self, t, z):
return z - t class ActivationFun:
'''
激活函数
'''
def __init__(self, atype="sigmoid"):
self.name = "activation function library"
self.type = atype; def cal(self, a):
z = 0
if self.type == "sigmoid":
z = self.sigmoid(a)
elif self.type == "relu":
z = self.relu(a)
return z def cal_deriv(self, a):
z = 0
if self.type == "sigmoid":
z = self.sigmoid_deriv(a)
elif self.type == "relu":
z = self.relu_deriv(a)
return z def sigmoid(self, a):
return 1 / (1 + np.exp(-a)) def sigmoid_deriv(self, a):
fa = self.sigmoid(a)
return fa * (1 - fa) def relu(self, a):
idx = a <= 0
a[idx] = 0.1 * a[idx]
return a # np.maximum(a, 0.0) def relu_deriv(self, a):
# print a
a[a > 0] = 1.0
a[a <= 0] = 0.1
# print a
return a class Layer:
'''
神经网络层
'''
def __init__(self, num_neural, af_type="sigmoid", learn_rate=0.5):
self.af_type = af_type # active function type
self.learn_rate = learn_rate
self.num_neural = num_neural
self.dim = None
self.W = None self.a = None
self.X = None
self.z = None
self.delta = None
self.theta = None
self.act_fun = ActivationFun(self.af_type) def fp(self, X):
'''
Foward Propagation
'''
self.X = X
xsize = X.shape
self.dim = xsize[0]
self.num = xsize[1] if self.W == None:
# self.W = np.random.random((self.dim, self.num_neural))-0.5
# self.W = np.random.uniform(-1,1,size=(self.dim,self.num_neural))
if(self.af_type == "sigmoid"):
self.W = np.random.normal(0, 1, size=(self.dim, self.num_neural)) / np.sqrt(self.num)
elif(self.af_type == "relu"):
self.W = np.random.normal(0, 1, size=(self.dim, self.num_neural)) * np.sqrt(2.0 / self.num)
if self.theta == None:
# self.theta = np.random.random((self.num_neural, 1))-0.5
# self.theta = np.random.uniform(-1,1,size=(self.num_neural,1)) if(self.af_type == "sigmoid"):
self.theta = np.random.normal(0, 1, size=(self.num_neural, 1)) / np.sqrt(self.num)
elif(self.af_type == "relu"):
self.theta = np.random.normal(0, 1, size=(self.num_neural, 1)) * np.sqrt(2.0 / self.num)
# calculate the foreward a
self.a = (self.W.T).dot(self.X)
###calculate the foreward z####
self.z = self.act_fun.cal(self.a)
return self.z def bp(self, delta):
'''
Back Propagation
'''
self.delta = delta * self.act_fun.cal_deriv(self.a)
self.theta = np.array([np.mean(self.theta - self.learn_rate * self.delta, 1)]).T # 求所有样本的theta均值
dW = self.X.dot(self.delta.T) / self.num
self.W = self.W - self.learn_rate * dW
delta_out = self.W.dot(self.delta);
return delta_out class BpNet:
'''
BP神经网络
'''
def __init__(self, net_struct, stop_crit, max_iter, batch_size=10):
self.name = "net work"
self.net_struct = net_struct
if len(self.net_struct) == 0:
print "no layer is specified!"
return self.stop_crit = stop_crit
self.max_iter = max_iter
self.batch_size = batch_size
self.layers = []
self.num_layers = 0;
# 创建网络
self.create_net(net_struct)
self.loss_fun = LossFun("least_square"); def create_net(self, net_struct):
'''
创建网络
'''
self.num_layers = len(net_struct)
for i in range(self.num_layers):
self.layers.append(Layer(net_struct[i][0], net_struct[i][1], net_struct[i][2])) def train(self, X, t, Xtest=None, ttest=None):
'''
训练网络
'''
eva_acc_list = []
eva_loss_list = [] xshape = X.shape;
num = xshape[0]
dim = xshape[1] for k in range(self.max_iter):
# i = random.randint(0,num-1)
idxs = random.sample(range(num), self.batch_size)
xi = np.array([X[idxs, :]]).T[:, :, 0]
ti = np.array([t[idxs, :]]).T[:, :, 0]
# 前向计算
zi = self.fp(xi) # 偏差计算
delta_i = self.loss_fun.cal_deriv(ti, zi) # 反馈计算
self.bp(delta_i) # 评估精度
if Xtest != None:
if k % 100 == 0:
[eva_acc, eva_loss] = self.test(Xtest, ttest)
eva_acc_list.append(eva_acc)
eva_loss_list.append(eva_loss)
print "%4d,%4f,%4f" % (k, eva_acc, eva_loss)
else:
print "%4d" % (k)
return [eva_acc_list, eva_loss_list] def test(self, X, t):
'''
测试模型精度
'''
xshape = X.shape;
num = xshape[0]
z = self.fp_eval(X.T)
t = t.T
est_pos = np.argmax(z, 0)
real_pos = np.argmax(t, 0)
corrct_count = np.sum(est_pos == real_pos)
acc = 1.0 * corrct_count / num
loss = self.loss_fun.cal(t, z)
# print "%4f,loss:%4f"%(loss)
return [acc, loss] def fp(self, X):
'''
前向计算
'''
z = X
for i in range(self.num_layers):
z = self.layers[i].fp(z)
return z def bp(self, delta):
'''
反馈计算
'''
z = delta
for i in range(self.num_layers - 1, -1, -1):
z = self.layers[i].bp(z)
return z def fp_eval(self, X):
'''
前向计算
'''
layers = self.layers
z = X
for i in range(self.num_layers):
z = layers[i].fp(z)
return z def z_score_normalization(x):
mu = np.mean(x)
sigma = np.std(x)
x = (x - mu) / sigma;
return x; def sigmoid(X, useStatus):
if useStatus:
return 1.0 / (1 + np.exp(-float(X)));
else:
return float(X); def plot_curve(data, title, lege, xlabel, ylabel):
num = len(data)
idx = range(num)
plt.plot(idx, data, color="r", linewidth=1) plt.xlabel(xlabel, fontsize="xx-large")
plt.ylabel(ylabel, fontsize="xx-large")
plt.title(title, fontsize="xx-large")
plt.legend([lege], fontsize="xx-large", loc='upper left');
plt.show() if __name__ == "__main__":
print ('This is main of module "bp_nn.py"') print("Import data")
raw_data = pd.read_csv('./train.csv', header=0)
data = raw_data.values
imgs = data[0::, 1::]
labels = data[::, 0]
train_features, test_features, train_labels, test_labels = ttsplit(
imgs, labels, test_size=0.33, random_state=23323) train_features = z_score_normalization(train_features)
test_features = z_score_normalization(test_features)
sample_num = train_labels.shape[0]
tr_labels = np.zeros([sample_num, 10])
for i in range(sample_num):
tr_labels[i][train_labels[i]] = 1 sample_num = test_labels.shape[0]
te_labels = np.zeros([sample_num, 10])
for i in range(sample_num):
te_labels[i][test_labels[i]] = 1 print train_features.shape
print tr_labels.shape
print test_features.shape
print te_labels.shape stop_crit = 100 # 停止
max_iter = 10000 # 最大迭代次数
batch_size = 100 # 每次训练的样本个数
net_struct = [[100, "relu", 0.01], [10, "sigmoid", 0.1]] # 网络结构[[batch_size,active function, learning rate]]
# net_struct = [[200,"sigmoid",0.5],[100,"sigmoid",0.5],[10,"sigmoid",0.5]] 网络结构[[batch_size,active function, learning rate]] bpNNCls = BpNet(net_struct, stop_crit, max_iter, batch_size);
# train model [acc, loss] = bpNNCls.train(train_features, tr_labels, test_features, te_labels)
# [acc, loss] = bpNNCls.train(train_features, tr_labels)
print("training model finished")
# create test data
plot_curve(acc, "Bp Network Accuracy", "accuracy", "iter", "Accuracy")
plot_curve(loss, "Bp Network Loss", "loss", "iter", "Loss") # test model
[acc, loss] = bpNNCls.test(test_features, te_labels);
print "test accuracy:%f" % (acc)
实验数据为mnist数据集合，可从以下地址下载：https://github.com/WenDesi/lihang_book_algorithm/blob/master/data/train.csv
a.使用sigmoid激活函数和net_struct = [10,"sigmoid"]的网络结构（可看作是softmax 回归），其校验精度和损失函数的变化，如下图所示：
测试精度达到0.916017，效果还是不错的。但是随机梯度法，依赖于参数的初始化，如果初始化不好，会收敛缓慢，甚至有不理想的结果。
b.使用sigmoid激活函数和net_struct = [200,"sigmoid",100,"sigmoid",10,"sigmoid"] 的网络结构（一个200的隐藏层，一个100的隐藏层，和一个10的输出层），其校验精度和损失函数的变化，如下图所示：
其校验精度达到0.963636，比softmax要好不少。从损失曲线可以看出，加入隐藏层后，算法收敛要比无隐藏层的稳定。