BP神经网络是深度学习的重要基础，它是深度学习的重要前行算法之一，因此理解BP神经网络原理以及实现技巧非常有必要。接下来，我们对原理和实现展开讨论。
1.原理
有空再慢慢补上，请先参考老外一篇不错的文章：A Step by Step Backpropagation Example
激活函数参考：深度学习常用激活函数之— Sigmoid & ReLU & Softmax
浅显易懂的初始化：CS231n课程笔记翻译：神经网络笔记 2
有效的Trick：神经网络训练中的Tricks之高效BP（反向传播算法）
通过简单演示BPNN的计算过程：一文弄懂神经网络中的反向传播法——BackPropagation
2.实现----Batch随机梯度法
这里实现了层数可定义的BP神经网络，可通过参数net_struct进行定义网络结果，如定义只有输出层，没有隐藏层的网络结构，激活函数为”sigmoid",学习率，可如下定义
net_struct = [[10,"sigmoid",0.01]] # 网络结构
如定义一层隐藏层为100个神经元，再接一层隐藏层为50个神经元，输出层为10个神经元的网络结构，如下
net_struct = [[100,"sigmoid",0.01],[50,"sigmoid",0.01],[10,"sigmoid",0.01]] # 网络结构
码农最爱的实现如下：
1 # # encoding=utf8View Code
2 '''
3 Created on 2017-7-3
4
5 @author: Administrator
6 '''
7 import random
8 import pandas as pd
9 import numpy as np
10 from matplotlib import pyplot as plt
11 from sklearn.model_selection import train_test_split as ttsplit
12
13 class LossFun:
14 def __init__(self, lf_type="least_square"):
15 self.name = "loss function"
16 self.type = lf_type
17
18 def cal(self, t, z):
19 loss = 0
20 if self.type == "least_square":
21 loss = self.least_square(t, z)
22 return loss
23
24 def cal_deriv(self, t, z):
25 delta = 0
26 if self.type == "least_square":
27 delta = self.least_square_deriv(t, z)
28 return delta
29
30 def least_square(self, t, z):
31 zsize = z.shape
32 sample_num = zsize[1]
33 return np.sum(0.5 * (t - z) * (t - z) * t) / sample_num
34
35 def least_square_deriv(self, t, z):
36 return z - t
37
38 class ActivationFun:
39 '''
40 激活函数
41 '''
42 def __init__(self, atype="sigmoid"):
43 self.name = "activation function library"
44 self.type = atype;
45
46 def cal(self, a):
47 z = 0
48 if self.type == "sigmoid":
49 z = self.sigmoid(a)
50 elif self.type == "relu":
51 z = self.relu(a)
52 return z
53
54 def cal_deriv(self, a):
55 z = 0
56 if self.type == "sigmoid":
57 z = self.sigmoid_deriv(a)
58 elif self.type == "relu":
59 z = self.relu_deriv(a)
60 return z
61
62 def sigmoid(self, a):
63 return 1 / (1 + np.exp(-a))
64
65 def sigmoid_deriv(self, a):
66 fa = self.sigmoid(a)
67 return fa * (1 - fa)
68
69 def relu(self, a):
70 idx = a <= 0
71 a[idx] = 0.1 * a[idx]
72 return a # np.maximum(a, 0.0)
73
74 def relu_deriv(self, a):
75 # print a
76 a[a > 0] = 1.0
77 a[a <= 0] = 0.1
78 # print a
79 return a
80
81 class Layer:
82 '''
83 神经网络层
84 '''
85 def __init__(self, num_neural, af_type="sigmoid", learn_rate=0.5):
86 self.af_type = af_type # active function type
87 self.learn_rate = learn_rate
88 self.num_neural = num_neural
89 self.dim = None
90 self.W = None
91
92 self.a = None
93 self.X = None
94 self.z = None
95 self.delta = None
96 self.theta = None
97 self.act_fun = ActivationFun(self.af_type)
98
99 def fp(self, X):
100 '''
101 Foward Propagation
102 '''
103 self.X = X
104 xsize = X.shape
105 self.dim = xsize[0]
106 self.num = xsize[1]
107
108 if self.W == None:
109 # self.W = np.random.random((self.dim, self.num_neural))-0.5
110 # self.W = np.random.uniform(-1,1,size=(self.dim,self.num_neural))
111 if(self.af_type == "sigmoid"):
112 self.W = np.random.normal(0, 1, size=(self.dim, self.num_neural)) / np.sqrt(self.num)
113 elif(self.af_type == "relu"):
114 self.W = np.random.normal(0, 1, size=(self.dim, self.num_neural)) * np.sqrt(2.0 / self.num)
115 if self.theta == None:
116 # self.theta = np.random.random((self.num_neural, 1))-0.5
117 # self.theta = np.random.uniform(-1,1,size=(self.num_neural,1))
118
119 if(self.af_type == "sigmoid"):
120 self.theta = np.random.normal(0, 1, size=(self.num_neural, 1)) / np.sqrt(self.num)
121 elif(self.af_type == "relu"):
122 self.theta = np.random.normal(0, 1, size=(self.num_neural, 1)) * np.sqrt(2.0 / self.num)
123 # calculate the foreward a
124 self.a = (self.W.T).dot(self.X)
125 ###calculate the foreward z####
126 self.z = self.act_fun.cal(self.a)
127 return self.z
128
129 def bp(self, delta):
130 '''
131 Back Propagation
132 '''
133 self.delta = delta * self.act_fun.cal_deriv(self.a)
134 self.theta = np.array([np.mean(self.theta - self.learn_rate * self.delta, 1)]).T # 求所有样本的theta均值
135 dW = self.X.dot(self.delta.T) / self.num
136 self.W = self.W - self.learn_rate * dW
137 delta_out = self.W.dot(self.delta);
138 return delta_out
139
140 class BpNet:
141 '''
142 BP神经网络
143 '''
144 def __init__(self, net_struct, stop_crit, max_iter, batch_size=10):
145 self.name = "net work"
146 self.net_struct = net_struct
147 if len(self.net_struct) == 0:
148 print "no layer is specified!"
149 return
150
151 self.stop_crit = stop_crit
152 self.max_iter = max_iter
153 self.batch_size = batch_size
154 self.layers = []
155 self.num_layers = 0;
156 # 创建网络
157 self.create_net(net_struct)
158 self.loss_fun = LossFun("least_square");
159
160 def create_net(self, net_struct):
161 '''
162 创建网络
163 '''
164 self.num_layers = len(net_struct)
165 for i in range(self.num_layers):
166 self.layers.append(Layer(net_struct[i][0], net_struct[i][1], net_struct[i][2]))
167
168 def train(self, X, t, Xtest=None, ttest=None):
169 '''
170 训练网络
171 '''
172 eva_acc_list = []
173 eva_loss_list = []
174
175 xshape = X.shape;
176 num = xshape[0]
177 dim = xshape[1]
178
179 for k in range(self.max_iter):
180 # i = random.randint(0,num-1)
181 idxs = random.sample(range(num), self.batch_size)
182 xi = np.array([X[idxs, :]]).T[:, :, 0]
183 ti = np.array([t[idxs, :]]).T[:, :, 0]
184 # 前向计算
185 zi = self.fp(xi)
186
187 # 偏差计算
188 delta_i = self.loss_fun.cal_deriv(ti, zi)
189
190 # 反馈计算
191 self.bp(delta_i)
192
193 # 评估精度
194 if Xtest != None:
195 if k % 100 == 0:
196 [eva_acc, eva_loss] = self.test(Xtest, ttest)
197 eva_acc_list.append(eva_acc)
198 eva_loss_list.append(eva_loss)
199 print "%4d,%4f,%4f" % (k, eva_acc, eva_loss)
200 else:
201 print "%4d" % (k)
202 return [eva_acc_list, eva_loss_list]
203
204 def test(self, X, t):
205 '''
206 测试模型精度
207 '''
208 xshape = X.shape;
209 num = xshape[0]
210 z = self.fp_eval(X.T)
211 t = t.T
212 est_pos = np.argmax(z, 0)
213 real_pos = np.argmax(t, 0)
214 corrct_count = np.sum(est_pos == real_pos)
215 acc = 1.0 * corrct_count / num
216 loss = self.loss_fun.cal(t, z)
217 # print "%4f,loss:%4f"%(loss)
218 return [acc, loss]
219
220 def fp(self, X):
221 '''
222 前向计算
223 '''
224 z = X
225 for i in range(self.num_layers):
226 z = self.layers[i].fp(z)
227 return z
228
229 def bp(self, delta):
230 '''
231 反馈计算
232 '''
233 z = delta
234 for i in range(self.num_layers - 1, -1, -1):
235 z = self.layers[i].bp(z)
236 return z
237
238 def fp_eval(self, X):
239 '''
240 前向计算
241 '''
242 layers = self.layers
243 z = X
244 for i in range(self.num_layers):
245 z = layers[i].fp(z)
246 return z
247
248 def z_score_normalization(x):
249 mu = np.mean(x)
250 sigma = np.std(x)
251 x = (x - mu) / sigma;
252 return x;
253
254 def sigmoid(X, useStatus):
255 if useStatus:
256 return 1.0 / (1 + np.exp(-float(X)));
257 else:
258 return float(X);
259
260 def plot_curve(data, title, lege, xlabel, ylabel):
261 num = len(data)
262 idx = range(num)
263 plt.plot(idx, data, color="r", linewidth=1)
264
265 plt.xlabel(xlabel, fontsize="xx-large")
266 plt.ylabel(ylabel, fontsize="xx-large")
267 plt.title(title, fontsize="xx-large")
268 plt.legend([lege], fontsize="xx-large", loc='upper left');
269 plt.show()
270
271 if __name__ == "__main__":
272 print ('This is main of module "bp_nn.py"')
273
274 print("Import data")
275 raw_data = pd.read_csv('./train.csv', header=0)
276 data = raw_data.values
277 imgs = data[0::, 1::]
278 labels = data[::, 0]
279 train_features, test_features, train_labels, test_labels = ttsplit(
280 imgs, labels, test_size=0.33, random_state=23323)
281
282 train_features = z_score_normalization(train_features)
283 test_features = z_score_normalization(test_features)
284 sample_num = train_labels.shape[0]
285 tr_labels = np.zeros([sample_num, 10])
286 for i in range(sample_num):
287 tr_labels[i][train_labels[i]] = 1
288
289 sample_num = test_labels.shape[0]
290 te_labels = np.zeros([sample_num, 10])
291 for i in range(sample_num):
292 te_labels[i][test_labels[i]] = 1
293
294 print train_features.shape
295 print tr_labels.shape
296 print test_features.shape
297 print te_labels.shape
298
299 stop_crit = 100 # 停止
300 max_iter = 10000 # 最大迭代次数
301 batch_size = 100 # 每次训练的样本个数
302 net_struct = [[100, "relu", 0.01], [10, "sigmoid", 0.1]] # 网络结构[[batch_size,active function, learning rate]]
303 # net_struct = [[200,"sigmoid",0.5],[100,"sigmoid",0.5],[10,"sigmoid",0.5]] 网络结构[[batch_size,active function, learning rate]]
304
305 bpNNCls = BpNet(net_struct, stop_crit, max_iter, batch_size);
306 # train model
307
308 [acc, loss] = bpNNCls.train(train_features, tr_labels, test_features, te_labels)
309 # [acc, loss] = bpNNCls.train(train_features, tr_labels)
310 print("training model finished")
311 # create test data
312 plot_curve(acc, "Bp Network Accuracy", "accuracy", "iter", "Accuracy")
313 plot_curve(loss, "Bp Network Loss", "loss", "iter", "Loss")
314
315
316 # test model
317 [acc, loss] = bpNNCls.test(test_features, te_labels);
318 print "test accuracy:%f" % (acc)
319
实验数据为mnist数据集合，可从以下地址下载：https://github.com/WenDesi/lihang_book_algorithm/blob/master/data/train.csv
a.使用sigmoid激活函数和net_struct = [10,"sigmoid"]的网络结构（可看作是softmax 回归），其校验精度和损失函数的变化，如下图所示：
测试精度达到0.916017，效果还是不错的。但是随机梯度法，依赖于参数的初始化，如果初始化不好，会收敛缓慢，甚至有不理想的结果。
b.使用sigmoid激活函数和net_struct = [200,"sigmoid",100,"sigmoid",10,"sigmoid"] 的网络结构（一个200的隐藏层，一个100的隐藏层，和一个10的输出层），其校验精度和损失函数的变化，如下图所示：
其校验精度达到0.963636，比softmax要好不少。从损失曲线可以看出，加入隐藏层后，算法收敛要比无隐藏层的稳定。