import numpy as np

'''极简两层反传(BP)神经网络'''

# 样本

X = np.array([[0,0,1],[0,1,1],[1,0,1],[1,1,1]])

y = np.array([0,0,1,1])

# 权值矩阵 初始化

Wi = 2 * np.random.random(3) - 1

for iter in range(10000):

    # 前向传播，计算误差

    li = X

    lo = 1 / (1 + np.exp(-np.dot(li, Wi))) # 激活函数：sigmoid

    lo_error = y - lo

    # 后向传播，更新权值

    lo_delta = lo_error * lo * (1 - lo)    # sigmoid函数的导数（梯度下降）

    Wi += np.dot(lo_delta, li)

print("训练效果：\n", lo)

import numpy as np

'''极简三层反传(BP)神经网络'''

# 样本

X = np.array([[0,0,1],[0,1,1],[1,0,1],[1,1,1]])

y = np.array([0,1,1,0])

# 权值矩阵

Wi = 2 * np.random.random((3, 5)) - 1

Wh = 2 * np.random.random(5) - 1

# 训练

for i in range(10000):

    # 前向传播，计算误差

    li = X

    lh = 1 / (1 + np.exp(-np.dot(li, Wi)))

    lo = 1 / (1 + np.exp(-np.dot(lh, Wh)))

    lo_error = y - lo

    # 后向传播，更新权值

    lo_delta = lo_error * (lo * (1 - lo))

    lh_delta = np.outer(lo_delta, Wh) * (lh * (1 - lh)) # 外积！感谢 numpy 的强大！

    Wh += np.dot(lh.T, lo_delta)

    Wi += np.dot(li.T, lh_delta)

print("训练之后：\n", lo)

import numpy as np

'''极简四层反传(BP)神经网络'''

# 样本

X = np.array([[0,0,1],[0,1,1],[1,0,1],[1,1,1]])

y = np.array([0,1,1,0])

# 权值矩阵

Wi  = 2 * np.random.random((3, 5)) - 1

Wh1 = 2 * np.random.random((5, 4)) - 1

Wh2 = 2 * np.random.random(4) - 1

# 训练

for i in range(10000):

    # 前向传播，计算误差

    li = X

    lh1 = 1 / (1 + np.exp(-np.dot(li,  Wi )))

    lh2 = 1 / (1 + np.exp(-np.dot(lh1, Wh1)))

    lo  = 1 / (1 + np.exp(-np.dot(lh2, Wh2)))

    lo_error = y - lo

    # 后向传播，更新权值

    lo_delta = lo_error * (lo * (1 - lo))

    lh2_delta = np.outer(lo_delta, Wh2.T) * (lh2 * (1 - lh2))

    lh1_delta = np.dot(lh2_delta, Wh1.T) * (lh1 * (1 - lh1))  # 注意：这里是dot!

    Wh2 += np.dot(lh2.T, lo_delta)

    Wh1 += np.dot(lh1.T, lh2_delta)

    Wi  += np.dot(li.T,  lh1_delta)

print("训练之后：\n", lo)

import numpy as np

# 样本

X = np.array([[0,0,1],[0,1,1],[1,0,1],[1,1,1]])

y = np.array([0,1,1,0])

ni = 3 # 输入层节点数

nh = 5 # 隐藏层节点数

no = 2 # 输出层节点数（注意这里是2!!）

# 初始化矩阵、偏置

Wi = np.random.randn(ni, nh) / np.sqrt(ni)

Wh = np.random.randn(nh, no) / np.sqrt(nh)

bh = np.zeros(nh)

bo = np.zeros(no)

# 训练

for i in range(1000):

    # 前向传播

    li = X

    lh = np.tanh(np.dot(X, Wi) + bh)     # tanh 函数

    lo = np.exp(np.dot(lh, Wh) + bo)

    probs = lo / np.sum(lo, axis=1, keepdims=True)

    # 后向传播

    lo_delta = probs

    lo_delta[range(X.shape[0]), y] += 1 # -=1

    lh_delta = np.dot(lo_delta, Wh.T) * (1 - np.power(lh, 2)) # tanh 函数的导数

    # 更新权值、偏置

    epsilon = 0.01    # 学习速率

    lamda = 0.01      # 正则化强度

    bo += -epsilon * np.sum(lo_delta, axis=0, keepdims=True).reshape(-1)

    Wh += -epsilon * (np.dot(lh.T, lo_delta) + lamda * Wh)

    bh += -epsilon * np.sum(lh_delta, axis=0)

    Wi += -epsilon * (np.dot(X.T, lh_delta) + lamda * Wi)

print("训练之后：\n", np.argmax(probs, axis=1))

import numpy as np

# 样本

X = np.array([[0,0,1],[0,1,1],[1,0,1],[1,1,1]])

y = np.array([0,1,1,0])

# 神经网络结构，层数任意！

sizes = [3,5,7,2]

# 初始化矩阵、偏置

biases = [np.random.randn(j) for j in sizes[1:]]

weights = [np.random.randn(i,j) for i,j in zip(sizes[:-1], sizes[1:])]

layers = [None] * len(sizes)

layers[0] = X

layers_delta = [None] * (len(sizes) - 1)

epsilon = 0.01 # 学习速率

lamda = 0.01   # 正则化强度

# 训练

for i in range(1000):

    # 前向传播

    for i in range(1, len(layers)):

        layers[i] = 1 / (1 + np.exp(-(np.dot(layers[i-1], weights[i-1]) + biases[i-1])))

    # 后向传播

    probs = layers[-1] / np.sum(layers[-1], axis=1, keepdims=True)

    layers_delta[-1] = probs

    layers_delta[-1][range(X.shape[0]), y] += 1

    for i in range(len(sizes)-2, 0, -1):

        layers_delta[i-1] = np.dot(layers_delta[i], weights[i].T) * (layers[i] * (1 - layers[i]))

    # 更新权值、偏置

    for i in range(len(sizes)-2, -1, -1):

        biases[i]  -= epsilon * np.sum(layers_delta[i], axis=0)

        weights[i] -= epsilon * (np.dot(layers[i].T, layers_delta[i]) + lamda * weights[i])

print("训练之后-->np.argmax(probs, axis=1)：\n", np.argmax(probs, axis=1))