由于RNN的梯度计算时,总会有一项Whh的k-i次方， 当i越小， 即(越靠前的层)，Whh的k-i次方会越来越大，所以越前的层的梯度越容易出现梯度爆炸现象。

Gradient clipping

LSTM(可以解决梯度爆炸问题)

RNN原始模型

LSTM结构

可以看到LSTM的梯度计算公式为4项的和，这样可以防止出现次方项使梯度过大，也不会因为某一项的大小影响总体的梯度稳定。

也可以这样看： 当remember gate way 接近于1时，Ct 将约等于Ct-1，memory将很大长度上得到保留，求梯度时，偏Ct/偏Ct-1 将约等于1，梯度信息将保留，接近于CNN的shortcort理解方式。

实战

import os
import tensorflow as tf
import numpy as np
from tensorflow import keras
from tensorflow.keras import layers

tf.random.set_seed(22)
np.random.seed(22)
os.environ["TF_CPP_MIN_LOG_LEVEL"] = '2'

batchsz = 128

# 常用单词数量, 其他不常用单词将表示为一种
total_words = 10000
# 一句中最大单词数量, 大于80将截取, 少于则padding
max_review_len = 80
# 每单词将用一个长度100的vector表示
embedding_len = 100
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)

# 对每个句子进行预处理, 使得每句单词数量为80
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)

db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
# drop_remainder=True丢弃掉最后一个batch
# 由于运算时要求每一个batch大小相等, 所以要丢弃掉最后一个batch
db_train = db_train.shuffle(10000).batch(batchsz, drop_remainder=True)

db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)

print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)

class MyRNN(keras.Model):
    # units: 输出维度
    def __init__(self, units):
        super(MyRNN, self).__init__()

        # [b, 64] H0 C0 初始化Ct 和 Ht
        self.state0 = [tf.zeros([batchsz, units]), tf.zeros([batchsz, units])]
        self.state1 = [tf.zeros([batchsz, units]), tf.zeros([batchsz, units])]

        # 将文本转换为embedding表示
        # [b, 80] => [b, 80, 100]
        # (总得单词数量, 每单词将用一个长度100的vector表示 , 每个句子的单词数量)
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
        # print(tf.reduce_max(self.embedding), tf.reduce_min(self.embedding))

        # [b, 80, 100]  hidden_dim:64
        # self.rnn_cell0 = layers.SimpleRNNCell(units, dropout=0.5)
        # self.rnn_cell1 = layers.SimpleRNNCell(units, dropout=0.5)
        self.rnn_cell0 = layers.LSTMCell(units, dropout=0.5)
        self.rnn_cell1 = layers.LSTMCell(units, dropout=0.5)

        # fc, [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = layers.Dense(1)

    def call(self, inputs, training=None):
        # [b, 80]
        x = inputs
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute
        # [b, 80, 100] => [b, 64]
        state0 = self.state0
        state1 = self.state1
        for word in tf.unstack(x, axis=1):
            # h1 = x * Wxh + h0 * Whh
            # out0: [b, 64]
            out0, state0 = self.rnn_cell0(word, state0, training)
            # out1: [b, 64]
            out1, state1 = self.rnn_cell1(out0, state1, training)
        # out: [b, 64] => [b, 1]
        x = self.outlayer(out1)
        # p(y is pos|x)
        prob = tf.sigmoid(x)

        return prob

def main():
    units = 64
    epochs = 4

    model = MyRNN(units)
    model.compile(optimizer=keras.optimizers.Adam(0.001),
                  loss=tf.losses.BinaryCrossentropy(),
                  metrics=["accuracy"])

    model.fit(db_train, epochs=epochs, validation_data=db_test)

    model.evaluate(db_test)

if __name__ == '__main__':
    main()

import os
import tensorflow as tf
import numpy as np
from tensorflow import keras
from tensorflow.keras import layers

tf.random.set_seed(22)
np.random.seed(22)
os.environ["TF_CPP_MIN_LOG_LEVEL"] = '2'

batchsz = 64

# 常用单词数量, 其他不常用单词将表示为一种
total_words = 10000
# 一句中最大单词数量, 大于80将截取, 少于则padding
max_review_len = 80
# 每单词将用一个长度100的vector表示
embedding_len = 100
(x_train, y_train), (x_test, y_test) = keras.datasets.imdb.load_data(num_words=total_words)

# 对每个句子进行预处理, 使得每句单词数量为80
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=max_review_len)
x_test = keras.preprocessing.sequence.pad_sequences(x_test, maxlen=max_review_len)

db_train = tf.data.Dataset.from_tensor_slices((x_train, y_train))
# drop_remainder=True丢弃掉最后一个batch
# 由于运算时要求每一个batch大小相等, 所以要丢弃掉最后一个batch
db_train = db_train.shuffle(10000).batch(batchsz, drop_remainder=True)

db_test = tf.data.Dataset.from_tensor_slices((x_test, y_test))
db_test = db_test.batch(batchsz, drop_remainder=True)

print('x_train shape:', x_train.shape, tf.reduce_max(y_train), tf.reduce_min(y_train))
print('x_test shape:', x_test.shape)

class MyRNN(keras.Model):
    # units: 输出维度
    def __init__(self, units):
        super(MyRNN, self).__init__()

        # [b, 64] h0
        self.state0 = [tf.zeros([batchsz, units])]
        self.state1 = [tf.zeros([batchsz, units])]  # h1

        # 将文本转换为embedding表示
        # [b, 80] => [b, 80, 100]
        # (总得单词数量, 每单词将用一个长度100的vector表示 , 每个句子的单词数量)
        self.embedding = layers.Embedding(total_words, embedding_len, input_length=max_review_len)
        # print(tf.reduce_max(self.embedding), tf.reduce_min(self.embedding))

        # [b, 80, 100]  hidden_dim:64
        self.rnn = keras.Sequential([
            # layers.SimpleRNN(units, dropout=0.5, return_sequences=True, unroll=True),
            # layers.SimpleRNN(units, dropout=0.5, unroll=True)

            layers.LSTM(units, dropout=0.5, return_sequences=True, unroll=True),
            layers.LSTM(units, dropout=0.5, unroll=True)
        ])

        # fc, [b, 80, 100] => [b, 64] => [b, 1]
        self.outlayer = layers.Dense(1)

    def call(self, inputs, training=None):
        # [b, 80]
        x = inputs
        # embedding: [b, 80] => [b, 80, 100]
        x = self.embedding(x)
        # rnn cell compute
        # [b, 80, 100] => [b, 64]
        x = self.rnn(x)
        # out: [b, 64] => [b, 1]
        x = self.outlayer(x)
        # p(y is pos|x)
        prob = tf.sigmoid(x)

        return prob

def main():
    units = 64
    epochs = 4

    model = MyRNN(units)
    model.compile(optimizer=keras.optimizers.Adam(0.001),
                  loss=tf.losses.BinaryCrossentropy(),
                  metrics=["accuracy"])

    model.fit(db_train, epochs=epochs, validation_data=db_test)

    model.evaluate(db_test)

if __name__ == '__main__':
    main()