一.利用transformer-encoder进行文本分类，用于在问答中的意图识别。

二.结构图

三.程序(完整程序:https://github.com/jiangnanboy/intent_classification/tree/master/transformer_encoder）

import os
import torch
from torchtext import data,datasets
from torchtext.data import Iterator, BucketIterator
from torchtext.vocab import Vectors
from torch import nn,optim
import torch.nn.functional as F
import pandas as pd
import pickle

DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

intent_classification_path = os.path.abspath(os.path.join(os.getcwd(), '../..'))
# 训练数据路径
train_data = os.path.join(intent_classification_path,'classification_data/knowledge_point_qa_data.csv')
# 读取数据
train_data = pd.read_csv(train_data)
# 按字分    
tokenize =lambda x: x.split(' ')

TEXT = data.Field(
                    sequential=True,
                    tokenize=tokenize,
                    lower=True,
                    use_vocab=True,
                    pad_token='<pad>',
                    unk_token='<unk>',
                    batch_first=True,
                    fix_length=20)

LABEL = data.Field(
                    sequential=False,
                    use_vocab=False)

# 获取训练或测试数据集
def get_dataset(csv_data, text_field, label_field, test=False):
    fields = [('id', None), ('text', text_field), ('label', label_field)]
    examples = []
    if test: #测试集，不加载label
        for text in csv_data['text']:
            examples.append(data.Example.fromlist([None, text, None], fields))
    else: # 训练集
        for text, label in zip(csv_data['text'], csv_data['label']):
            examples.append(data.Example.fromlist([None, text, label], fields))
    return examples, fields

train_examples,train_fields = get_dataset(train_data, TEXT, LABEL)

train = data.Dataset(train_examples, train_fields)
# 预训练数据
#pretrained_embedding = os.path.join(os.getcwd(), 'sgns.sogou.char')
#vectors = Vectors(name=pretrained_embedding)
# 构建词典
#TEXT.build_vocab(train, min_freq=1, vectors = vectors)

TEXT.build_vocab(train, min_freq=1)
words_path = os.path.join(os.getcwd(), 'words.pkl')
with open(words_path, 'wb') as f_words:
    pickle.dump(TEXT.vocab, f_words)
    
BATCH_SIZE = 163
# 构建迭代器
train_iter = BucketIterator(
                            dataset=train,
                            batch_size=BATCH_SIZE,
                            shuffle=True,
                            sort_within_batch=False)


'''
    1.输入是序列中token的embedding与位置embedding
    2.token的embedding与其位置embedding相加，得到一个vector(这个向量融合了token与position信息)
    3.在2之前，token的embedding乘上一个scale(防止点积变大，造成梯度过小)向量[sqrt(emb_dim)]，这个假设为了减少embedding中的变化，没有这个scale，很难稳定的去训练model。
    4.加入dropout
    5.通过N个encoder layer，得到Z。此输出Z被传入一个全连接层作分类。
    src_mask对于非<pad>值为1,<pad>为0。为了计算attention而遮挡<pad>这个无意义的token。与source 句子shape一致。
'''
class TransformerEncoder(nn.Module):
    def __init__(self, input_dim, output_dim, emb_dim, n_layers, n_heads, pf_dim, dropout, position_length, pad_idx):
        super(TransformerEncoder, self).__init__()
        
        self.pad_idx = pad_idx
        self.scale = torch.sqrt(torch.FloatTensor([emb_dim])).to(DEVICE)

        # 词的embedding
        self.token_embedding = nn.Embedding(input_dim, emb_dim)
        # 对词的位置进行embedding
        self.position_embedding = nn.Embedding(position_length, emb_dim)
        # encoder层，有几个encoder层，每个encoder有几个head
        self.layers = nn.ModuleList([EncoderLayer(emb_dim, n_heads, pf_dim, dropout) for _ in range(n_layers)])
        self.dropout = nn.Dropout(dropout)
        self.fc = nn.Linear(emb_dim, output_dim)

    def mask_src_mask(self, src):
        # src=[batch_size, src_len]

        # src_mask=[batch_size, 1, 1, src_len]
        src_mask = (src != self.pad_idx).unsqueeze(1).unsqueeze(2)
        return src_mask
    
    def forward(self, src):
        # src=[batch_size, seq_len]
        # src_mask=[batch_size, 1, 1, seq_len]
        src_mask = self.mask_src_mask(src)
        
        batch_size = src.shape[0]
        src_len = src.shape[1]

        # 构建位置tensor -> [batch_size, seq_len]，位置序号从(0)开始到(src_len-1)
        position = torch.arange(0, src_len).unsqueeze(0).repeat(batch_size, 1).to(DEVICE)

        # 对词和其位置进行embedding -> [batch_size,seq_len,embdim]
        token_embeded = self.token_embedding(src) * self.scale
        position_embeded = self.position_embedding(position)

        # 对词和其位置的embedding进行按元素加和 -> [batch_size, seq_len, embdim]
        src = self.dropout(token_embeded + position_embeded)

        for layer in self.layers:
            src = layer(src, src_mask)

        # [batch_size, seq_len, emb_dim] -> [batch_size, output_dim]
        src = src.permute(0, 2, 1)
        src = torch.sum(src, dim=-1)
        src = self.fc(src)
        return src

'''
encoder layers：
    1.将src与src_mask传入多头attention层(multi-head attention)
    2.dropout
    3.使用残差连接后传入layer-norm层(输入+输出后送入norm)后得到的输出
    4.输出通过前馈网络feedforward层
    5.dropout
    6.一个残差连接后传入layer-norm层后得到的输出喂给下一层
    注意：
        layer之间不共享参数
        多头注意力层用到的是多个自注意力层self-attention
'''
class EncoderLayer(nn.Module):
    def __init__(self, emb_dim, n_heads, pf_dim, dropout):
        super(EncoderLayer, self).__init__()
        # 注意力层后的layernorm
        self.self_attn_layer_norm = nn.LayerNorm(emb_dim)
        # 前馈网络层后的layernorm
        self.ff_layer_norm = nn.LayerNorm(emb_dim)
        # 多头注意力层
        self.self_attention = MultiHeadAttentionLayer(emb_dim, n_heads, dropout)
        # 前馈层
        self.feedforward = FeedforwardLayer(emb_dim, pf_dim, dropout)

        self.dropout = nn.Dropout(dropout)

    def forward(self, src, src_mask):
        #src=[batch_size, seq_len, emb_dim]
        #src_mask=[batch_size, 1, 1, seq_len]

        # self-attention
        # _src=[batch size, query_len, emb_dim]
        _src, _ = self.self_attention(src, src, src, src_mask)

        # dropout, 残差连接以及layer-norm
        # src=[batch_size, seq_len, emb_dim]
        src = self.self_attn_layer_norm(src + self.dropout(_src))

        # 前馈网络
        # _src=[batch_size, seq_len, emb_dim]
        _src = self.feedforward(src)

        # dropout, 残差连接以及layer-norm
        # src=[batch_size, seq_len, emb_dim]
        src = self.ff_layer_norm(src + self.dropout(_src))

        return src
'''
多头注意力层的计算:
    1.q,k,v的计算是通过线性层fc_q,fc_k,fc_v
    2.对query,key,value的emb_dim split成n_heads
    3.通过计算Q*K/scale计算energy
    4.利用mask遮掩不需要关注的token
    5.利用softmax与dropout
    6.5的结果与V矩阵相乘
    7.最后通过一个前馈fc_o输出结果
注意:Q,K,V的长度一致
'''
class MultiHeadAttentionLayer(nn.Module):
    def __init__(self, emb_dim, n_heads, dropout):
        super(MultiHeadAttentionLayer, self).__init__()
        assert emb_dim % n_heads == 0
        self.emb_dim = emb_dim
        self.n_heads = n_heads
        self.head_dim = emb_dim//n_heads

        self.fc_q = nn.Linear(emb_dim, emb_dim)
        self.fc_k = nn.Linear(emb_dim, emb_dim)
        self.fc_v = nn.Linear(emb_dim, emb_dim)

        self.fc_o = nn.Linear(emb_dim, emb_dim)

        self.dropout = nn.Dropout(dropout)

        self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(DEVICE)

    def forward(self, query, key, value, mask=None):
        # query=[batch_size, query_len, emb_dim]
        # key=[batch_size, key_len, emb_dim]
        # value=[batch_size, value_len, emb_dim]
        batch_size = query.shape[0]

        # Q=[batch_size, query_len, emb_dim]
        # K=[batch_size, key_len, emb_dim]
        # V=[batch_size, value_len, emb_dim]
        Q = self.fc_q(query)
        K = self.fc_k(key)
        V = self.fc_v(value)

        '''
        view与reshape的异同：
        
        torch的view()与reshape()方法都可以用来重塑tensor的shape，区别就是使用的条件不一样。view()方法只适用于满足连续性条件的tensor，并且该操作不会开辟新的内存空间，
        只是产生了对原存储空间的一个新别称和引用，返回值是视图。而reshape()方法的返回值既可以是视图，也可以是副本，当满足连续性条件时返回view，
        否则返回副本[ 此时等价于先调用contiguous()方法在使用view() ]。因此当不确能否使用view时，可以使用reshape。如果只是想简单地重塑一个tensor的shape，
        那么就是用reshape，但是如果需要考虑内存的开销而且要确保重塑后的tensor与之前的tensor共享存储空间，那就使用view()。
        '''

        # Q=[batch_size, n_heads, query_len, head_dim]
        # K=[batch_size, n_heads, key_len, head_dim]
        # V=[batch_size, n_heads, value_len, head_dim]
        Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
        K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)
        V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)

        # 注意力打分矩阵 [batch_size, n_heads, query_len, head_dim] * [batch_size, n_heads, head_dim, key_len] = [batch_size, n_heads, query_len, key_len]
        energy = torch.matmul(Q, K.permute(0, 1, 3, 2)) / self.scale

        if mask is not None:
            energy = energy.masked_fill(mask == 0, -1e10)
        # [batch_size, n_heads, query_len, key_len]
        attention = torch.softmax(energy , dim = -1)

        # [batch_size, n_heads, query_len, key_len]*[batch_size, n_heads, value_len, head_dim]=[batch_size, n_heads, query_len, head_dim]
        x = torch.matmul(self.dropout(attention), V)

        # [batch_size, query_len, n_heads, head_dim]
        x = x.permute(0, 2, 1, 3).contiguous()

        # [batch_size, query_len, emb_dim]
        x = x.view(batch_size, -1, self.emb_dim)

        # [batch_size, query_len, emb_dim]
        x = self.fc_o(x)

        return x, attention

'''
前馈层
'''
class FeedforwardLayer(nn.Module):
    def __init__(self, emb_dim, pf_dim, dropout):
        super(FeedforwardLayer, self).__init__()
        self.fc_1 = nn.Linear(emb_dim, pf_dim)
        self.fc_2 = nn.Linear(pf_dim, emb_dim)

        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        # x=[batch_size, seq_len, emb_dim]

        # x=[batch_size, seq_len, pf_dim]
        x = self.dropout(torch.relu(self.fc_1(x)))

        # x=[batch_size, seq_len, emb_dim]
        x = self.fc_2(x)

        return x

from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter(os.getcwd()+'/log', comment='transformer-encoder')

# 训练
input_dim = len(TEXT.vocab) 
output_dim = 9
emb_dim = 256
n_layers = 3
n_heads = 8
pf_dim = 512
dropout = 0.5
position_length = 20

# <pad>
pad_index = TEXT.vocab.stoi[TEXT.pad_token]

# 构建model
model = TransformerEncoder(input_dim, output_dim, emb_dim, n_layers, n_heads, pf_dim, dropout, position_length, pad_index).to(DEVICE)
# 利用预训练模型初始化embedding，requires_grad=True，可以fine-tune
# model.embedding.weight.data.copy_(TEXT.vocab.vectors)
# 训练模式
model.train()
# 优化和损失
optimizer = torch.optim.Adam(model.parameters(),lr=0.001, weight_decay=0.01)
# optimizer = torch.optim.SGD(model.parameters(),lr=0.001, momentum=0.9, nesterov=True)
criterion = nn.CrossEntropyLoss()

with writer:
    for iter in range(300):
        for i, batch in enumerate(train_iter):
            train_text = batch.text
            train_label = batch.label
            train_text = train_text.to(DEVICE)
            train_label = train_label.to(DEVICE)
            out = model(train_text)
            loss = criterion(out, train_label)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if (iter+1) % 10 == 0:
                    print ('iter [{}/{}], Loss: {:.4f}'.format(iter+1, 300, loss.item()))
            #writer.add_graph(model, input_to_model=train_text,verbose=False)
            writer.add_scalar('loss',loss.item(),global_step=iter+1)
    writer.flush()
    writer.close()
            
model_path = os.path.join(os.getcwd(), "model.h5")
torch.save(model.state_dict(), model_path)