斯坦福公开课
论文
- Learning Phrase Representations using RNN Encoder-Decoder for Statistical Machine Translation
- Effective Approaches to Attention-based Neural Machine Translation
- Neural Machine Translation by Jointly Learning to Align and Translate
PyTorch代码（学习代码编写方式）
更多关于Machine Translation
- Beam Search
- Pointer network 文本摘要
- Copy Mechanism 文本摘要
- Converage Loss
- ConvSeq2Seq
- Transformer
- Tensor2Tensor
TODO
- 建议同学尝试对中文进行分词
NER
部分代码
读入中英文数据
- 英文我们使用nltk的word tokenizer来分词，并且使用小写字母
- 中文我们直接使用单个汉字作为基本单元
import os
import sys
import math
from collections import Counter
import numpy as np
import random
import torch
import torch.nn as nn
import torch.nn.functional as F
import nltk
def load_data(in_file):
cn = []
en = []
num_examples = 0
with open(in_file, 'r') as f:
for line in f:
line = line.strip().split("\t")
en.append(["BOS"] + nltk.word_tokenize(line[0].lower()) + ["EOS"])
# split chinese sentence into characters
cn.append(["BOS"] + [c for c in line[1]] + ["EOS"])
return en, cn
train_file = "nmt/en-cn/train.txt"
dev_file = "nmt/en-cn/dev.txt"
train_en, train_cn = load_data(train_file)
dev_en, dev_cn = load_data(dev_file)
encoder/attention/decoder
class Encoder(nn.Module):
def __init__(self, vocab_size, embed_size, enc_hidden_size, dec_hidden_size, dropout=0.2):
super(Encoder, self).__init__()
self.embed = nn.Embedding(vocab_size, embed_size)
self.rnn = nn.GRU(embed_size, enc_hidden_size, batch_first=True, bidirectional=True)
self.dropout = nn.Dropout(dropout)
self.fc = nn.Linear(enc_hidden_size * 2, dec_hidden_size)
def forward(self, x, lengths):
sorted_len, sorted_idx = lengths.sort(0, descending=True)
x_sorted = x[sorted_idx.long()]
embedded = self.dropout(self.embed(x_sorted))
packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, sorted_len.long().cpu().data.numpy(), batch_first=True)
packed_out, hid = self.rnn(packed_embedded)
out, _ = nn.utils.rnn.pad_packed_sequence(packed_out, batch_first=True)
_, original_idx = sorted_idx.sort(0, descending=False)
out = out[original_idx.long()].contiguous()
hid = hid[:, original_idx.long()].contiguous()
hid = torch.cat([hid[-2], hid[-1]], dim=1)
hid = torch.tanh(self.fc(hid)).unsqueeze(0)
return out, hid
class Attention(nn.Module):
def __init__(self, enc_hidden_size, dec_hidden_size):
super(Attention, self).__init__()
self.enc_hidden_size = enc_hidden_size
self.dec_hidden_size = dec_hidden_size
self.linear_in = nn.Linear(enc_hidden_size*2, dec_hidden_size, bias=False)
self.linear_out = nn.Linear(enc_hidden_size*2 + dec_hidden_size, dec_hidden_size)
def forward(self, output, context, mask):
# output: batch_size, output_len, dec_hidden_size
# context: batch_size, context_len, 2*enc_hidden_size
batch_size = output.size(0)
output_len = output.size(1)
input_len = context.size(1)
context_in = self.linear_in(context.view(batch_size*input_len, -1)).view(
batch_size, input_len, -1) # batch_size, context_len, dec_hidden_size
# context_in.transpose(1,2): batch_size, dec_hidden_size, context_len
# output: batch_size, output_len, dec_hidden_size
attn = torch.bmm(output, context_in.transpose(1,2))
# batch_size, output_len, context_len
attn.data.masked_fill(mask, -1e6)
attn = F.softmax(attn, dim=2)
# batch_size, output_len, context_len
context = torch.bmm(attn, context)
# batch_size, output_len, enc_hidden_size
output = torch.cat((context, output), dim=2) # batch_size, output_len, hidden_size*2
output = output.view(batch_size*output_len, -1)
output = torch.tanh(self.linear_out(output))
output = output.view(batch_size, output_len, -1)
return output, attn
class Decoder(nn.Module):
def __init__(self, vocab_size, embed_size, enc_hidden_size, dec_hidden_size, dropout=0.2):
super(Decoder, self).__init__()
self.embed = nn.Embedding(vocab_size, embed_size)
self.attention = Attention(enc_hidden_size, dec_hidden_size)
self.rnn = nn.GRU(embed_size, hidden_size, batch_first=True)
self.out = nn.Linear(dec_hidden_size, vocab_size)
self.dropout = nn.Dropout(dropout)
def create_mask(self, x_len, y_len):
# a mask of shape x_len * y_len
device = x_len.device
max_x_len = x_len.max()
max_y_len = y_len.max()
x_mask = torch.arange(max_x_len, device=x_len.device)[None, :] < x_len[:, None]
y_mask = torch.arange(max_y_len, device=x_len.device)[None, :] < y_len[:, None]
mask = (1 - x_mask[:, :, None] * y_mask[:, None, :]).byte()
return mask
def forward(self, ctx, ctx_lengths, y, y_lengths, hid):
sorted_len, sorted_idx = y_lengths.sort(0, descending=True)
y_sorted = y[sorted_idx.long()]
hid = hid[:, sorted_idx.long()]
y_sorted = self.dropout(self.embed(y_sorted)) # batch_size, output_length, embed_size
packed_seq = nn.utils.rnn.pack_padded_sequence(y_sorted, sorted_len.long().cpu().data.numpy(), batch_first=True)
out, hid = self.rnn(packed_seq, hid)
unpacked, _ = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)
_, original_idx = sorted_idx.sort(0, descending=False)
output_seq = unpacked[original_idx.long()].contiguous()
hid = hid[:, original_idx.long()].contiguous()
mask = self.create_mask(y_lengths, ctx_lengths)
output, attn = self.attention(output_seq, ctx, mask)
output = F.log_softmax(self.out(output), -1)
return output, hid, attn
seq2seq
class Seq2Seq(nn.Module):
def __init__(self, encoder, decoder):
super(Seq2Seq, self).__init__()
self.encoder = encoder
self.decoder = decoder
def forward(self, x, x_lengths, y, y_lengths):
encoder_out, hid = self.encoder(x, x_lengths)
output, hid, attn = self.decoder(ctx=encoder_out,
ctx_lengths=x_lengths,
y=y,
y_lengths=y_lengths,
hid=hid)
return output, attn
def translate(self, x, x_lengths, y, max_length=100):
encoder_out, hid = self.encoder(x, x_lengths)
preds = []
batch_size = x.shape[0]
attns = []
for i in range(max_length):
output, hid, attn = self.decoder(ctx=encoder_out,
ctx_lengths=x_lengths,
y=y,
y_lengths=torch.ones(batch_size).long().to(y.device),
hid=hid)
y = output.max(2)[1].view(batch_size, 1)
preds.append(y)
attns.append(attn)
return torch.cat(preds, 1), torch.cat(attns, 1)