softmax回归模型

softmax回归跟线性回归一样将输入特征与权重做线性叠加。与线性回归的一个主要不同在于，softmax回归的输出值个数等于标签里的类别数。假设有4种特征和3种输出动物类别，则有

softmax回归同线性回归一样，也是一个单层神经网络。由于每个输出的计算都要依赖于所有的输入，softmax回归的输出层也是一个全连接层。

softmax运算

既然分类问题需要得到离散的预测输出，一个简单的办法是将输出值当作预测类别是iii的置信度，并将值最大的输出所对应的类作为预测输出，即输出。 

直接使用输出层的输出有两个问题。一方面，由于输出层的输出值的范围不确定，我们难以直观上判断这些值的意义。另一方面，由于真实标签是离散值，这些离散值与不确定范围的输出值之间的误差难以衡量。

softmax运算符（softmax operator）解决了以上两个问题。它通过下式将输出值变换成值为正且和为1的概率分布：

，其中， ，

容易看出且，因此是一个合法的概率分布。注意到

，因此softmax运算不改变预测类别输出。

给定一个小批量样本，其批量大小为，输入个数（特征数）为，输出个数（类别数）为。softmax回归的矢量计算表达式为其中的加法运算使用了广播机制，批量特征为，softmax回归的权重和偏差参数分别为和，且这两个矩阵的第行分别为样本的输出和概率分布。

 交叉熵损失函数

softmax运算将输出变换成一个合法的类别预测分布。实际上，真实标签也可以用类别分布表达：对于样本iii，我们构造向量，使其第（样本iii类别的离散数值）个元素为1，其余为0。这样我们的训练目标可以设为使预测概率分布尽可能接近真实的标签概率分布。

衡量两个概率分布差异的测量函数。其中，交叉熵（cross entropy）是一个常用的衡量方法

其中带下标的是向量中非0即1的元素，需要注意将它与样本iii类别的离散数值，即不带下标的y(i)y^{(i)}y(i)区分。在上式中，我们知道向量中只有第个元素为1，其余全为0，于是。也就是说，交叉熵只关心对正确类别的预测概率，因为只要其值足够大，就可以确保分类结果正确

假设训练数据集的样本数为nnn，交叉熵损失函数定义为

其中代表模型参数。同样地，如果每个样本只有一个标签，那么交叉熵损失可以简写成。从另一个角度来看，我们知道最小化等价于最大化，即最小化交叉熵损失函数等价于最大化训练数据集所有标签类别的联合预测概率。

模型预测及评价

在训练好softmax回归模型后，给定任一样本特征，就可以预测每个输出类别的概率。通常，我们把预测概率最大的类别作为输出类别。如果它与真实类别（标签）一致，说明这次预测是正确的。在3.6节的实验中，我们将使用准确率（accuracy）来评价模型的表现。它等于正确预测数量与总预测数量之比。

import torch
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
from IPython import display
import numpy as np
import sys
import time


# 获得文本标签
def get_fashion_mnist_labels(labels):
    text_labels = ['t-shirt', 'trouser', 'pullover', 'dress', 'coat',
                   'sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
    return [text_labels[int(i)] for i in labels]

def show_fashion_mnist(images, labels):
    display.set_matplotlib_formats('svg')
    _, figs = plt.subplots(1, len(images), figsize=(12, 12))
    for f, img, lbl in zip(figs, images, labels):
        f.imshow(img.view((28, 28)).numpy())
        f.set_title(lbl)
        f.axes.get_xaxis().set_visible(False)
        f.axes.get_yaxis().set_visible(False)
    plt.show()

# 读取小批量
def load_data_fashion_mnist(batch_size):
    if sys.platform.startswith('win'):
        num_workers = 0  # 0表示不用额外的进程来加速读取数据
    else:
        num_workers = 4
    train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=num_workers)
    test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False, num_workers=num_workers)
    return train_iter,test_iter

def softmax(X):
    X_exp = X.exp()
    partition = X_exp.sum(dim=1, keepdim=True)
    return X_exp / partition  # 这里应用了广播机制

def net(X):
    return softmax(torch.mm(X.view((-1, num_inputs)), W) + b)

def cross_entropy(y_hat, y):
    return - torch.log(y_hat.gather(1, y.view(-1, 1)))

# 优化函数
def sgd(params, lr, batch_size):
    for param in params:
        param.data -= lr * param.grad / batch_size # 注意这里更改param时用的param.data

#
def accuracy(y_hat, y):
    return (y_hat.argmax(dim=1) == y).float().mean().item()

def evaluate_accuracy(data_iter, net):
    acc_sum, n = 0.0, 0
    for X, y in data_iter:
        acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()
        n += y.shape[0]
    return acc_sum / n

# 训练网络
def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size,
              params=None, lr=None, optimizer=None):
    for epoch in range(num_epochs):
        train_l_sum, train_acc_sum, n = 0.0, 0.0, 0
        for X, y in train_iter:
            y_hat = net(X)
            l = loss(y_hat, y).sum()

            # 梯度清零
            if optimizer is not None:
                optimizer.zero_grad()
            elif params is not None and params[0].grad is not None:
                for param in params:
                    param.grad.data.zero_()

            l.backward()
            if optimizer is None:
                sgd(params, lr, batch_size)
            else:
                optimizer.step()

            train_l_sum += l.item()
            train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()
            n += y.shape[0]
        test_acc = evaluate_accuracy(test_iter, net)
        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
              % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))

if __name__ == '__main__':
    # 获取数据
    mnist_train = torchvision.datasets.FashionMNIST(root='./Datasets/FashionMNIST', train=True, download=True,
                                                    transform=transforms.ToTensor())
    mnist_test = torchvision.datasets.FashionMNIST(root='./Datasets/FashionMNIST', train=False, download=True,
                                                   transform=transforms.ToTensor())
    # 由于transform，feature是torch.Tensor类型
    # feature, label = mnist_train[0]
    # print(feature.shape, label)  # Channel x Height x Width

    # # 显示前10幅图像
    # X, y = [], []
    # for i in range(10):
    #     X.append(mnist_train[i][0])
    #     y.append(mnist_train[i][1])
    # show_fashion_mnist(X, get_fashion_mnist_labels(y))

    # 小批量读取数据
    batch_size = 256
    train_iter,test_iter = load_data_fashion_mnist(batch_size)

    # start = time.time()
    # for X, y in train_iter:
    #     continue
    # print('%.2f sec' % (time.time() - start))

    # 初始化模型参数，使用向量表示每个样本。已知每个样本输入是高和宽均为28像素的图像。模型的输入向量的长度是 28×28=78428×28=784
    # 由于图像有10个类别，单层神经网络输出层的输出个数为10
    num_inputs = 784
    num_outputs = 10

    W = torch.tensor(np.random.normal(0, 0.01, (num_inputs, num_outputs)), dtype=torch.float)
    b = torch.zeros(num_outputs, dtype=torch.float)

    W.requires_grad_(requires_grad=True)
    b.requires_grad_(requires_grad=True)

    # X = torch.tensor([[1, 2, 3], [4, 5, 6]])
    # print(X.sum(dim=(0), keepdim=True))
    # print(X.sum(dim=1, keepdim=True))

    # 训练
    num_epochs, lr = 5, 0.1
    train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, batch_size, [W, b], lr)

    X, y = iter(test_iter).next()

    true_labels = get_fashion_mnist_labels(y.numpy())
    pred_labels = get_fashion_mnist_labels(net(X).argmax(dim=1).numpy())
    titles = [true + '\n' + pred for true, pred in zip(true_labels, pred_labels)]

    show_fashion_mnist(X[0:9], titles[0:9])

import torch
import torchvision
from torchvision import transforms
from torch import nn
from torch.nn import init
import numpy as np
import sys
from collections import OrderedDict

class LinearNet(nn.Module):
    def __init__(self, num_inputs, num_outputs):
        super(LinearNet, self).__init__()
        self.linear = nn.Linear(num_inputs, num_outputs)
    def forward(self, x): # x shape: (batch, 1, 28, 28)
        y = self.linear(x.view(x.shape[0], -1))
        return y

class FlattenLayer(nn.Module):
    def __init__(self):
        super(FlattenLayer, self).__init__()
    def forward(self, x): # x shape: (batch, *, *, ...)
        return x.view(x.shape[0], -1)

def load_data_fashion_mnist(batch_size):
    mnist_train = torchvision.datasets.FashionMNIST(root='./Datasets/FashionMNIST', train=True, download=True,
                                                    transform=transforms.ToTensor())
    mnist_test = torchvision.datasets.FashionMNIST(root='./Datasets/FashionMNIST', train=False, download=True,
                                                   transform=transforms.ToTensor())
    if sys.platform.startswith('win'):
        num_workers = 0  # 0表示不用额外的进程来加速读取数据
    else:
        num_workers = 4
    train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=num_workers)
    test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False, num_workers=num_workers)
    return train_iter,test_iter

def evaluate_accuracy(data_iter, net):
    acc_sum, n = 0.0, 0
    for X, y in data_iter:
        acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()
        n += y.shape[0]
    return acc_sum / n

def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size,
              params=None, lr=None, optimizer=None):
    for epoch in range(num_epochs):
        train_l_sum, train_acc_sum, n = 0.0, 0.0, 0
        for X, y in train_iter:
            y_hat = net(X)
            l = loss(y_hat, y).sum()

            # 梯度清零
            if optimizer is not None:
                optimizer.zero_grad()
            elif params is not None and params[0].grad is not None:
                for param in params:
                    param.grad.data.zero_()

            l.backward()
            if optimizer is None:
                sgd(params, lr, batch_size)
            else:
                optimizer.step()

            train_l_sum += l.item()
            train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()
            n += y.shape[0]
        test_acc = evaluate_accuracy(test_iter, net)
        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
              % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))

if __name__ == '__main__':
    # 获取数据

    batch_size = 256
    train_iter, test_iter = load_data_fashion_mnist(batch_size)

    num_inputs = 784
    num_outputs = 10
    net = LinearNet(num_inputs, num_outputs)
    # net = nn.Sequential(
    #     # FlattenLayer(),
    #     # nn.Linear(num_inputs, num_outputs)
    #     OrderedDict([
    #         ('flatten', FlattenLayer()),
    #         ('linear', nn.Linear(num_inputs, num_outputs))
    #     ])
    # )

    init.normal_(net.linear.weight, mean=0, std=0.01)
    init.constant_(net.linear.bias, val=0)

    loss = nn.CrossEntropyLoss()

    optimizer = torch.optim.SGD(net.parameters(), lr=0.1)

    num_epochs = 5
    train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, None, None, optimizer)