1. 基本数据结构
和pytorch等中的tensor类似，mxnet中的ndarray或者nd，用来操作矩阵或者张量数据。基本操作类似于Numpy， 支持计算，索引等。
创建矩阵
from mxnet import nd #或者 from mxnet import ndarray as nd #创建矩阵 x1 = nd.array([[1, 2,], [3, 4]]) x2 = nd.random.uniform(1, 10, shape=(3, 3)) #3*3的矩阵 x3 = nd.random.randn(2,3) #2*3 的矩阵 x4 = nd.random.randint(1, 10, shape=(2, 3)) #2*3 的矩阵 x5 = nd.ones(shape=(2, 2)) #2*2 的矩阵 x6 = nd.full(shape=(2, 3), val=2) #2*3 的矩阵, 值为2 print(x1.shape, x1.size, x1.dtype) #(2, 2) 4 <class ‘numpy.float32‘>
操作矩阵
x = nd.random.randn(2, 3) y = nd.random.randn(2, 3) print(y.exp()) # 2*3 的矩阵 print(x*y) # 2*3 的矩阵 print(nd.dot(x, y.T)) # 2*2 的矩阵
#和numpy相互转换
a = y.asnumpy()
print(a)
a = nd.array(np.ones((2, 3)))
print(a)
矩阵索引
y = nd.random.randint(1, 10, shape=(3, 3)) print(y[1, 2]) # [5] print(y[:, 1:3]) # 3*2 y[:,1:3] = 2 #赋值 y[1:2,0:2] = 4 #赋值 print(y)
2. 创建神经网络
mxnet中gluon包中包含神经网络创建中的相关操作，和pytorch类似，可以继承block来创建神经网络，只需定义网络结构和实现前向传播函数。
方式一： 继承nn.Block
class MyNet(nn.Block): def __init__(self): super(MyNet, self).__init__() self.features = nn.Sequential() self.features.add( nn.Conv2D(channels=16, kernel_size=5, strides=(1, 1), padding=(0, 0), activation="relu"), #和pytorch不同之处：不需要设置输入通道数，可以设置激活函数 nn.MaxPool2D(pool_size=(2, 2), stides=2, padding=0), nn.Conv2D(channels=32, kernel_size=3, strides=(1, 1), padding=(0, 0), activation="relu"), nn.MaxPool2D(pool_size=(2, 2), stides=2, padding=0), ) self.fc = nn.Sequential() self.fc.add( nn.Dense(units=120, activation="relu"), #和pytorch不同之处：不需要设置输入向量的大小，可以设置激活函数 nn.Dense(units=84, activation="relu"), nn.Dense(units=10) ) def forward(self, x): x = self.features(x) x = self.fc(x) return x
net = MyNet()
net.initialize() # 网络内部的参数必须先进行初始化 (pytorch中需要逐层进行初始化)
x = nd.random.uniform(shape=(1, 3, 300, 300))
print(net(x))
方式二：直接利用nn.Sequential
net = nn.Sequential() net.add( nn.Conv2D(channels=16, kernel_size=5, strides=(1, 1), padding=(0, 0), activation="relu"), # 和pytorch不同之处：不需要设置输入通道数，可以设置激活函数 nn.MaxPool2D(pool_size=(2, 2), strides=2, padding=0), nn.Conv2D(channels=32, kernel_size=3, strides=(1, 1), padding=(0, 0), activation="relu"), nn.MaxPool2D(pool_size=(2, 2), strides=2, padding=0), nn.Dense(units=120, activation="relu"), # 和pytorch不同之处：不需要设置输入向量的大小，可以设置激活函数 nn.Dense(units=84, activation="relu"), nn.Dense(units=10) ) net.initialize() # 网络内部的参数必须先进行初始化 (pytorch中需要逐层进行初始化) x = nd.random.uniform(shape=(1, 3, 300, 300)) print(net(x))
3. 神经网络训练
梯度反向传播，mxnet会自动求导，需要利用mxnet的autograd,如下：
from mxnet import nd from mxnet import autograd x = nd.array([[1, 2], [3, 4]]) x.attach_grad() #1. 声明存储导数的地方 with autograd.record(): #2. 该上下文中的过程，反向传播时会自动求导 y = 2*x*x y.backward() #3. 反向传播; 会自动求和再计算导数，相当于y.sum().backward() print(x.grad) #4. 取导数值
3.1. 加载数据
自己加载数据，主要需要继承mxnet.gluon.data.Dataset，然后传递给mxnet.gluon.data.DataLoader。有几个坑：
A. Dataset返回img和label， label不能为字符串格式
B. Dataloader中的num_workers设置大于0时, 对于windows系统，由于采用多进程，需要写在__main__中；若还是报错时，num_workers改为0
#coding:utf-8 import mxnet as mx from mxnet import gluon from mxnet.gluon.data import Dataset, DataLoader from mxnet.gluon.data.vision import transforms import os import cv2 #1.继承mxnet.gluon.data.Dataset, 实现__len__和__getitem__（返回每张图片和标注） class MyDataset(Dataset): def __init__(self, img_root, anno_file): assert os.path.exists(anno_file), print("Annotation file {} not exist".format(anno_file)) self.img_root = img_root self.anno_file = anno_file with open(anno_file, "r", encoding="utf-8") as f: lines = f.readlines() self.items = [line.strip().split() for line in lines if line.strip()] def __len__(self): return len(self.items) def __getitem__(self, x): img_name, label = self.items[x] img_path = os.path.join(self.img_root, img_name) assert os.path.exists(img_path), print("img_file {} does not exist".format(img_path)) img = mx.image.imread(img_path) return img, label #注意此处label为字符串会报错 if __name__ == "__main__": #2. 将dataset传入mxnet.gluon.data.Dataloader img_root = r"D:\data\synthtext" anno_file = r"D:\data\synthtext\labels.txt" dataset = MyDataset(img_root, anno_file) transformer = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)) ]) # dataset.transform_first(transformer), 对图片进行增强(即对__getitem__返回的第一项进行处理) train_data = DataLoader(dataset.transform_first(transformer), batch_size=2, shuffle=True, num_workers=0) print(train_data) for img, label in train_data: print(label) print(img.shape)
3.2 定义网络
见文章上面第二点
3.3 定义损失函数
gluon.loss包含了部分常用的Loss，如下：
loss = gluon.loss.SoftmaxCrossEntropyLoss() #交叉熵损失函数 loss = gluon.loss.L2Loss() #均方差损失函数 loss = gluon.loss.CTCLoss() # CTC损失函数 loss = gluon.loss.L1Loss() # L1 损失函数 #位找到smoothL1,发现两个相关的 mx.nd.smooth_l1()； mx.metric.Loss("SmoothL1")
_all__ = [‘Loss‘, ‘L2Loss‘, ‘L1Loss‘,
‘SigmoidBinaryCrossEntropyLoss‘, ‘SigmoidBCELoss‘,
‘SoftmaxCrossEntropyLoss‘, ‘SoftmaxCELoss‘,
‘KLDivLoss‘, ‘CTCLoss‘, ‘HuberLoss‘, ‘HingeLoss‘,
‘SquaredHingeLoss‘, ‘LogisticLoss‘, ‘TripletLoss‘, ‘PoissonNLLLoss‘, ‘CosineEmbeddingLoss‘]
3.4 定义优化器
优化器定义在gluon.Trainer() ，第一个参数params为网络参数，第二个参数optimizer为优化器的名字，第三个参数optimizer_params为传给优化器的参数
支持的optimizer如下：
__all__ = [ ‘AdaDelta‘, ‘AdaGrad‘, ‘Adam‘, ‘Adamax‘, ‘DCASGD‘, ‘FTML‘, ‘Ftrl‘, ‘LBSGD‘, ‘NAG‘, ‘NDabs‘, ‘Nadam‘, ‘Optimizer‘, ‘RMSProp‘, ‘SGD‘, ‘SGLD‘, ‘Signum‘, ‘Test‘, ‘ccSGD‘, ]
共同支持的optimizer_params如下： （不同优化器还有其特定的参数）
Parameters rescale_grad (float, optional, default 1.0) – Multiply the gradient with rescale_grad before updating. Often choose to be 1.0/batch_size. param_idx2name (dict from int to string, optional, default None) – A dictionary that maps int index to string name. clip_gradient (float, optional, default None) – Clip the gradient by projecting onto the box [-clip_gradient, clip_gradient]. learning_rate (float) – The initial learning rate. If None, the optimization will use the learning rate from lr_scheduler. If not None, it will overwrite the learning rate in lr_scheduler. If None and lr_scheduler is also None, then it will be set to 0.01 by default. lr_scheduler (LRScheduler, optional, default None) – The learning rate scheduler. wd (float, optional, default 0.0) – The weight decay (or L2 regularization) coefficient. Modifies objective by adding a penalty for having large weights. sym (Symbol, optional, default None) – The Symbol this optimizer is applying to. begin_num_update (int, optional, default 0) – The initial number of updates. multi_precision (bool, optional, default False) – Flag to control the internal precision of the optimizer. False: results in using the same precision as the weights (default), True: makes internal 32-bit copy of the weights and applies gradients in 32-bit precision even if actual weights used in the model have lower precision. Turning this on can improve convergence and accuracy when training with float16. param_dict (dict of int -> gluon.Parameter, default None) – Dictionary of parameter index to gluon.Parameter, used to lookup parameter attributes such as lr_mult, wd_mult, etc. param_dict shall not be deep copied. aggregate_num (int, optional, default None) – Number of weights to be aggregated in a list. They are passed to the optimizer for a single optimization step. In default, only one weight is aggregated. When aggregate_num is set to numpy.inf, all the weights are aggregated. use_fused_step (bool, optional, default None) – Whether or not to use fused kernels for optimizer. When use_fused_step=False, step is called, otherwise, fused_step is called. Properties – ---------- – learning_rate – The current learning rate of the optimizer. Given an Optimizer object optimizer, its learning rate can be accessed as optimizer.learning_rate.
常用优化器使用如下：
#优化器 #1.动量法 gluon.Trainer(params=net.collect_params(), optimizer="SGD", optimizer_params={"learning_rate":0.001, "wd":0.00005, "momentum":0.9}) #2. 自适应 #AdaGrad gluon.Trainer(params=net.collect_params(), optimizer="AdaGrad", optimizer_params={"learning_rate":0.001, "wd":0.00005,}) #RMSProp gluon.Trainer(params=net.collect_params(), optimizer="RMSProp", optimizer_params={"learning_rate": 0.001, "wd": 0.00005, "momentum":0.9}) #Adam gluon.Trainer(params=net.collect_params(), optimizer="RMSProp", optimizer_params={"learning_rate": 0.001, "wd": 0.00005})
3.5 模型训练
for epoch in range(10): train_loss, train_acc, valid_acc = 0., 0., 0. tic = time.time() for data, label in train_data: # forward + backward with autograd.record(): output = net(data) loss = softmax_cross_entropy(output, label) loss.backward() # update parameters trainer.step(batch_size) # calculate training metrics train_loss += loss.mean().asscalar() train_acc += acc(output, label) # calculate validation accuracy for data, label in valid_data: valid_acc += acc(net(data), label) print("Epoch %d: loss %.3f, train acc %.3f, test acc %.3f, in %.1f sec" % ( epoch, train_loss / len(train_data), train_acc / len(train_data), valid_acc / len(valid_data), time.time() - tic))
4. 网络参数保存和加载
Block 只能保存网络参数，如下：
net = nn.Sequential()
net.add(
nn.Conv2D(channels=16, kernel_size=5, strides=(1, 1),
padding=(0, 0), activation="relu"), # 和pytorch不同之处：不需要设置输入通道数，可以设置激活函数
nn.MaxPool2D(pool_size=(2, 2), strides=2, padding=0),
nn.Conv2D(channels=32, kernel_size=3, strides=(1, 1),
padding=(0, 0), activation="relu"),
nn.MaxPool2D(pool_size=(2, 2), strides=2, padding=0),
nn.Dense(units=120, activation="relu"), # 和pytorch不同之处：不需要设置输入向量的大小，可以设置激活函数
nn.Dense(units=84, activation="relu"),
nn.Dense(units=10)
)
#1 保存网络权重参数 net.save_parameters("checkpoint.params") #2 加载权重参数 net.load_parameters("checkpoint.params", ctx=None, allow_missing=False, ignore_extra=False, cast_dtype=False, dtype_source=‘current‘) ctx: 默认为Cpu allow_missing: True时表示：网络结构中存在, 参数文件中不存在参数，不加载 ignore_extra: True时表示: 参数文件中存在，网络结构中不存在的参数，不加载
HybridBlock可以向Block一样保存网络参数，也可以同时保存网络结构和网络参数， 如下：
net = nn.HybridSequential() net.add( nn.Conv2D(channels=16, kernel_size=5, strides=(1, 1), padding=(0, 0), activation="relu"), # 和pytorch不同之处：不需要设置输入通道数，可以设置激活函数 nn.MaxPool2D(pool_size=(2, 2), strides=2, padding=0), nn.Conv2D(channels=32, kernel_size=3, strides=(1, 1), padding=(0, 0), activation="relu"), nn.MaxPool2D(pool_size=(2, 2), strides=2, padding=0), nn.Dense(units=120, activation="relu"), # 和pytorch不同之处：不需要设置输入向量的大小，可以设置激活函数 nn.Dense(units=84, activation="relu"), nn.Dense(units=10) ) # 1.对于HybridBlock, 可以同时保存网络结构和权重参数 #首先要进行hybridize()和一次前向传播，才能进行export net.initialize() net.hybridize() x = mx.nd.zeros((1, 3, 100, 100)) print(net(x)) net.export(path="./checkpoint", epoch=1) #同时生成checkpoint-0001.params 和 checkpoint-symbol.json # # net.save_parameters("./checkpoint.params")
#2. 加载export的网络结构(json)和权重参数(params)
#或者mx.SymbolBlock.imports()
net = gluon.SymbolBlock.imports(symbol_file="./checkpoint-symbol.json",
input_names=["data"],
param_file="./checkpoint-0100.params",
ctx=mx.cpu())
net.hybridize()
x = mx.nd.zeros((1, 3, 100, 100))
print(net(x))
# net = mx.mod.Module.load(prefix="./checkpoint", epoch=100)
5. 使用GPU
在进行训练和计算时网络参数和数据必须在同一环境下，同在CPU或同在GPU，采用GPU计算矩阵时能加速运算；可以在GPU上操作数据和网络，如下：
数据：可以在GPU上创建数据，也可以在CPU上创建数据，载移动到GPU
#1. 在GPU上创建数据，或者将数据从cpu移动到GPU #GPU创建 x = mx.nd.zeros((1, 3, 100, 100), ctx=mx.gpu(0)) print(x) #cpu创建，复制一份到GPU x = mx.nd.zeros((1, 3, 100, 100)) x = x.copyto(mx.gpu(0)) print(x) # cpu创建，复制一份移动到GPU x = mx.nd.zeros((1, 3, 100, 100)) x = x.as_in_context(mx.gpu(0)) print(x)
网络：可以在GPU上加载网络参数，或者在CPU上加载，随后移动到GPU
#2.在GPU上加载网络参数，或者将网络参数移动到GPU net = nn.Sequential() net.add( nn.Conv2D(channels=16, kernel_size=3, strides=1, padding=1), nn.Dense(18) ) #GPU上初始化参数 net.initialize(init=mx.init.Xavier(), ctx=mx.gpu(0)) net.load_parameters("./checkpoint.params", ctx=mx.gpu(0)) # #CPU上初始化参数，移动到GPU net.initialize(init=mx.init.Xavier()) net.collect_params().reset_ctx(mx.gpu())
https://github.com/apache/incubator-mxnet
https://zhuanlan.zhihu.com/p/39420301
http://mxnet.incubator.apache.org/