第十二章 超参数设定和网络训练
12.2 训练技巧
12.2.3 批规范化操作
在PyTorch中，可按如下调用示例使用批规范化操作（Batch Normalization，BN）操作：
1# batch normalization (PyTorch)
2import torch
3import torch.nn as nn
4
5# Ln.6-Ln.8: Apply batch normalization.
6m = nn.BatchNorm2d(4)
7inp = torch.randn(1, 4, 3, 3)
8oup = m(inp)
9print(m.weight, m.bias)
在MindSpore中的实现为：
1# batch normalization (MindSpore)
2import numpy as np
3import mindspore.nn as nn
4import mindspore as ms
5
6x = ms.Tensor(np.array([[[[1, 2], [1, 2]], [[3, 4], [3, 4]]]]).astype(np.float32))
7# Ln.8-Ln.9: Apply batch normalization.
8bn = nn.BatchNorm2d(num_features=2, momentum=0.8)
9output = bn(x)
10print(output)
BN更适用于mini-batch较大的场景，且需对训练数据做好充分的随机打乱从而获得更加接近整体分布的mini-batch统计量。此外，BN操作还需在运行过程中统计每个mini-batch的一阶和二阶统计量，这也限制了其在递归神经网络（recurrent neural networks）和动态神经网络（dynamic neural networks）中的应用。针对BN的不足，层规范化（layer normalization）操作被提出。该操作针对一整层的所有维度的输入计算对应均值与方差，之后，使用类似BN的规范化操作来转换各个维度的输入。层规范化操作基于PyTorch的具体实现细节可参考如下示例：
1# implementations of layer normalization (PyTorch)
2class LayerNorm(nn.Module):
3
4 def __init__(self, num_channels, eps=1e-05, affine=True):
5 super().__init__()
6 self.num_channels = num_channels
7 self.eps = eps
8 self.affine = affine
9 if self.affine:
10 self.weight = nn.Parameter(torch.ones(num_channels))
11 self.bias = nn.Parameter(torch.zeros(num_channels))
12 else:
13 self.weight = None
14 self.bias = None
15 self.reset_parameters()
16
17 def reset_parameters(self):
18 if self.affine:
19 nn.init.ones_(self.weight)
20 nn.init.zeros_(self.bias)
21
22 def forward(self, x):
23 N, C, H, W = x.shape
24 assert C == self.num_channels
25 x = x.reshape(N, -1)
26
27 # Ln.28-Ln.35: Calculate mean and variance and apply batch normalization.
28 mean = x.mean(axis=1, keepdim=True)
29 var = (x ** 2).mean(axis=1, keepdim=True) - mean * mean
30
31 x = (x - mean) / (var + self.eps).sqrt()
32 x = x.reshape(N, C, H, W)
33 if self.affine:
34 x = self.weight.reshape(1, -1, 1, 1) * x + self.bias.reshape(1, -1, 1, 1)
35 return x
在MindSpore中的实现为：
1# implementations of layer normalization (MindSpore)
2class LayerNorm(Cell):
3 """
4 Applies Layer Normalization over a mini-batch of inputs.
5
6 Inputs:
7 - **x** (Tensor) - The shape of `x` is :math:`(x_1, x_2, ..., x_R)`,
8 and `input_shape[begin_norm_axis:]` is equal to `normalized_shape`.
9
10 Outputs:
11 Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `x`.
12 """
13
14 def __init__(self,
15 normalized_shape,
16 begin_norm_axis=-1,
17 begin_params_axis=-1,
18 gamma_init='ones',
19 beta_init='zeros',
20 epsilon=1e-7
21 ):
22 """Initialize LayerNorm."""
23 super(LayerNorm, self).__init__()
24 if not isinstance(normalized_shape, (tuple, list)):
25 raise TypeError(f"For '{self.cls_name}', the type of 'normalized_shape' must be tuple[int] or list[int], "
26 f"but got {normalized_shape} and the type is {type(normalized_shape)}.")
27 self.normalized_shape = normalized_shape
28 self.begin_norm_axis = begin_norm_axis
29 self.begin_params_axis = begin_params_axis
30 self.epsilon = epsilon
31 self.gamma = Parameter(initializer(
32 gamma_init, normalized_shape), name="gamma")
33 self.beta = Parameter(initializer(
34 beta_init, normalized_shape), name="beta")
35 # Ln.36-Ln.38: Initialize the LayerNorm funtion.
36 self.layer_norm = P.LayerNorm(begin_norm_axis=self.begin_norm_axis,
37 begin_params_axis=self.begin_params_axis,
38 epsilon=self.epsilon)
39
40 def construct(self, input_x):
41 # Ln.42: Apply the LayerNorm funtion.
42 y, _, _ = self.layer_norm(input_x, self.gamma.astype(input_x.dtype), self.beta.astype(input_x.dtype))
43 return y
44
45 def extend_repr(self):
46 return 'normalized_shape={}, begin_norm_axis={}, begin_params_axis={}, gamma{}, beta={}'.format(
47 self.normalized_shape, self.begin_norm_axis, self.begin_params_axis, self.gamma, self.beta)
不过，层规范化潜在假设了同一规范化操作应作用于整层神经元，在某种程度限制了神经元多样性，这也为后续的组规范化（group normalization）操作的提出埋下了伏笔。该操作对卷积通道进行了分组，既有效解决了mini-batch大小对于BN的影响，也在一定程度缓解了层规范化对于层内神经元多样性的局限。组规范化操作基于PyTorch的具体实现细节可参考如下示例：
1# implementations of group normalization (PyTorch)
2class GroupNorm(nn.Module):
3
4 def __init__(self, num_groups, num_channels, eps=1e-05, affine=True):
5 super().__init__()
6 assert num_channels % num_groups == 0
7 self.num_groups = num_groups
8 self.num_channels = num_channels
9 self.eps = eps
10 self.affine = affine
11 if self.affine:
12 self.weight = nn.Parameter(torch.ones(num_channels))
13 self.bias = nn.Parameter(torch.zeros(num_channels))
14 else:
15 self.weight = None
16 self.bias = None
17 self.reset_parameters()
18
19 def reset_parameters(self):
20 if self.affine:
21 nn.init.ones_(self.weight)
22 nn.init.zeros_(self.bias)
23
24 def forward(self, x):
25 N, C, H, W = x.shape
26 assert C == self.num_channels
27 # Ln.28-Ln.30: Group the inputs and calculate the mean and variance.
28 x = x.reshape(N, self.num_groups, -1)
29 mean = x.mean(axis=2, keepdims=True)
30 var = (x ** 2).mean(axis=2, keepdim=True) - mean * mean
31
32 # Ln.33-Ln.36: Apply group normalization.
33 x = (x - mean) / (var + self.eps).sqrt()
34 x = x.reshape(N, C, H, W)
35 if self.affine:
36 x = self.weight.reshape(1, -1, 1, 1) * x + self.bias.reshape(1, -1, 1, 1)
37
38 return x
在MindSpore中的实现为：
1# implementations of group normalization (MindSpore)
2class GroupNorm(Cell):
3 """
4 Group Normalization over a mini-batch of inputs.
5
6 Inputs:
7 - **x** (Tensor) - The input feature with shape (N, C, H, W) .
8
9 Outputs:
10 Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `x`.
11
12 """
13
14 def __init__(self, num_groups, num_channels, eps=1e-05, affine=True, gamma_init='ones', beta_init='zeros'):
15 """Initialize GroupNorm."""
16 super(GroupNorm, self).__init__()
17 self.num_groups = validator.check_positive_int(num_groups, "num_groups", self.cls_name)
18 self.num_channels = validator.check_positive_int(num_channels, "num_channels", self.cls_name)
19 if num_channels % num_groups != 0:
20 raise ValueError(f"For '{self.cls_name}', the 'num_channels' must be divided by 'num_groups', "
21 f"but got 'num_channels': {num_channels}, 'num_groups': {num_groups}.")
22 self.eps = validator.check_value_type('eps', eps, (float,), type(self).__name__)
23 self.affine = validator.check_bool(affine, arg_name="affine", prim_name=self.cls_name)
24
25 self.gamma = Parameter(initializer(
26 gamma_init, num_channels), name="gamma", requires_grad=affine)
27 self.beta = Parameter(initializer(
28 beta_init, num_channels), name="beta", requires_grad=affine)
29 self.shape = F.shape
30 self.reshape = F.reshape
31 self.reduce_mean = P.ReduceMean(keep_dims=True)
32 self.square = F.square
33 self.reduce_sum = P.ReduceSum(keep_dims=True)
34 self.sqrt = P.Sqrt()
35
36 def _cal_output(self, x):
37 """calculate groupnorm output"""
38 batch, channel, height, width = self.shape(x)
39 _channel_check(channel, self.num_channels, self.cls_name)
40
41 # Ln.42-Ln.45: Group the inputs and calculate the mean, variance and standard deviation.
42 x = self.reshape(x, (batch, self.num_groups, -1))
43 mean = self.reduce_mean(x, 2)
44 var = self.reduce_sum(self.square(x - mean), 2) / (channel * height * width / self.num_groups)
45 std = self.sqrt(var + self.eps)
46
47 # Ln.48-Ln.50: Apply group normalization.
48 x = (x - mean) / std
49 x = self.reshape(x, (batch, channel, height, width))
50 output = x * self.reshape(self.gamma, (-1, 1, 1)) + self.reshape(self.beta, (-1, 1, 1))
51 return output
52
53 def construct(self, x):
54 _shape_check(self.shape(x), self.cls_name)
55 _check_dtype(x.dtype, [mstype.float16, mstype.float32], "input", self.cls_name)
56 output = self._cal_output(x)
57 return output
58
59 def extend_repr(self):
60 return 'num_groups={}, num_channels={}'.format(self.num_groups, self.num_channels)
在规范化操作的研究中，研究者还提出了一种更为极致的规范化操作，即示例规范化（instance normalization，或contrast normalization），它针对每个样本的每个通道单独计算其统计信息。示例规范化操作基于PyTorch的具体实现细节可参考如下示例：
1# implementations of instance normalization (PyTorch)
2class InstanceNorm(nn.Module):
3
4 def __init__(self, num_channels, eps=1e-05, affine=True):
5 super().__init__()
6 self.num_channels = num_channels
7 self.eps = eps
8 self.affine = affine
9 if self.affine:
10 self.weight = nn.Parameter(torch.ones(num_channels))
11 self.bias = nn.Parameter(torch.zeros(num_channels))
12 else:
13 self.weight = None
14 self.bias = None
15 self.reset_parameters()
16
17 def reset_parameters(self):
18 if self.affine:
19 nn.init.ones_(self.weight)
20 nn.init.zeros_(self.bias)
21
22 def forward(self, x):
23 N, C, H, W = x.shape
24 assert C == self.num_channels
25 # Ln.26-Ln.28: Calculate the mean and variance for a single sample of a single channel.
26 x = x.reshape(N, C, -1)
27 mean = x.mean(axis=2, keepdim=True)
28 var = (x ** 2).mean(axis=2, keepdim=True) - mean * mean
29
30 # Ln.31-Ln.34: Apply instance normalization.
31 x = (x - mean) / (var + self.eps).sqrt()
32 x = x.reshape(N, C, H, W)
33 if self.affine:
34 x = self.weight.reshape(1, -1, 1, 1) * x + self.bias.reshape(1, -1, 1, 1)
35
36 return x
在MindSpore中的实现为：
1# implementations of instance normalization (MindSpore)
2class _InstanceNorm(Cell):
3 """Instance Normalization base class."""
4
5 @cell_attr_register
6 def __init__(self,
7 num_features,
8 eps=1e-5,
9 momentum=0.1,
10 affine=True,
11 gamma_init='ones',
12 beta_init='zeros',
13 input_dims='2d'):
14 """Initialize Normalization base class."""
15 super(_InstanceNorm, self).__init__()
16 validator.check_value_type('num_features', num_features, [int], self.cls_name)
17 validator.check_value_type('eps', eps, [float], self.cls_name)
18 validator.check_value_type('momentum', momentum, [float], self.cls_name)
19 validator.check_value_type('affine', affine, [bool], self.cls_name)
20 args_input = {"gamma_init": gamma_init, "beta_init": beta_init}
21 self.check_types_valid(args_input, 'InstanceNorm2d')
22 if num_features < 1:
23 raise ValueError(f"For '{self.cls_name}', the 'num_features' must be at least 1, but got {num_features}.")
24
25 if momentum < 0 or momentum > 1:
26 raise ValueError(f"For '{self.cls_name}', the 'momentum' must be a number in range [0, 1], "
27 f"but got {momentum}.")
28 self.num_features = num_features
29 self.eps = eps
30 self.input_dims = input_dims
31 self.moving_mean = Parameter(initializer('zeros', num_features), name="mean", requires_grad=False)
32 self.moving_variance = Parameter(initializer('ones', num_features), name="variance", requires_grad=False)
33 self.gamma = Parameter(initializer(
34 gamma_init, num_features), name="gamma", requires_grad=affine)
35 self.beta = Parameter(initializer(
36 beta_init, num_features), name="beta", requires_grad=affine)
37
38 self.shape = P.Shape()
39 self.momentum = momentum
40 # Ln.41: Initialize the InstanceNorm funtion.
41 self.instance_bn = P.InstanceNorm(epsilon=self.eps, momentum=self.momentum)
42
43 def construct(self, x):
44 _shape_check_in(self.shape(x), self.input_dims, self.cls_name)
45 # Ln.46-Ln.50: Apply instance normalization.
46 return self.instance_bn(x,
47 self.gamma,
48 self.beta,
49 self.moving_mean,
50 self.moving_variance)[0]
51
52 def extend_repr(self):
53 return 'num_features={}, eps={}, momentum={}, gamma={}, beta={}, moving_mean={}, moving_variance={}'.format(
54 self.num_features, self.eps, self.momentum, self.gamma, self.beta, self.moving_mean, self.moving_variance)
55
56 def check_types_valid(self, args_dict, name):
57 for key, _ in args_dict.items():
58 val = args_dict[key]
59 if not isinstance(val, (Tensor, numbers.Number, str, Initializer)):
60 raise TypeError(f"For '{self.cls_name}', the type of '{key}' must be in "
61 f"[Tensor, numbers.Number, str, Initializer], but got type {type(val).__name__}.")
62 if isinstance(val, Tensor) and val.dtype != mstype.float32:
63 raise TypeError(f"For '{self.cls_name}', the type of '{key}' must be float32, "
64 f"but got {val.dtype}.")
12.2.4 网络模型优化算法选择
12.2.4.1 随机梯度下降法
SGD基于PyTorch的实现可参考：
1# SGD (PyTorch)
2for param in params:
3 # Ln.4-Ln.5: Obtain the gradient and update it.
4 d_p = param.grad.data
5 param.add_(d_p, alpha=-lr)
基于MindSpore的实现为：
1# SGD (MindSpore)
2for param in params:
3 # Ln.4-Ln.5: Obtain the gradient and update it.
4 d_p = param.grad.asnumpy()
5 param = ops.AssignAdd(param, Tensor(-lr * d_p))
12.2.4.2 基于动量的随机梯度下降法
然而，SGD在优化神经网络时也面临一些挑战。它可能会因为更新步长较大或梯度波动导致的震荡而使收敛速度缓慢，并因此陷入到局部极小值。这种情况下，普通的SGD可能表现出不稳定性和收敛速度慢的问题。为了克服这些问题，动量被引入到优化算法中。基于动量的随机梯度下降法在PyTorch中的具体实现可参考如下示例：
1# momentum SGD (PyTorch)
2for i, param in enumerate(params):
3 d_p = param.grad.data
4
5 # Ln.6-Ln.11: Obtain the momentum and update it.
6 buf = momentum_buffer_list[i]
7 if buf is None:
8 buf = torch.clone(d_p).detach()
9 momentum_buffer_list[i] = buf
10 else:
11 buf.mul_(momentum).add_(d_p, alpha=1 - dampening)
12
13 # Ln.14-Ln.15: Update the gradient and parameters.
14 d_p = buf
15 param.add_(d_p, alpha=-lr)
其基于MindSpore的实现为：
1# momentum SGD (MindSpore)
2for i, param in enumerate(params):
3 d_p = param.grad.asnumpy()
4
5 # Ln.6-Ln.11: Obtain the momentum and update it.
6 buf = momentum_buffer_list[i]
7 if buf is None:
8 buf = Tensor(d_p.copy())
9 momentum_buffer_list[i] = buf
10 else:
11 buf = ops.Mul()(buf, momentum) + ops.Mul()(d_p, 1 - dampening)
12
13 # Ln.14-Ln.15: Update the gradient and parameters.
14 d_p = buf.asnumpy()
15 param = ops.AssignAdd()(param, Tensor(-lr * d_p))
12.2.4.3 Nesterov型动量随机梯度下降法
Nesterov动量在计算梯度之前，首先根据之前的动量更新来调整当前位置。这种预测下一个位置的方式有助于更准确地估计当前位置的梯度，从而减少了一些摆动和不必要的更新。在PyTorch中，Nesterov型动量随机下降法可实现为：
1# Nesterov momentum SGD (PyTorch)
2for i, param in enumerate(params):
3 d_p = param.grad.data
4
5 # Ln.6-Ln.11: Obtain the momentum and update it.
6 buf = momentum_buffer_list[i]
7 if buf is None:
8 buf = torch.clone(d_p).detach()
9 momentum_buffer_list[i] = buf
10 else:
11 buf.mul_(momentum).add_(d_p, alpha=1 - dampening)
12
13 # Ln.14-Ln.15: Update the gradient by momentum and parameters.
14 d_p = d_p.add(buf, alpha=momentum)
15 param.add_(d_p, alpha=-lr)
其基于MindSpore的实现为：
1# Nesterov momentum SGD (MindSpore)
2for i, param in enumerate(params):
3 d_p = param.grad.asnumpy()
4
5 # Ln.6-Ln.11: Obtain the momentum and update it.
6 buf = momentum_buffer_list[i]
7 if buf is None:
8 buf = Tensor(d_p.copy())
9 momentum_buffer_list[i] = buf
10 else:
11 buf = ops.Mul()(buf, momentum) + ops.Mul()(d_p, 1 - dampening)
12
13 # Ln.14-Ln.15: Update the gradient by momentum and parameters.
14 d_p = ops.TensorAdd()(d_p, ops.Mul()(buf, momentum)).asnumpy()
15 param = ops.AssignAdd()(param, Tensor(-lr * d_p))
12.2.4.4 Adagrad法
Adagrad是一种自适应学习率的优化算法，它针对每个参数应用不同的学习率。在PyTorch中，Adagrad法的实现可参考如下示例：
1# Adagrad (PyTorch)
2for (param, grad, state_sum, step) in zip(params, grads, state_sums, state_steps):
3 if weight_decay != 0:
4 if grad.is_sparse:
5 raise RuntimeError("weight_decay option is not compatible with sparse gradients")
6 grad = grad.add(param, alpha=weight_decay)
7
8 # Ln.9: Calculate the dynamic learning rate associated with the step.
9 clr = lr / (1 + (step - 1) * lr_decay)
10
11 # Ln.12-Ln.25: Update the gradient and parameters by the dynamic learning rate.
12 if grad.is_sparse:
13 grad = grad.coalesce() # the update is non-linear so indices must be unique
14 grad_indices = grad._indices()
15 grad_values = grad._values()
16 size = grad.size()
17
18 state_sum.add_(_make_sparse(grad, grad_indices, grad_values.pow(2)))
19 std = state_sum.sparse_mask(grad)
20 std_values = std._values().sqrt_().add_(eps)
21 param.add_(_make_sparse(grad, grad_indices, grad_values / std_values), alpha=-clr)
22 else:
23 state_sum.addcmul_(grad, grad, value=1)
24 std = state_sum.sqrt().add_(eps)
25 param.addcdiv_(grad, std, value=-clr)
其基于MindSpore的实现为：
1# Adagrad (MindSpore)
2from mindspore import Tensor
3import mindspore.ops as ops
4
5for (param, grad, state_sum, step) in zip(params, grads, state_sums, state_steps):
6 if weight_decay != 0:
7 if grad.is_sparse():
8 raise RuntimeError("weight_decay option is not compatible with sparse gradients")
9 grad = grad + weight_decay * param
10
11 # Ln.9: Calculate the dynamic learning rate associated with the step.
12 clr = lr / (1 + (step - 1) * lr_decay)
13
14 # Ln.12-Ln.25: Update the gradient and parameters by the dynamic learning rate.
15 if grad.is_sparse():
16 grad = grad.coalesce() # the update is non-linear so indices must be unique
17 grad_indices = grad.indices
18 grad_values = grad.values
19 size = grad.shape
20
21 state_sum = state_sum + ops.SparseApplyFtrl(grad_indices, grad_values**2)
22 std = ops.SparseApplyFtrl(grad_indices, state_sum).sqrt() + eps
23 param = param - clr * ops.SparseApplyFtrl(grad_indices, grad_values / std)
24 else:
25 state_sum = state_sum + ops.Mul()(grad, grad)
26 std = ops.Sqrt()(state_sum) + eps
27 param = param - clr * (grad / std)
12.2.4.5 Adadelta法
Adadelta是一种自适应学习率的优化算法，旨在解决 Adagrad算法中学习率急剧下降的问题。在PyTorch中，Adadelta法的实现可参考如下示例：
1# Adadelta (PyTorch)
2for (param, grad, square_avg, acc_delta) in zip(params, grads, square_avgs, acc_deltas):
3 if weight_decay != 0:
4 grad = grad.add(param, alpha=weight_decay)
5
6 # Ln.7-Ln.11: Calculate the running average of the gradient squares by rho, and update the gradient and parameters with it.
7 square_avg.mul_(rho).addcmul_(grad, grad, value=1 - rho)
8 std = square_avg.add(eps).sqrt_()
9 delta = acc_delta.add(eps).sqrt_().div_(std).mul_(grad)
10 param.add_(delta, alpha=-lr)
11 acc_delta.mul_(rho).addcmul_(delta, delta, value=1 - rho)
其基于MindSpore的实现为：
1# Adadelta (MindSpore)
2for (param, grad, square_avg, acc_delta) in zip(params, grads, square_avgs, acc_deltas):
3 if weight_decay != 0:
4 grad = ops.TensorAdd()(grad, ops.Mul()(param, weight_decay))
5
6 # Ln.7-Ln.11: Calculate the running average of the gradient squares by rho, and update the gradient and parameters with it.
7 square_avg = ops.TensorAdd()(ops.Mul()(square_avg, rho), ops.Mul()(ops.Sub()(1, rho), ops.Square()(grad)))
8 std = ops.Sqrt()(ops.TensorAdd()(square_avg, eps))
9 delta = ops.Mul()(ops.Div()(ops.Sqrt()(ops.TensorAdd()(acc_delta, eps)), std), grad)
10 param = ops.AssignAdd()(param, ops.Neg()(ops.Mul()(lr, delta)))
11 acc_delta = ops.TensorAdd()(ops.Mul()(acc_delta, rho), ops.Mul()(ops.Sub()(1, rho), ops.Square()(delta)))
12.2.4.6 RMSProp法
RMSProp（Root Mean Square Propagation）则在解决Adagrad算法中学习率急剧下降的问题的同时，并改善Adadelta算法在某些情况下学习率过于稳定的问题。在PyTorch中，RMSProp法的实现可参考如下示例：
1# RMSProp (PyTorch)
2for i, param in enumerate(params):
3 grad = grads[i]
4 square_avg = square_avgs[i]
5
6 if weight_decay != 0:
7 grad = grad.add(param, alpha=weight_decay)
8
9 square_avg.mul_(alpha).addcmul_(grad, grad, value=1 - alpha)
10
11 if centered:
12 grad_avg = grad_avgs[i]
13 grad_avg.mul_(alpha).add_(grad, alpha=1 - alpha)
14 avg = square_avg.addcmul(grad_avg, grad_avg, value=-1).sqrt_().add_(eps)
15 else:
16 avg = square_avg.sqrt().add_(eps)
17
18 # Ln.19-Ln.24: Update parameters by gradient, global learning rate and the momentum `avg`.
19 if momentum > 0:
20 buf = momentum_buffer_list[i]
21 buf.mul_(momentum).addcdiv_(grad, avg)
22 param.add_(buf, alpha=-lr)
23 else:
24 param.addcdiv_(grad, avg, value=-lr)
其基于MindSpore的实现为：
1# RMSProp (MindSpore)
2for i, param in enumerate(params):
3 grad = grads[i]
4 square_avg = square_avgs[i]
5
6 if weight_decay != 0:
7 grad = ops.TensorAdd()(grad, ops.Mul()(param, weight_decay))
8
9 square_avg = ops.TensorAdd()(ops.Mul()(square_avg, alpha), ops.Mul()(ops.Sub()(1, alpha), ops.Square()(grad)))
10
11 if centered:
12 grad_avg = grad_avgs[i]
13 grad_avg = ops.TensorAdd()(ops.Mul()(grad_avg, alpha), ops.Mul()(ops.Sub()(1, alpha), grad))
14 avg = ops.Sqrt()(ops.TensorAdd()(square_avg, ops.Neg()(ops.Square()(grad_avg)))).add_(eps)
15 else:
16 avg = ops.Sqrt()(square_avg).add_(eps)
17
18 # Ln.19-Ln.24: Update parameters by gradient, global learning rate and the momentum `avg`.
19 if momentum > 0:
20 buf = momentum_buffer_list[i]
21 buf = ops.TensorAdd()(ops.Mul()(buf, momentum), ops.Div()(grad, avg))
22 param = ops.AssignAdd()(param, ops.Neg()(ops.Mul()(lr, buf)))
23 else:
24 param = ops.AssignAdd()(param, ops.Neg()(ops.Mul()(lr, ops.Div()(grad, avg))))
12.2.4.7 Adam法
Adam（Adaptive Moment Estimation）则结合了动量和自适应学习率算法的优点。在PyTorch中，Adam法的实现可参考如下示例：
1# Adam (PyTorch)
2for i, param in enumerate(params):
3 grad = grads[i]
4 exp_avg = exp_avgs[i]
5 exp_avg_sq = exp_avg_sqs[i]
6 step = state_steps[i]
7
8 bias_correction1 = 1 - beta1 ** step
9 bias_correction2 = 1 - beta2 ** step
10
11 if weight_decay != 0:
12 grad = grad.add(param, alpha=weight_decay)
13
14 # Ln.15-Ln.16: Decay the first and second moment running average coefficient
15 exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
16 exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
17 if amsgrad:
18 # Ln.19: Maintains the maximum of all 2nd moment running avg. till now
19 torch.maximum(max_exp_avg_sqs[i], exp_avg_sq, out=max_exp_avg_sqs[i])
20 # Ln.21: Use the max. for normalizing running avg. of gradient
21 denom = (max_exp_avg_sqs[i].sqrt() / math.sqrt(bias_correction2)).add_(eps)
22 else:
23 denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps)
24
25 step_size = lr / bias_correction1
26
27 param.addcdiv_(exp_avg, denom, value=-step_size)
其基于MindSpore的实现为：
1# Adam (MindSpore)
2for i, param in enumerate(params):
3 grad = grads[i]
4 exp_avg = exp_avgs[i]
5 exp_avg_sq = exp_avg_sqs[i]
6 step = state_steps[i]
7
8 bias_correction1 = 1 - beta1 ** step
9 bias_correction2 = 1 - beta2 ** step
10
11 if weight_decay != 0:
12 grad = ops.TensorAdd()(grad, ops.Mul()(param, weight_decay))
13
14 # Ln.15-Ln.16: Decay the first and second moment running average coefficient
15 exp_avg = ops.TensorAdd()(ops.Mul()(exp_avg, beta1), ops.Mul()(ops.Sub()(1, beta1), grad))
16 exp_avg_sq = ops.TensorAdd()(ops.Mul()(exp_avg_sq, beta2), ops.Mul()(ops.Sub()(1, beta2), ops.Square()(grad)))
17
18 if amsgrad:
19 # Ln.19: Maintains the maximum of all 2nd moment running avg. till now
20 max_exp_avg_sq = max_exp_avg_sqs[i]
21 max_exp_avg_sq = ops.Maximum()(max_exp_avg_sq, exp_avg_sq)
22 # Ln.21: Use the max. for normalizing running avg. of gradient
23 denom = ops.TensorAdd()(ops.Sqrt()(ops.Div()(max_exp_avg_sq, math.sqrt(bias_correction2))), eps)
24 else:
25 denom = ops.TensorAdd()(ops.Sqrt()(ops.Div()(exp_avg_sq, math.sqrt(bias_correction2))), eps)
26
27 step_size = lr / bias_correction1
28
29 param = ops.AssignAdd()(param, ops.Neg()(ops.Mul()(step_size, ops.Div()(exp_avg, denom))))
12.2.5 微调神经网络
关于多目标学习方法的实现，可参考基于PyTorch的实现示例：
1# multi-task (PyTorch)
2# Ln.3-Ln.4: Load pre-trained model.
3model = resnet50(pretrained=True)
4model.eval()
5
6# Ln.7-Ln.18: Find neiborhood.
7orginal_shallow_features = {}
8for i, image in enumerate(original_dataloader):
9 features = model.extract_feature(image)
10 original_shallow_features[i] = features['res1']
11
12target_shallow_features = {}
13for i, image in enumerate(target_dataloader):
14 features = model.extract_feature(image)
15 target_shallow_features[i] = features['res1']
16
17original_subset = nearest_neighbor(orginal_shallow_features, target_shallow_features)
18original_subset_dataloader = make_subset(orginal_dataloader, original_subset)
19
20# Ln.21-Ln.39: Finetune.
21model.train()
22classifier = nn.Linear(2048, C)
23
24optimizer = optim.SGD([
25 {'params': model.parameters()},
26 {'params': classifier.parameters()}], lr=lr)
27criterion = nn.CrossEntropyLoss()
28
29for (origin_image, origin_label), (target_image, target_label) in zip(original_subset_dataloader, target_dataloader):
30 origin_pred = model(orgin_image)
31 origin_loss = criterion(origin_pred, origin_label)
32 target_vector = model.extract_features(target_image)['res5']
33 target_vector = torch.flatten(F.avg_pool2d(target_vector, 7), 1)
34 target_pred = classifier(target_vector)
35 target_loss = criterion(target_pred, target_label)
36 loss = origin_loss + target_loss
37 optimizer.zero_grad()
38 optimizer.backward(loss)
39 optimizer.step()
其基于MindSpore的实现为：
1# multi-task (MindSpore)
2# Ln.3-Ln.4: Load pre-trained model.
3model = resnet50(pretrained=True)
4model.set_train(False)
5
6# Ln.7-Ln.18: Find neiborhood.
7original_shallow_features = {}
8for i, image in enumerate(original_dataloader):
9 features = model.extract_feature(Tensor(image))
10 original_shallow_features[i] = features['res1']
11
12target_shallow_features = {}
13for i, image in enumerate(target_dataloader):
14 features = model.extract_feature(Tensor(image))
15 target_shallow_features[i] = features['res1']
16
17original_subset = nearest_neighbor(original_shallow_features, target_shallow_features)
18original_subset_dataloader = make_subset(original_dataloader, original_subset)
19
20# Ln.21-Ln.39: Finetune.
21model.set_train(True)
22classifier = nn.Dense(2048, C)
23
24optimizer = nn.SGD([
25 {'params': model.trainable_params()},
26 {'params': classifier.trainable_params()}], lr=lr)
27criterion = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
28
29for (origin_image, origin_label), (target_image, target_label) in zip(original_subset_dataloader, target_dataloader):
30 origin_pred = model(Tensor(origin_image))
31 origin_loss = criterion(origin_pred, Tensor(origin_label))
32 target_vector = model.extract_features(Tensor(target_image))['res5']
33 target_vector = ops.Reshape()(ops.ReduceMean()(target_vector, (2, 3)), (-1, 2048))
34 target_pred = classifier(target_vector)
35 target_loss = criterion(target_pred, Tensor(target_label))
36 loss = ops.Add()(origin_loss, target_loss)
37 optimizer.zero_grad()
38 loss.backward()
39 optimizer.step()