****************************************************** 第十二章 超参数设定和网络训练 ****************************************************** 12.2 训练技巧 ====================== ------------------------------------- 12.2.3 批规范化操作 ------------------------------------- 在PyTorch中，可按如下调用示例使用批规范化操作（Batch Normalization，BN）操作： .. code-block:: python :linenos: # batch normalization (PyTorch) import torch import torch.nn as nn # Ln.6-Ln.8: Apply batch normalization. m = nn.BatchNorm2d(4) inp = torch.randn(1, 4, 3, 3) oup = m(inp) print(m.weight, m.bias) 在MindSpore中的实现为： .. code-block:: python :linenos: # batch normalization (MindSpore) import numpy as np import mindspore.nn as nn import mindspore as ms x = ms.Tensor(np.array([[[[1, 2], [1, 2]], [[3, 4], [3, 4]]]]).astype(np.float32)) # Ln.8-Ln.9: Apply batch normalization. bn = nn.BatchNorm2d(num_features=2, momentum=0.8) output = bn(x) print(output) BN更适用于mini-batch较大的场景，且需对训练数据做好充分的随机打乱从而获得更加接近整体分布的mini-batch统计量。此外，BN操作还需在运行过程中统计每个mini-batch的一阶和二阶统计量，这也限制了其在递归神经网络（recurrent neural networks）和动态神经网络（dynamic neural networks）中的应用。针对BN的不足，层规范化（layer normalization）操作被提出。该操作针对一整层的所有维度的输入计算对应均值与方差，之后，使用类似BN的规范化操作来转换各个维度的输入。层规范化操作基于PyTorch的具体实现细节可参考如下示例： .. code-block:: python :linenos: # implementations of layer normalization (PyTorch) class LayerNorm(nn.Module): def __init__(self, num_channels, eps=1e-05, affine=True): super().__init__() self.num_channels = num_channels self.eps = eps self.affine = affine if self.affine: self.weight = nn.Parameter(torch.ones(num_channels)) self.bias = nn.Parameter(torch.zeros(num_channels)) else: self.weight = None self.bias = None self.reset_parameters() def reset_parameters(self): if self.affine: nn.init.ones_(self.weight) nn.init.zeros_(self.bias) def forward(self, x): N, C, H, W = x.shape assert C == self.num_channels x = x.reshape(N, -1) # Ln.28-Ln.35: Calculate mean and variance and apply batch normalization. mean = x.mean(axis=1, keepdim=True) var = (x ** 2).mean(axis=1, keepdim=True) - mean * mean x = (x - mean) / (var + self.eps).sqrt() x = x.reshape(N, C, H, W) if self.affine: x = self.weight.reshape(1, -1, 1, 1) * x + self.bias.reshape(1, -1, 1, 1) return x 在MindSpore中的实现为： .. code-block:: python :linenos: # implementations of layer normalization (MindSpore) class LayerNorm(Cell): """ Applies Layer Normalization over a mini-batch of inputs. Inputs: - **x** (Tensor) - The shape of `x` is :math:`(x_1, x_2, ..., x_R)`, and `input_shape[begin_norm_axis:]` is equal to `normalized_shape`. Outputs: Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `x`. """ def __init__(self, normalized_shape, begin_norm_axis=-1, begin_params_axis=-1, gamma_init='ones', beta_init='zeros', epsilon=1e-7 ): """Initialize LayerNorm.""" super(LayerNorm, self).__init__() if not isinstance(normalized_shape, (tuple, list)): raise TypeError(f"For '{self.cls_name}', the type of 'normalized_shape' must be tuple[int] or list[int], " f"but got {normalized_shape} and the type is {type(normalized_shape)}.") self.normalized_shape = normalized_shape self.begin_norm_axis = begin_norm_axis self.begin_params_axis = begin_params_axis self.epsilon = epsilon self.gamma = Parameter(initializer( gamma_init, normalized_shape), name="gamma") self.beta = Parameter(initializer( beta_init, normalized_shape), name="beta") # Ln.36-Ln.38: Initialize the LayerNorm funtion. self.layer_norm = P.LayerNorm(begin_norm_axis=self.begin_norm_axis, begin_params_axis=self.begin_params_axis, epsilon=self.epsilon) def construct(self, input_x): # Ln.42: Apply the LayerNorm funtion. y, _, _ = self.layer_norm(input_x, self.gamma.astype(input_x.dtype), self.beta.astype(input_x.dtype)) return y def extend_repr(self): return 'normalized_shape={}, begin_norm_axis={}, begin_params_axis={}, gamma{}, beta={}'.format( self.normalized_shape, self.begin_norm_axis, self.begin_params_axis, self.gamma, self.beta) 不过，层规范化潜在假设了同一规范化操作应作用于整层神经元，在某种程度限制了神经元多样性，这也为后续的组规范化（group normalization）操作的提出埋下了伏笔。该操作对卷积通道进行了分组，既有效解决了mini-batch大小对于BN的影响，也在一定程度缓解了层规范化对于层内神经元多样性的局限。组规范化操作基于PyTorch的具体实现细节可参考如下示例： .. code-block:: python :linenos: # implementations of group normalization (PyTorch) class GroupNorm(nn.Module): def __init__(self, num_groups, num_channels, eps=1e-05, affine=True): super().__init__() assert num_channels % num_groups == 0 self.num_groups = num_groups self.num_channels = num_channels self.eps = eps self.affine = affine if self.affine: self.weight = nn.Parameter(torch.ones(num_channels)) self.bias = nn.Parameter(torch.zeros(num_channels)) else: self.weight = None self.bias = None self.reset_parameters() def reset_parameters(self): if self.affine: nn.init.ones_(self.weight) nn.init.zeros_(self.bias) def forward(self, x): N, C, H, W = x.shape assert C == self.num_channels # Ln.28-Ln.30: Group the inputs and calculate the mean and variance. x = x.reshape(N, self.num_groups, -1) mean = x.mean(axis=2, keepdims=True) var = (x ** 2).mean(axis=2, keepdim=True) - mean * mean # Ln.33-Ln.36: Apply group normalization. x = (x - mean) / (var + self.eps).sqrt() x = x.reshape(N, C, H, W) if self.affine: x = self.weight.reshape(1, -1, 1, 1) * x + self.bias.reshape(1, -1, 1, 1) return x 在MindSpore中的实现为： .. code-block:: python :linenos: # implementations of group normalization (MindSpore) class GroupNorm(Cell): """ Group Normalization over a mini-batch of inputs. Inputs: - **x** (Tensor) - The input feature with shape (N, C, H, W) . Outputs: Tensor, the normalized and scaled offset tensor, has the same shape and data type as the `x`. """ def __init__(self, num_groups, num_channels, eps=1e-05, affine=True, gamma_init='ones', beta_init='zeros'): """Initialize GroupNorm.""" super(GroupNorm, self).__init__() self.num_groups = validator.check_positive_int(num_groups, "num_groups", self.cls_name) self.num_channels = validator.check_positive_int(num_channels, "num_channels", self.cls_name) if num_channels % num_groups != 0: raise ValueError(f"For '{self.cls_name}', the 'num_channels' must be divided by 'num_groups', " f"but got 'num_channels': {num_channels}, 'num_groups': {num_groups}.") self.eps = validator.check_value_type('eps', eps, (float,), type(self).__name__) self.affine = validator.check_bool(affine, arg_name="affine", prim_name=self.cls_name) self.gamma = Parameter(initializer( gamma_init, num_channels), name="gamma", requires_grad=affine) self.beta = Parameter(initializer( beta_init, num_channels), name="beta", requires_grad=affine) self.shape = F.shape self.reshape = F.reshape self.reduce_mean = P.ReduceMean(keep_dims=True) self.square = F.square self.reduce_sum = P.ReduceSum(keep_dims=True) self.sqrt = P.Sqrt() def _cal_output(self, x): """calculate groupnorm output""" batch, channel, height, width = self.shape(x) _channel_check(channel, self.num_channels, self.cls_name) # Ln.42-Ln.45: Group the inputs and calculate the mean, variance and standard deviation. x = self.reshape(x, (batch, self.num_groups, -1)) mean = self.reduce_mean(x, 2) var = self.reduce_sum(self.square(x - mean), 2) / (channel * height * width / self.num_groups) std = self.sqrt(var + self.eps) # Ln.48-Ln.50: Apply group normalization. x = (x - mean) / std x = self.reshape(x, (batch, channel, height, width)) output = x * self.reshape(self.gamma, (-1, 1, 1)) + self.reshape(self.beta, (-1, 1, 1)) return output def construct(self, x): _shape_check(self.shape(x), self.cls_name) _check_dtype(x.dtype, [mstype.float16, mstype.float32], "input", self.cls_name) output = self._cal_output(x) return output def extend_repr(self): return 'num_groups={}, num_channels={}'.format(self.num_groups, self.num_channels) 在规范化操作的研究中，研究者还提出了一种更为极致的规范化操作，即示例规范化（instance normalization，或contrast normalization），它针对每个样本的每个通道单独计算其统计信息。示例规范化操作基于PyTorch的具体实现细节可参考如下示例： .. code-block:: python :linenos: # implementations of instance normalization (PyTorch) class InstanceNorm(nn.Module): def __init__(self, num_channels, eps=1e-05, affine=True): super().__init__() self.num_channels = num_channels self.eps = eps self.affine = affine if self.affine: self.weight = nn.Parameter(torch.ones(num_channels)) self.bias = nn.Parameter(torch.zeros(num_channels)) else: self.weight = None self.bias = None self.reset_parameters() def reset_parameters(self): if self.affine: nn.init.ones_(self.weight) nn.init.zeros_(self.bias) def forward(self, x): N, C, H, W = x.shape assert C == self.num_channels # Ln.26-Ln.28: Calculate the mean and variance for a single sample of a single channel. x = x.reshape(N, C, -1) mean = x.mean(axis=2, keepdim=True) var = (x ** 2).mean(axis=2, keepdim=True) - mean * mean # Ln.31-Ln.34: Apply instance normalization. x = (x - mean) / (var + self.eps).sqrt() x = x.reshape(N, C, H, W) if self.affine: x = self.weight.reshape(1, -1, 1, 1) * x + self.bias.reshape(1, -1, 1, 1) return x 在MindSpore中的实现为： .. code-block:: python :linenos: # implementations of instance normalization (MindSpore) class _InstanceNorm(Cell): """Instance Normalization base class.""" @cell_attr_register def __init__(self, num_features, eps=1e-5, momentum=0.1, affine=True, gamma_init='ones', beta_init='zeros', input_dims='2d'): """Initialize Normalization base class.""" super(_InstanceNorm, self).__init__() validator.check_value_type('num_features', num_features, [int], self.cls_name) validator.check_value_type('eps', eps, [float], self.cls_name) validator.check_value_type('momentum', momentum, [float], self.cls_name) validator.check_value_type('affine', affine, [bool], self.cls_name) args_input = {"gamma_init": gamma_init, "beta_init": beta_init} self.check_types_valid(args_input, 'InstanceNorm2d') if num_features < 1: raise ValueError(f"For '{self.cls_name}', the 'num_features' must be at least 1, but got {num_features}.") if momentum < 0 or momentum > 1: raise ValueError(f"For '{self.cls_name}', the 'momentum' must be a number in range [0, 1], " f"but got {momentum}.") self.num_features = num_features self.eps = eps self.input_dims = input_dims self.moving_mean = Parameter(initializer('zeros', num_features), name="mean", requires_grad=False) self.moving_variance = Parameter(initializer('ones', num_features), name="variance", requires_grad=False) self.gamma = Parameter(initializer( gamma_init, num_features), name="gamma", requires_grad=affine) self.beta = Parameter(initializer( beta_init, num_features), name="beta", requires_grad=affine) self.shape = P.Shape() self.momentum = momentum # Ln.41: Initialize the InstanceNorm funtion. self.instance_bn = P.InstanceNorm(epsilon=self.eps, momentum=self.momentum) def construct(self, x): _shape_check_in(self.shape(x), self.input_dims, self.cls_name) # Ln.46-Ln.50: Apply instance normalization. return self.instance_bn(x, self.gamma, self.beta, self.moving_mean, self.moving_variance)[0] def extend_repr(self): return 'num_features={}, eps={}, momentum={}, gamma={}, beta={}, moving_mean={}, moving_variance={}'.format( self.num_features, self.eps, self.momentum, self.gamma, self.beta, self.moving_mean, self.moving_variance) def check_types_valid(self, args_dict, name): for key, _ in args_dict.items(): val = args_dict[key] if not isinstance(val, (Tensor, numbers.Number, str, Initializer)): raise TypeError(f"For '{self.cls_name}', the type of '{key}' must be in " f"[Tensor, numbers.Number, str, Initializer], but got type {type(val).__name__}.") if isinstance(val, Tensor) and val.dtype != mstype.float32: raise TypeError(f"For '{self.cls_name}', the type of '{key}' must be float32, " f"but got {val.dtype}.") ------------------------------------- 12.2.4 网络模型优化算法选择 ------------------------------------- 12.2.4.1 随机梯度下降法 ------------------------------------- SGD基于PyTorch的实现可参考： .. code-block:: python :linenos: # SGD (PyTorch) for param in params: # Ln.4-Ln.5: Obtain the gradient and update it. d_p = param.grad.data param.add_(d_p, alpha=-lr) 基于MindSpore的实现为： .. code-block:: python :linenos: # SGD (MindSpore) for param in params: # Ln.4-Ln.5: Obtain the gradient and update it. d_p = param.grad.asnumpy() param = ops.AssignAdd(param, Tensor(-lr * d_p)) 12.2.4.2 基于动量的随机梯度下降法 -------------------------------------------- 然而，SGD在优化神经网络时也面临一些挑战。它可能会因为更新步长较大或梯度波动导致的震荡而使收敛速度缓慢，并因此陷入到局部极小值。这种情况下，普通的SGD可能表现出不稳定性和收敛速度慢的问题。为了克服这些问题，动量被引入到优化算法中。基于动量的随机梯度下降法在PyTorch中的具体实现可参考如下示例： .. code-block:: python :linenos: # momentum SGD (PyTorch) for i, param in enumerate(params): d_p = param.grad.data # Ln.6-Ln.11: Obtain the momentum and update it. buf = momentum_buffer_list[i] if buf is None: buf = torch.clone(d_p).detach() momentum_buffer_list[i] = buf else: buf.mul_(momentum).add_(d_p, alpha=1 - dampening) # Ln.14-Ln.15: Update the gradient and parameters. d_p = buf param.add_(d_p, alpha=-lr) 其基于MindSpore的实现为： .. code-block:: python :linenos: # momentum SGD (MindSpore) for i, param in enumerate(params): d_p = param.grad.asnumpy() # Ln.6-Ln.11: Obtain the momentum and update it. buf = momentum_buffer_list[i] if buf is None: buf = Tensor(d_p.copy()) momentum_buffer_list[i] = buf else: buf = ops.Mul()(buf, momentum) + ops.Mul()(d_p, 1 - dampening) # Ln.14-Ln.15: Update the gradient and parameters. d_p = buf.asnumpy() param = ops.AssignAdd()(param, Tensor(-lr * d_p)) 12.2.4.3 Nesterov型动量随机梯度下降法 ---------------------------------------------------- Nesterov动量在计算梯度之前，首先根据之前的动量更新来调整当前位置。这种预测下一个位置的方式有助于更准确地估计当前位置的梯度，从而减少了一些摆动和不必要的更新。在PyTorch中，Nesterov型动量随机下降法可实现为： .. code-block:: python :linenos: # Nesterov momentum SGD (PyTorch) for i, param in enumerate(params): d_p = param.grad.data # Ln.6-Ln.11: Obtain the momentum and update it. buf = momentum_buffer_list[i] if buf is None: buf = torch.clone(d_p).detach() momentum_buffer_list[i] = buf else: buf.mul_(momentum).add_(d_p, alpha=1 - dampening) # Ln.14-Ln.15: Update the gradient by momentum and parameters. d_p = d_p.add(buf, alpha=momentum) param.add_(d_p, alpha=-lr) 其基于MindSpore的实现为： .. code-block:: python :linenos: # Nesterov momentum SGD (MindSpore) for i, param in enumerate(params): d_p = param.grad.asnumpy() # Ln.6-Ln.11: Obtain the momentum and update it. buf = momentum_buffer_list[i] if buf is None: buf = Tensor(d_p.copy()) momentum_buffer_list[i] = buf else: buf = ops.Mul()(buf, momentum) + ops.Mul()(d_p, 1 - dampening) # Ln.14-Ln.15: Update the gradient by momentum and parameters. d_p = ops.TensorAdd()(d_p, ops.Mul()(buf, momentum)).asnumpy() param = ops.AssignAdd()(param, Tensor(-lr * d_p)) 12.2.4.4 Adagrad法 ------------------------------------- Adagrad是一种自适应学习率的优化算法，它针对每个参数应用不同的学习率。在PyTorch中，Adagrad法的实现可参考如下示例： .. code-block:: python :linenos: # Adagrad (PyTorch) for (param, grad, state_sum, step) in zip(params, grads, state_sums, state_steps): if weight_decay != 0: if grad.is_sparse: raise RuntimeError("weight_decay option is not compatible with sparse gradients") grad = grad.add(param, alpha=weight_decay) # Ln.9: Calculate the dynamic learning rate associated with the step. clr = lr / (1 + (step - 1) * lr_decay) # Ln.12-Ln.25: Update the gradient and parameters by the dynamic learning rate. if grad.is_sparse: grad = grad.coalesce() # the update is non-linear so indices must be unique grad_indices = grad._indices() grad_values = grad._values() size = grad.size() state_sum.add_(_make_sparse(grad, grad_indices, grad_values.pow(2))) std = state_sum.sparse_mask(grad) std_values = std._values().sqrt_().add_(eps) param.add_(_make_sparse(grad, grad_indices, grad_values / std_values), alpha=-clr) else: state_sum.addcmul_(grad, grad, value=1) std = state_sum.sqrt().add_(eps) param.addcdiv_(grad, std, value=-clr) 其基于MindSpore的实现为： .. code-block:: python :linenos: # Adagrad (MindSpore) from mindspore import Tensor import mindspore.ops as ops for (param, grad, state_sum, step) in zip(params, grads, state_sums, state_steps): if weight_decay != 0: if grad.is_sparse(): raise RuntimeError("weight_decay option is not compatible with sparse gradients") grad = grad + weight_decay * param # Ln.9: Calculate the dynamic learning rate associated with the step. clr = lr / (1 + (step - 1) * lr_decay) # Ln.12-Ln.25: Update the gradient and parameters by the dynamic learning rate. if grad.is_sparse(): grad = grad.coalesce() # the update is non-linear so indices must be unique grad_indices = grad.indices grad_values = grad.values size = grad.shape state_sum = state_sum + ops.SparseApplyFtrl(grad_indices, grad_values**2) std = ops.SparseApplyFtrl(grad_indices, state_sum).sqrt() + eps param = param - clr * ops.SparseApplyFtrl(grad_indices, grad_values / std) else: state_sum = state_sum + ops.Mul()(grad, grad) std = ops.Sqrt()(state_sum) + eps param = param - clr * (grad / std) 12.2.4.5 Adadelta法 ------------------------------------- Adadelta是一种自适应学习率的优化算法，旨在解决 Adagrad算法中学习率急剧下降的问题。在PyTorch中，Adadelta法的实现可参考如下示例： .. code-block:: python :linenos: # Adadelta (PyTorch) for (param, grad, square_avg, acc_delta) in zip(params, grads, square_avgs, acc_deltas): if weight_decay != 0: grad = grad.add(param, alpha=weight_decay) # Ln.7-Ln.11: Calculate the running average of the gradient squares by rho, and update the gradient and parameters with it. square_avg.mul_(rho).addcmul_(grad, grad, value=1 - rho) std = square_avg.add(eps).sqrt_() delta = acc_delta.add(eps).sqrt_().div_(std).mul_(grad) param.add_(delta, alpha=-lr) acc_delta.mul_(rho).addcmul_(delta, delta, value=1 - rho) 其基于MindSpore的实现为： .. code-block:: python :linenos: # Adadelta (MindSpore) for (param, grad, square_avg, acc_delta) in zip(params, grads, square_avgs, acc_deltas): if weight_decay != 0: grad = ops.TensorAdd()(grad, ops.Mul()(param, weight_decay)) # Ln.7-Ln.11: Calculate the running average of the gradient squares by rho, and update the gradient and parameters with it. square_avg = ops.TensorAdd()(ops.Mul()(square_avg, rho), ops.Mul()(ops.Sub()(1, rho), ops.Square()(grad))) std = ops.Sqrt()(ops.TensorAdd()(square_avg, eps)) delta = ops.Mul()(ops.Div()(ops.Sqrt()(ops.TensorAdd()(acc_delta, eps)), std), grad) param = ops.AssignAdd()(param, ops.Neg()(ops.Mul()(lr, delta))) acc_delta = ops.TensorAdd()(ops.Mul()(acc_delta, rho), ops.Mul()(ops.Sub()(1, rho), ops.Square()(delta))) 12.2.4.6 RMSProp法 ------------------------------------- RMSProp（Root Mean Square Propagation）则在解决Adagrad算法中学习率急剧下降的问题的同时，并改善Adadelta算法在某些情况下学习率过于稳定的问题。在PyTorch中，RMSProp法的实现可参考如下示例： .. code-block:: python :linenos: # RMSProp (PyTorch) for i, param in enumerate(params): grad = grads[i] square_avg = square_avgs[i] if weight_decay != 0: grad = grad.add(param, alpha=weight_decay) square_avg.mul_(alpha).addcmul_(grad, grad, value=1 - alpha) if centered: grad_avg = grad_avgs[i] grad_avg.mul_(alpha).add_(grad, alpha=1 - alpha) avg = square_avg.addcmul(grad_avg, grad_avg, value=-1).sqrt_().add_(eps) else: avg = square_avg.sqrt().add_(eps) # Ln.19-Ln.24: Update parameters by gradient, global learning rate and the momentum `avg`. if momentum > 0: buf = momentum_buffer_list[i] buf.mul_(momentum).addcdiv_(grad, avg) param.add_(buf, alpha=-lr) else: param.addcdiv_(grad, avg, value=-lr) 其基于MindSpore的实现为： .. code-block:: python :linenos: # RMSProp (MindSpore) for i, param in enumerate(params): grad = grads[i] square_avg = square_avgs[i] if weight_decay != 0: grad = ops.TensorAdd()(grad, ops.Mul()(param, weight_decay)) square_avg = ops.TensorAdd()(ops.Mul()(square_avg, alpha), ops.Mul()(ops.Sub()(1, alpha), ops.Square()(grad))) if centered: grad_avg = grad_avgs[i] grad_avg = ops.TensorAdd()(ops.Mul()(grad_avg, alpha), ops.Mul()(ops.Sub()(1, alpha), grad)) avg = ops.Sqrt()(ops.TensorAdd()(square_avg, ops.Neg()(ops.Square()(grad_avg)))).add_(eps) else: avg = ops.Sqrt()(square_avg).add_(eps) # Ln.19-Ln.24: Update parameters by gradient, global learning rate and the momentum `avg`. if momentum > 0: buf = momentum_buffer_list[i] buf = ops.TensorAdd()(ops.Mul()(buf, momentum), ops.Div()(grad, avg)) param = ops.AssignAdd()(param, ops.Neg()(ops.Mul()(lr, buf))) else: param = ops.AssignAdd()(param, ops.Neg()(ops.Mul()(lr, ops.Div()(grad, avg)))) 12.2.4.7 Adam法 ------------------------------------- Adam（Adaptive Moment Estimation）则结合了动量和自适应学习率算法的优点。在PyTorch中，Adam法的实现可参考如下示例： .. code-block:: python :linenos: # Adam (PyTorch) for i, param in enumerate(params): grad = grads[i] exp_avg = exp_avgs[i] exp_avg_sq = exp_avg_sqs[i] step = state_steps[i] bias_correction1 = 1 - beta1 ** step bias_correction2 = 1 - beta2 ** step if weight_decay != 0: grad = grad.add(param, alpha=weight_decay) # Ln.15-Ln.16: Decay the first and second moment running average coefficient exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) if amsgrad: # Ln.19: Maintains the maximum of all 2nd moment running avg. till now torch.maximum(max_exp_avg_sqs[i], exp_avg_sq, out=max_exp_avg_sqs[i]) # Ln.21: Use the max. for normalizing running avg. of gradient denom = (max_exp_avg_sqs[i].sqrt() / math.sqrt(bias_correction2)).add_(eps) else: denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(eps) step_size = lr / bias_correction1 param.addcdiv_(exp_avg, denom, value=-step_size) 其基于MindSpore的实现为： .. code-block:: python :linenos: # Adam (MindSpore) for i, param in enumerate(params): grad = grads[i] exp_avg = exp_avgs[i] exp_avg_sq = exp_avg_sqs[i] step = state_steps[i] bias_correction1 = 1 - beta1 ** step bias_correction2 = 1 - beta2 ** step if weight_decay != 0: grad = ops.TensorAdd()(grad, ops.Mul()(param, weight_decay)) # Ln.15-Ln.16: Decay the first and second moment running average coefficient exp_avg = ops.TensorAdd()(ops.Mul()(exp_avg, beta1), ops.Mul()(ops.Sub()(1, beta1), grad)) exp_avg_sq = ops.TensorAdd()(ops.Mul()(exp_avg_sq, beta2), ops.Mul()(ops.Sub()(1, beta2), ops.Square()(grad))) if amsgrad: # Ln.19: Maintains the maximum of all 2nd moment running avg. till now max_exp_avg_sq = max_exp_avg_sqs[i] max_exp_avg_sq = ops.Maximum()(max_exp_avg_sq, exp_avg_sq) # Ln.21: Use the max. for normalizing running avg. of gradient denom = ops.TensorAdd()(ops.Sqrt()(ops.Div()(max_exp_avg_sq, math.sqrt(bias_correction2))), eps) else: denom = ops.TensorAdd()(ops.Sqrt()(ops.Div()(exp_avg_sq, math.sqrt(bias_correction2))), eps) step_size = lr / bias_correction1 param = ops.AssignAdd()(param, ops.Neg()(ops.Mul()(step_size, ops.Div()(exp_avg, denom)))) ---------------------------------------------- 12.2.5 微调神经网络 ---------------------------------------------- 关于多目标学习方法的实现，可参考基于PyTorch的实现示例： .. code-block:: python :linenos: # multi-task (PyTorch) # Ln.3-Ln.4: Load pre-trained model. model = resnet50(pretrained=True) model.eval() # Ln.7-Ln.18: Find neiborhood. orginal_shallow_features = {} for i, image in enumerate(original_dataloader): features = model.extract_feature(image) original_shallow_features[i] = features['res1'] target_shallow_features = {} for i, image in enumerate(target_dataloader): features = model.extract_feature(image) target_shallow_features[i] = features['res1'] original_subset = nearest_neighbor(orginal_shallow_features, target_shallow_features) original_subset_dataloader = make_subset(orginal_dataloader, original_subset) # Ln.21-Ln.39: Finetune. model.train() classifier = nn.Linear(2048, C) optimizer = optim.SGD([ {'params': model.parameters()}, {'params': classifier.parameters()}], lr=lr) criterion = nn.CrossEntropyLoss() for (origin_image, origin_label), (target_image, target_label) in zip(original_subset_dataloader, target_dataloader): origin_pred = model(orgin_image) origin_loss = criterion(origin_pred, origin_label) target_vector = model.extract_features(target_image)['res5'] target_vector = torch.flatten(F.avg_pool2d(target_vector, 7), 1) target_pred = classifier(target_vector) target_loss = criterion(target_pred, target_label) loss = origin_loss + target_loss optimizer.zero_grad() optimizer.backward(loss) optimizer.step() 其基于MindSpore的实现为： .. code-block:: python :linenos: # multi-task (MindSpore) # Ln.3-Ln.4: Load pre-trained model. model = resnet50(pretrained=True) model.set_train(False) # Ln.7-Ln.18: Find neiborhood. original_shallow_features = {} for i, image in enumerate(original_dataloader): features = model.extract_feature(Tensor(image)) original_shallow_features[i] = features['res1'] target_shallow_features = {} for i, image in enumerate(target_dataloader): features = model.extract_feature(Tensor(image)) target_shallow_features[i] = features['res1'] original_subset = nearest_neighbor(original_shallow_features, target_shallow_features) original_subset_dataloader = make_subset(original_dataloader, original_subset) # Ln.21-Ln.39: Finetune. model.set_train(True) classifier = nn.Dense(2048, C) optimizer = nn.SGD([ {'params': model.trainable_params()}, {'params': classifier.trainable_params()}], lr=lr) criterion = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') for (origin_image, origin_label), (target_image, target_label) in zip(original_subset_dataloader, target_dataloader): origin_pred = model(Tensor(origin_image)) origin_loss = criterion(origin_pred, Tensor(origin_label)) target_vector = model.extract_features(Tensor(target_image))['res5'] target_vector = ops.Reshape()(ops.ReduceMean()(target_vector, (2, 3)), (-1, 2048)) target_pred = classifier(target_vector) target_loss = criterion(target_pred, Tensor(target_label)) loss = ops.Add()(origin_loss, target_loss) optimizer.zero_grad() loss.backward() optimizer.step()