第八章 网络参数初始化
8.1 全零初始化
全零初始化在PyTorch中的调用函数为：
1# zero initialization (PyTorch)
2import torch.nn as nn
3
4def reset_parameters(m) -> None:
5 # Ln.6-Ln.8: Zero initialization weight and bias
6 nn.init.zeros_(m.weight)
7 if m.bias is not None:
8 nn.init.zeros_(m.bias)
在MindSpore中的调用函数为：
1# zero initialization (MindSpore)
2import numpy as np
3import mindspore.nn as nn
4import mindspore as ms
5from mindspore.common.initializer import Zero
6
7input_data = ms.Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))
8
9# Ln.10: Apply zero initialization.
10net = nn.Conv2d(3, 64, 3, weight_init=Zero())
11output = net(input_data)
PyTorch中全零初始化的具体实现为：
1# implementations of zero initialization (PyTorch)
2def zeros_(tensor: Tensor) -> Tensor:
3 """
4 Fills the input Tensor with the scalar value `0`.
5
6 Args:
7 tensor: an n-dimensional `torch.Tensor`
8 """
9 # Ln. 10: Apply zero initialization.
10 return _no_grad_zero_(tensor)
在MindSpore中的实现为：
1# implementations of zero initialization (MindSpore)
2@_register('zeros')
3class Zero(Initializer):
4 """
5 Generates an array with constant value of zero in order to initialize a tensor.
6 """
7 def _initialize(self, arr):
8 # Ln. 9: Apply zero initialization.
9 arr.fill(0)
8.2 随机初始化
在PyTorch中，对参数进行服从高斯分布的参数随机初始化的调用函数为：
1# Gaussian initialization (PyTorch)
2import torch.nn as nn
3
4def reset_parameters(m) -> None:
5 # Ln.6: Apply Gaussian initialization weight.
6 nn.init.normal_(m.weight, mean=0, std=1)
7 if m.bias is not None:
8 nn.init.zeros_(m.bias)
MindSpore中的调用函数为：
1# Gaussian initialization (MindSpore)
2import numpy as np
3import mindspore.nn as nn
4import mindspore as ms
5from mindspore.common.initializer import Normal
6
7input_data = ms.Tensor(np.ones([1, 3, 16, 50], dtype=np.float32))
8
9# Ln.10: Apply Gaussian initialization.
10net = nn.Conv2d(3, 64, 3, weight_init=Normal(sigma=1.0, mean=0.0))
11output = net(input_data)
PyTorch中服从高斯分布的参数随机初始化的具体实现为：
1# implementations of Gaussian initialization (PyTorch)
2def normal_(tensor: Tensor, mean: float = 0., std: float = 1.) -> Tensor:
3 """
4 Fills the input Tensor with values drawn from the normal distribution.
5
6 Args:
7 tensor: an n-dimensional `torch.Tensor`
8 mean: the mean of the normal distribution
9 std: the standard deviation of the normal distribution
10 """
11 # Ln.12-Ln.14: Apply Gaussian initialization.
12 if torch.overrides.has_torch_function_variadic(tensor):
13 return torch.overrides.handle_torch_function(normal_, (tensor,), tensor=tensor, mean=mean, std=std)
14 return _no_grad_normal_(tensor, mean, std)
在MindSpore中的具体实现为：
1# implementations of Gaussian initialization (MindSpore)
2@_register()
3class Normal(Initializer):
4 """
5 Generates an array with values sampled from Normal distribution in order to initialize a tensor.
6
7 Args:
8 sigma (float): The standard deviation of Normal distribution. Default: 0.01.
9 mean (float): The mean of Normal distribution. Default: 0.0.
10 """
11 def __init__(self, sigma=0.01, mean=0.0):
12 super(Normal, self).__init__(sigma=sigma, mean=mean)
13 self.sigma = sigma
14 self.mean = mean
15
16 def _initialize(self, arr):
17 # Ln.18-Ln.19: Apply Gaussian initialization.
18 data = _init_random_normal(self.mean, self.sigma, arr.shape)
19 _assignment(arr, data)
但是，上述做法仍会带来一个问题，即网络输出数据分布的方差会随着输入神经元个数而改变。为解决这一问题，一般会在初始化的同时加上对方差大小的规范化。Xavier初始化就可以根据每一层输入与输出的连接数目来确定权重的初始化范围，以保持信号在前向传播过程中的稳定性，其PyTorch实现如下所示：
1# Xavier initialization (PyTorch)
2def xavier_normal_(tensor: Tensor, gain: float = 1.) -> Tensor:
3 """
4 Fills the input `Tensor` with values according to the method
5 described in `Understanding the difficulty of training deep feedforward
6 neural networks` - Glorot, X. & Bengio, Y. (2010), using a normal
7 distribution.
8
9 Also known as Glorot initialization.
10
11 Args:
12 tensor: an n-dimensional `torch.Tensor`
13 gain: an optional scaling factor
14 """
15 # Ln.16-Ln.17: Calculate the normalized standard deviation.
16 fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
17 std = gain * math.sqrt(2.0 / float(fan_in + fan_out))
18
19 return _no_grad_normal_(tensor, 0., std)
20
21def _calculate_fan_in_and_fan_out(tensor):
22 dimensions = tensor.dim()
23 if dimensions < 2:
24 raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions")
25
26 num_input_fmaps = tensor.size(1)
27 num_output_fmaps = tensor.size(0)
28 receptive_field_size = 1
29 if tensor.dim() > 2:
30 for s in tensor.shape[2:]:
31 receptive_field_size *= s
32 fan_in = num_input_fmaps * receptive_field_size
33 fan_out = num_output_fmaps * receptive_field_size
34
35 return fan_in, fan_out
在MindSpore中的实现为：
1# Xavier initialization (MindSpore)
2@_register('xavier_normal')
3class XavierNormal(Initializer):
4 """
5 Generates an array with values sampled from Xavier normal distribution in order to initialize a tensor.
6
7 Args:
8 gain (float): An optional scaling factor. Default: 1.
9 """
10 def __init__(self, gain=1):
11 super().__init__(gain=gain)
12 self.gain = gain
13
14 def _initialize(self, arr):
15 # Ln.16-Ln.27: Calculate the normalized standard deviation.
16 fan_in, fan_out = _calculate_fan_in_and_fan_out(arr.shape)
17 std = self.gain * math.sqrt(2.0 / float(fan_in + fan_out))
18 data = _init_random_normal(0, std, arr.shape)
19
20 _assignment(arr, data)
不过，细心的读者应能发现Xavier方法仍有不甚完美之处，即该方法并未考虑非线性映射函数对输入s的影响。2015年He等人对此提出改进——将非线性映射造成的影响考虑进参数初始化中，他们提出原本Xavier方法中方差规范化的分母应为sqrt(n/2)而不是sqrt(n)。He方法在PyTorch中的调用方式为kaiming_normal_()函数：
1# He initialization (PyTorch)
2def kaiming_normal_(
3 tensor: Tensor, a: float = 0, mode: str = 'fan_in', nonlinearity: str = 'leaky_relu'
4):
5 """
6 Fills the input `Tensor` with values according to the method
7 described in `Delving deep into rectifiers: Surpassing human-level
8 performance on ImageNet classification` - He, K. et al. (2015), using a
9 normal distribution.
10
11 Also known as He initialization.
12
13 Args:
14 tensor: an n-dimensional `torch.Tensor`
15 a: the negative slope of the rectifier used after this layer (only
16 used with ``'leaky_relu'``)
17 mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'``
18 preserves the magnitude of the variance of the weights in the
19 forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the
20 backwards pass.
21 nonlinearity: the non-linear function (`nn.functional` name),
22 recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default).
23 """
24 if 0 in tensor.shape:
25 warnings.warn("Initializing zero-element tensors is a no-op")
26 return tensor
27 # Ln.28-Ln.32: Apply He normalization according to the `mode` and the `nonlinearity` function.
28 fan = _calculate_correct_fan(tensor, mode)
29 gain = calculate_gain(nonlinearity, a)
30 std = gain / math.sqrt(fan)
31 with torch.no_grad():
32 return tensor.normal_(0, std)
33
34 def calculate_gain(nonlinearity, param=None):
35 """
36 Return the recommended gain value for the given nonlinearity function.
37
38 Args:
39 nonlinearity: the non-linear function (`nn.functional` name)
40 param: optional parameter for the non-linear function
41 """
42 linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d']
43 if nonlinearity in linear_fns or nonlinearity == 'sigmoid':
44 return 1
45 elif nonlinearity == 'tanh':
46 return 5.0 / 3
47 elif nonlinearity == 'relu':
48 return math.sqrt(2.0)
49 elif nonlinearity == 'leaky_relu':
50 if param is None:
51 negative_slope = 0.01
52 elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float):
53 negative_slope = param
54 else:
55 raise ValueError("negative_slope {} not a valid number".format(param))
56 return math.sqrt(2.0 / (1 + negative_slope ** 2))
57 elif nonlinearity == 'selu':
58 return 3.0 / 4 # Value found empirically (https://github.com/pytorch/pytorch/pull/50664)
59 else:
60 raise ValueError("Unsupported nonlinearity {}".format(nonlinearity))
61
62 def _calculate_correct_fan(tensor, mode):
63 mode = mode.lower()
64 valid_modes = ['fan_in', 'fan_out']
65 if mode not in valid_modes:
66 raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes))
67
68 fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
69 return fan_in if mode == 'fan_in' else fan_out
在MindSpore中的实现为：
1# He initialization (MindSpore)
2@_register('he_normal')
3class HeNormal(Initializer):
4 """
5 Generates an array with values sampled from HeKaiming Normal distribution
6 in order to initialize a tensor.
7
8 For details of HeNormal algorithm, please check `<https://arxiv.org/abs/1502.01852>`_.
9
10 Args:
11 negative_slope (int, float): The negative slope of the rectifier used after this layer
12 (only used when `nonlinearity` is 'leaky_relu'). Default: 0.
13 mode (str): Either 'fan_in' or 'fan_out'. Choosing 'fan_in' preserves the magnitude of the
14 variance of the weights in the forward pass. Choosing 'fan_out' preserves the magnitudes
15 in the backwards pass. Default: 'fan_in'.
16 nonlinearity (str): The non-linear function, recommended to use only with 'relu' or 'leaky_relu'.
17 Default: 'leaky_relu'.
18 """
19 def __init__(self, negative_slope=0, mode='fan_in', nonlinearity='leaky_relu'):
20 super(HeNormal, self).__init__(negative_slope=negative_slope, mode=mode, nonlinearity=nonlinearity)
21 self.negative_slope = negative_slope
22 self.mode = mode
23 self.nonlinearity = nonlinearity
24
25 def _initialize(self, arr):
26 # Ln.27-Ln.31: Apply He normalization according to the `mode` and the `nonlinearity` function.
27 fan = _calculate_correct_fan(arr.shape, self.mode)
28 gain = _calculate_gain(self.nonlinearity, self.negative_slope)
29 std = gain / math.sqrt(fan)
30 data = _init_random_normal(0, std, arr.shape)
31 _assignment(arr, data)
以上是参数初始化分布服从高斯分布的情形。 均匀分布也是一种很好的初始化分布，当参数初始化分布服从均匀分布（uniform distribution）时，由于分布性质的不同，对于均匀分布需指定其取值区间，则Xavier初始化方法和He初始化方法在PyTorch中分别修改为：
1# Xavier initialization (uniform distribution, PyTorch)
2def xavier_uniform_(tensor: Tensor, gain: float = 1.) -> Tensor:
3 """
4 Fills the input `Tensor` with values according to the method
5 described in `Understanding the difficulty of training deep feedforward
6 neural networks` - Glorot, X. & Bengio, Y. (2010), using a uniform
7 distribution.
8
9 Also known as Glorot initialization.
10
11 Args:
12 tensor: an n-dimensional `torch.Tensor`
13 gain: an optional scaling factor
14 """
15 # Ln.16-Ln.18: Calculate uniform bounds from standard deviation.
16 fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
17 std = gain * math.sqrt(2.0 / float(fan_in + fan_out))
18 a = math.sqrt(3.0) * std
19
20 return _no_grad_uniform_(tensor, -a, a)
1# He initialization (uniform distribution, PyTorch)
2def kaiming_uniform_(
3 tensor: Tensor, a: float = 0, mode: str = 'fan_in', nonlinearity: str = 'leaky_relu'
4):
5 """
6 Fills the input `Tensor` with values according to the method
7 described in `Delving deep into rectifiers: Surpassing human-level
8 performance on ImageNet classification` - He, K. et al. (2015), using a
9 uniform distribution.
10
11 Also known as He initialization.
12
13 Args:
14 tensor: an n-dimensional `torch.Tensor`
15 a: the negative slope of the rectifier used after this layer (only
16 used with ``'leaky_relu'``)
17 mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'``
18 preserves the magnitude of the variance of the weights in the
19 forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the
20 backwards pass.
21 nonlinearity: the non-linear function (`nn.functional` name),
22 recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default).
23 """
24 if torch.overrides.has_torch_function_variadic(tensor):
25 return torch.overrides.handle_torch_function(
26 kaiming_uniform_,
27 (tensor,),
28 tensor=tensor,
29 a=a,
30 mode=mode,
31 nonlinearity=nonlinearity)
32
33 if 0 in tensor.shape:
34 warnings.warn("Initializing zero-element tensors is a no-op")
35 return tensor
36 # Ln.37-Ln.40: Calculate uniform bounds from standard deviation.
37 fan = _calculate_correct_fan(tensor, mode)
38 gain = calculate_gain(nonlinearity, a)
39 std = gain / math.sqrt(fan)
40 bound = math.sqrt(3.0) * std
41 with torch.no_grad():
42 return tensor.uniform_(-bound, bound)
在MindSpore中的实现为：
1# Xavier initialization (uniform distribution, MindSpore)
2@_register('xavier_uniform')
3class XavierUniform(Initializer):
4 """
5 Generates an array with values sampled from Xavier uniform distribution
6 in order to initialize a tensor.
7
8 For details of XavierUniform algorithm, please check
9 `<http://proceedings.mlr.press/v9/glorot10a.html>`_.
10
11 Args:
12 gain (float): An optional scaling factor. Default: 1.
13 """
14 def __init__(self, gain=1):
15 super(XavierUniform, self).__init__(gain=gain)
16 self.gain = gain
17
18 def _initialize(self, arr):
19 # Ln.20-Ln.21: Calculate uniform bounds from standard deviation.
20 n_in, n_out = _calculate_fan_in_and_fan_out(arr.shape)
21 boundary = self.gain * math.sqrt(6.0 / (n_in + n_out))
22 data = _init_random_uniform(-boundary, boundary, arr.shape)
23 _assignment(arr, data)
1# He initialization (uniform distribution, MindSpore)
2@_register('he_uniform')
3class HeUniform(Initializer):
4 """
5 Generates an array with values sampled from HeKaiming Uniform distribution
6 in order to initialize a tensor.
7
8 For details of HeUniform algorithm, please check
9 `<https://arxiv.org/abs/1502.01852>`_.
10
11 Args:
12 negative_slope (int, float, bool): The negative slope of the rectifier used after this layer
13 (only used when `nonlinearity` is 'leaky_relu'). Default: 0.
14 mode (str): Either 'fan_in' or 'fan_out'. Choosing 'fan_in' preserves the magnitude of the
15 variance of the weights in the forward pass. Choosing 'fan_out' preserves the magnitudes
16 in the backwards pass. Default: 'fan_in'.
17 nonlinearity (str): The non-linear function, recommended to use only with 'relu' or 'leaky_relu'.
18 Default: 'leaky_relu'.
19 """
20 def __init__(self, negative_slope=0, mode='fan_in', nonlinearity='leaky_relu'):
21 super(HeUniform, self).__init__(negative_slope=negative_slope, mode=mode, nonlinearity=nonlinearity)
22 self.negative_slope = negative_slope
23 self.mode = mode
24 self.nonlinearity = nonlinearity
25
26 def _initialize(self, arr):
27 # Ln.28-Ln.31: Calculate uniform bounds from standard deviation.
28 fan = _calculate_correct_fan(arr.shape, self.mode)
29 gain = _calculate_gain(self.nonlinearity, self.negative_slope)
30 std = gain / math.sqrt(fan)
31 boundary = math.sqrt(3.0) * std
32 data = _init_random_uniform(-boundary, boundary, arr.shape)
33 _assignment(arr, data)