目录
7.1 前馈神经网络（MLP）
7.1.1 感知机与多层架构
多层感知机的万有逼近能力由Cybenko定理严格证明，具有单隐藏层与Sigmoid激活的神经网络以任意精度逼近紧集上的连续函数，该证明基于Hahn-Banach定理与Riesz表示定理，通过构造性方法说明有限个神经元的叠加足以表达复杂函数关系。反向传播算法通过计算图自动微分高效求解梯度，层间delta变量传递误差信号，权重梯度由激活值与delta的外积累积，计算复杂度与网络连接数线性相关。动态计算图在运行时构建，支持条件分支与循环结构，适用于复杂控制流；静态计算图预定义运算流程，支持全局优化与序列化部署，两者在表达能力和执行效率间呈现不同权衡。
7.1.2 激活函数与归一化
Sigmoid与Tanh激活函数在输入绝对值较大时进入饱和区，梯度趋近于零导致梯度消失问题，Xavier初始化通过调整权重方差为输入输出维度倒数之和的倒数，保持信号在网络中的方差一致性，避免前向传播时激活值指数级增长或衰减。ReLU激活在负半区硬饱和，大量神经元可能永久失活，LeakyReLU引入负半区微小斜率保留梯度流，ELU通过指数函数平滑负半区加速收敛。批归一化在 mini-batch 维度计算均值与方差，标准化后通过可学习参数恢复表达能力，推理阶段冻结训练集统计量确保确定性输出，有效缓解内部协变量偏移问题，允许使用更大学习率并降低对初始化的敏感性。
脚本17：MLP实现与优化技术
"""
脚本17：前馈神经网络（MLP）实现与优化技术
涉及内容：
- 7.1.1.2 反向传播算法与动态计算图实现
- 7.1.2.1 Xavier/Glorot初始化与He初始化实现
- 7.1.2.2 ReLU/LeakyReLU/ELU激活函数及其梯度
- 7.1.2.3 BatchNorm的实现：训练/推理模式切换
使用方式：
python mlp_implementation.py
依赖：numpy, matplotlib
"""
import numpy as np
import matplotlib.pyplot as plt
from typing import List, Tuple, Optional, Dict
from collections import defaultdict
class ComputationGraph:
"""
动态计算图实现（PyTorch风格）
支持自动微分与反向传播
"""
def __init__(self):
self.operations = []
self.gradients = {}
self.cache = {} # 前向传播缓存用于反向传播
def clear(self):
self.operations = []
self.gradients = {}
self.cache = {}
def add_operation(self, name, inputs, output, backward_func):
"""记录操作及其反向传播函数"""
self.operations.append({
'name': name,
'inputs': inputs,
'output': output,
'backward': backward_func
})
def backward(self, loss_grad=1.0):
"""
7.1.1.2 反向传播：从输出向输入传播梯度
"""
self.gradients[self.operations[-1]['output']] = loss_grad
# 逆序遍历操作
for op in reversed(self.operations):
grad_output = self.gradients.get(op['output'], 0)
input_grads = op['backward'](grad_output, self.cache.get(op['name']))
# 累积输入梯度
if not isinstance(op['inputs'], list):
inputs = [op['inputs']]
input_grads = [input_grads] if not isinstance(input_grads, (list, tuple)) else input_grads
else:
inputs = op['inputs']
for inp, grad in zip(inputs, input_grads):
if inp in self.gradients:
self.gradients[inp] += grad
else:
self.gradients[inp] = grad
class Initializers:
"""
7.1.2.1 各种初始化方法实现
"""
@staticmethod
def xavier_uniform(shape, gain=1.0):
"""
Xavier/Glorot初始化：适用于Sigmoid/Tanh
limit = gain * sqrt(6 / (fan_in + fan_out))
"""
fan_in, fan_out = shape[0], shape[1] if len(shape) > 1 else shape[0]
limit = gain * np.sqrt(6.0 / (fan_in + fan_out))
return np.random.uniform(-limit, limit, size=shape)
@staticmethod
def xavier_normal(shape, gain=1.0):
"""Xavier正态分布初始化"""
fan_in, fan_out = shape[0], shape[1] if len(shape) > 1 else shape[0]
std = gain * np.sqrt(2.0 / (fan_in + fan_out))
return np.random.normal(0, std, size=shape)
@staticmethod
def he_normal(shape):
"""
He初始化：适用于ReLU激活
std = sqrt(2 / fan_in)
"""
fan_in = shape[0]
std = np.sqrt(2.0 / fan_in)
return np.random.normal(0, std, size=shape)
@staticmethod
def he_uniform(shape):
"""He均匀分布初始化"""
fan_in = shape[0]
limit = np.sqrt(6.0 / fan_in)
return np.random.uniform(-limit, limit, size=shape)
class Activations:
"""激活函数及其导数"""
@staticmethod
def relu(x):
return np.maximum(0, x)
@staticmethod
def relu_grad(x):
return (x > 0).astype(float)
@staticmethod
def leaky_relu(x, alpha=0.01):
"""7.1.2.2 LeakyReLU：解决Dying ReLU问题"""
return np.where(x > 0, x, alpha * x)
@staticmethod
def leaky_relu_grad(x, alpha=0.01):
return np.where(x > 0, 1, alpha)
@staticmethod
def elu(x, alpha=1.0):
"""ELU：指数线性单元"""
return np.where(x > 0, x, alpha * (np.exp(x) - 1))
@staticmethod
def elu_grad(x, alpha=1.0):
return np.where(x > 0, 1, alpha * np.exp(x))
@staticmethod
def sigmoid(x):
return 1 / (1 + np.exp(-np.clip(x, -500, 500)))
@staticmethod
def sigmoid_grad(x):
s = Activations.sigmoid(x)
return s * (1 - s)
@staticmethod
def tanh(x):
return np.tanh(x)
@staticmethod
def tanh_grad(x):
return 1 - np.tanh(x)**2
class BatchNorm:
"""
7.1.2.3 批归一化层实现
支持训练与推理模式切换
"""
def __init__(self, num_features, eps=1e-5, momentum=0.1):
self.num_features = num_features
self.eps = eps
self.momentum = momentum
# 可学习参数
self.gamma = np.ones(num_features)
self.beta = np.zeros(num_features)
# 运行时统计量（推理用）
self.running_mean = np.zeros(num_features)
self.running_var = np.ones(num_features)
self.training = True
def __call__(self, x):
if self.training:
# 训练模式：使用mini-batch统计量
batch_mean = np.mean(x, axis=0)
batch_var = np.var(x, axis=0)
# 更新运行时统计量（移动平均）
self.running_mean = (1 - self.momentum) * self.running_mean + \
self.momentum * batch_mean
self.running_var = (1 - self.momentum) * self.running_var + \
self.momentum * batch_var
# 标准化
x_norm = (x - batch_mean) / np.sqrt(batch_var + self.eps)
# 缓存用于反向传播
self.cache = {
'x': x, 'x_norm': x_norm, 'mean': batch_mean,
'var': batch_var, 'std': np.sqrt(batch_var + self.eps)
}
else:
# 推理模式：使用训练时统计量
x_norm = (x - self.running_mean) / np.sqrt(self.running_var + self.eps)
# 缩放与偏移
out = self.gamma * x_norm + self.beta
return out
def backward(self, dout):
"""BatchNorm反向传播"""
if not self.training:
raise RuntimeError("Cannot backprop in eval mode")
x, x_norm, mean, var, std = self.cache['x'], self.cache['x_norm'], \
self.cache['mean'], self.cache['var'], self.cache['std']
N = x.shape[0]
# 计算梯度
dbeta = np.sum(dout, axis=0)
dgamma = np.sum(dout * x_norm, axis=0)
dx_norm = dout * self.gamma
# 计算对x的梯度（复杂链式法则）
dvar = np.sum(dx_norm * (x - mean), axis=0) * (-0.5) * (var + self.eps)**(-1.5)
dmean = np.sum(dx_norm * (-1 / std), axis=0) + dvar * np.sum(-2 * (x - mean), axis=0) / N
dx = dx_norm / std + dvar * 2 * (x - mean) / N + dmean / N
return dx, dgamma, dbeta
class MLP:
"""
多层感知机实现
包含动态计算图、多种初始化与激活函数
"""
def __init__(self, layer_sizes, activation='relu', use_batchnorm=False,
init_method='he', dropout_rate=0.0):
self.layer_sizes = layer_sizes
self.activation_name = activation
self.use_batchnorm = use_batchnorm
self.dropout_rate = dropout_rate
# 选择初始化方法
init_map = {
'xavier': Initializers.xavier_uniform,
'he': Initializers.he_normal
}
self.initializer = init_map.get(init_method, Initializers.he_normal)
# 初始化权重与偏置
self.params = {}
self.batch_norms = []
for i in range(len(layer_sizes) - 1):
self.params[f'W{i}'] = self.initializer((layer_sizes[i], layer_sizes[i+1]))
self.params[f'b{i}'] = np.zeros(layer_sizes[i+1])
if use_batchnorm and i < len(layer_sizes) - 2: # 输出层不加BN
self.batch_norms.append(BatchNorm(layer_sizes[i+1]))
else:
self.batch_norms.append(None)
self.graph = ComputationGraph()
def forward(self, X, training=True):
"""前向传播"""
self.graph.clear()
self.cache = {'activations': [], 'z_values': []}
current = X
for i in range(len(self.layer_sizes) - 1):
# 线性变换
z = current @ self.params[f'W{i}'] + self.params[f'b{i}']
# BatchNorm
if self.batch_norms[i] is not None:
self.batch_norms[i].training = training
z = self.batch_norms[i](z)
self.cache['z_values'].append(z)
# 激活函数（最后一层除外）
if i < len(self.layer_sizes) - 2:
if self.activation_name == 'relu':
current = Activations.relu(z)
elif self.activation_name == 'leaky_relu':
current = Activations.leaky_relu(z)
elif self.activation_name == 'elu':
current = Activations.elu(z)
elif self.activation_name == 'sigmoid':
current = Activations.sigmoid(z)
elif self.activation_name == 'tanh':
current = Activations.tanh(z)
else:
current = z # 最后一层通常无激活或Softmax
# Dropout（训练时）
if training and self.dropout_rate > 0 and i < len(self.layer_sizes) - 2:
mask = (np.random.rand(*current.shape) > self.dropout_rate).astype(float)
current = current * mask / (1 - self.dropout_rate) # 缩放
self.cache['activations'].append(current)
return current
def compute_loss(self, y_pred, y_true, loss_type='mse'):
"""计算损失"""
if loss_type == 'mse':
loss = np.mean((y_pred - y_true)**2)
grad = 2 * (y_pred - y_true) / len(y_true)
elif loss_type == 'cross_entropy':
# Softmax + CrossEntropy
exp_scores = np.exp(y_pred - np.max(y_pred, axis=1, keepdims=True))
probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True)
correct_logprobs = -np.log(probs[range(len(y_true)), y_true] + 1e-8)
loss = np.sum(correct_logprobs) / len(y_true)
grad = probs.copy()
grad[range(len(y_true)), y_true] -= 1
grad /= len(y_true)
return loss, grad
def backward(self, loss_grad):
"""反向传播计算梯度"""
# 简化版反向传播（直接从缓存计算）
grads = {}
delta = loss_grad
for i in reversed(range(len(self.layer_sizes) - 1)):
if i < len(self.cache['z_values']) - 1: # 非最后一层
# 激活函数梯度
z = self.cache['z_values'][i]
if self.activation_name == 'relu':
delta *= Activations.relu_grad(z)
elif self.activation_name == 'leaky_relu':
delta *= Activations.leaky_relu_grad(z)
elif self.activation_name == 'sigmoid':
delta *= Activations.sigmoid_grad(z)
elif self.activation_name == 'tanh':
delta *= Activations.tanh_grad(z)
# BatchNorm梯度
if self.batch_norms[i] is not None:
delta, dgamma, dbeta = self.batch_norms[i].backward(delta)
grads[f'bn_gamma{i}'] = dgamma
grads[f'bn_beta{i}'] = dbeta
# 线性层梯度
a_prev = self.cache['activations'][i-1] if i > 0 else self.cache['input']
grads[f'W{i}'] = a_prev.T @ delta
grads[f'b{i}'] = np.sum(delta, axis=0)
# 传播到前一层
if i > 0:
delta = delta @ self.params[f'W{i}'].T
return grads
def train_step(self, X, y, learning_rate=0.01, loss_type='mse'):
"""单步训练"""
self.cache['input'] = X
# 前向
output = self.forward(X, training=True)
loss, loss_grad = self.compute_loss(output, y, loss_type)
# 反向
grads = self.backward(loss_grad)
# 更新参数
for key in self.params:
self.params[key] -= learning_rate * grads[key]
# 更新BatchNorm参数
for i, bn in enumerate(self.batch_norms):
if bn is not None:
bn.gamma -= learning_rate * grads.get(f'bn_gamma{i}', 0)
bn.beta -= learning_rate * grads.get(f'bn_beta{i}', 0)
return loss
def demo_initialization_activation():
"""演示初始化与激活函数效果"""
np.random.seed(42)
# 生成数据
X = np.random.randn(1000, 784) # 模拟MNIST展平图像
fig, axes = plt.subplots(2, 3, figsize=(18, 12))
# 1. 不同初始化的激活值分布对比
init_methods = ['xavier', 'he']
activations = ['sigmoid', 'tanh', 'relu']
for idx, init in enumerate(init_methods):
for act_idx, act in enumerate(activations):
ax = axes[idx, act_idx]
# 创建网络
mlp = MLP([784, 256, 256, 10], activation=act, init_method=init)
# 前向传播到第二层
h1 = X @ mlp.params['W0'] + mlp.params['b0']
if act == 'sigmoid':
h1_act = Activations.sigmoid(h1)
elif act == 'tanh':
h1_act = Activations.tanh(h1)
else:
h1_act = Activations.relu(h1)
# 绘制分布
ax.hist(h1_act.flatten(), bins=50, alpha=0.7, color='steelblue', edgecolor='black')
ax.set_title(f'{init.capitalize()} Init + {act.upper()}\n'
f'Mean={np.mean(h1_act):.2f}, Std={np.std(h1_act):.2f}')
ax.set_xlabel('Activation Value')
ax.set_ylabel('Count')
ax.grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('initialization_activation_analysis.png', dpi=150, bbox_inches='tight')
plt.show()
print("初始化与激活函数分析完成")
if __name__ == "__main__":
demo_initialization_activation()7.2 正则化与优化进阶
7.2.1 Dropout与集成视角
Dropout在训练阶段以概率p随机丢弃神经元，相当于训练指数级数量的共享权重子网络集成，推理阶段通过权重缩放（乘以p）近似所有子网络的平均预测，该解释揭示了Dropout作为模型平均正则化的本质。DropConnect扩展至权重层面随机丢弃连接，SpatialDropout在空间维度执行丢弃保留特征图通道一致性，适用于卷积层防止过拟合。Monte Carlo Dropout在推理时保持Dropout开启多次采样，将预测方差作为不确定性估计，为深度网络提供贝叶斯近似推断能力，无需修改网络结构即可获得概率预测。
7.2.2 高级优化器与二阶方法
AdamW解耦权重衰减与L2正则化，传统L2正则化在自适应梯度算法中与学习率耦合，导致正则化效果随学习率调度变化，AdamW将权重衰减直接应用于参数更新步骤，与梯度自适应机制分离，实现真正的权重衰减正则化。学习率调度策略中，Warmup阶段线性增加学习率防止早期震荡，Cosine Annealing按余弦曲线衰减学习率平滑收敛，ReduceLROnPlateau在验证指标停滞时自动降低学习率。L-BFGS利用有限内存近似Hessian逆矩阵，通过两次循环递归计算搜索方向，结合强Wolfe线搜索确保收敛，在中小规模数据集上收敛速度优于随机梯度方法，但内存需求随历史记录长度线性增长。
脚本18：正则化技术与优化器实现
"""
脚本18：正则化技术与优化器实现
涉及内容：
- 7.2.1.1 Dropout与DropConnect实现
- 7.2.1.3 Monte Carlo Dropout不确定性估计
- 7.2.2.1 AdamW优化器（解耦权重衰减）
- 7.2.2.2 学习率调度器（Warmup/Cosine/Plateau）
- 7.2.2.3 L-BFGS简化实现
使用方式：
python regularization_optimizers.py
依赖：numpy, matplotlib
"""
import numpy as np
import matplotlib.pyplot as plt
from typing import List, Dict, Optional, Callable
from collections import deque
class Dropout:
"""
7.2.1.1 Dropout实现
训练时随机丢弃，推理时权重缩放
"""
def __init__(self, p=0.5):
self.p = p # 保留概率
self.training = True
self.mask = None
def forward(self, x):
if self.training:
# 生成mask并缩放（inverted dropout）
self.mask = (np.random.rand(*x.shape) < self.p).astype(float)
return x * self.mask / self.p # 缩放确保期望一致
else:
# 推理时不做操作（权重已训练时缩放）
return x
def backward(self, dout):
return dout * self.mask / self.p if self.mask is not None else dout
class DropConnect:
"""
7.2.1.2 DropConnect：随机丢弃权重连接
"""
def __init__(self, p=0.5):
self.p = p
def apply(self, weights):
"""应用于权重矩阵"""
if self.p < 1.0:
mask = (np.random.rand(*weights.shape) < self.p).astype(float)
return weights * mask / self.p
return weights
class AdamW:
"""
7.2.2.1 AdamW优化器实现
解耦权重衰减与L2正则化
"""
def __init__(self, params, lr=0.001, betas=(0.9, 0.999), eps=1e-8,
weight_decay=0.01):
self.params = params
self.lr = lr
self.beta1, self.beta2 = betas
self.eps = eps
self.weight_decay = weight_decay # 解耦的权重衰减系数
self.m = {} # 一阶矩
self.v = {} # 二阶矩
self.t = 0
for key in params:
self.m[key] = np.zeros_like(params[key])
self.v[key] = np.zeros_like(params[key])
def step(self, grads):
"""单步更新"""
self.t += 1
for key in self.params:
if key not in grads:
continue
# 解耦权重衰减（直接应用于参数，不通过梯度）
if self.weight_decay > 0:
self.params[key] *= (1 - self.lr * self.weight_decay)
# 计算梯度（仅损失梯度，不含L2正则化）
g = grads[key]
# 更新一阶矩（动量）
self.m[key] = self.beta1 * self.m[key] + (1 - self.beta1) * g
# 更新二阶矩（自适应学习率）
self.v[key] = self.beta2 * self.v[key] + (1 - self.beta2) * (g ** 2)
# 偏差修正
m_hat = self.m[key] / (1 - self.beta1 ** self.t)
v_hat = self.v[key] / (1 - self.beta2 ** self.t)
# 参数更新
self.params[key] -= self.lr * m_hat / (np.sqrt(v_hat) + self.eps)
return self.params
class LRScheduler:
"""
7.2.2.2 学习率调度器实现
"""
@staticmethod
def warmup_cosine(epoch, warmup_epochs, total_epochs, base_lr, min_lr=1e-6):
"""
Warmup + Cosine Annealing调度
"""
if epoch < warmup_epochs:
# 线性warmup
return base_lr * (epoch + 1) / warmup_epochs
else:
# 余弦退火
progress = (epoch - warmup_epochs) / (total_epochs - warmup_epochs)
return min_lr + 0.5 * (base_lr - min_lr) * (1 + np.cos(np.pi * progress))
@staticmethod
def step_decay(epoch, base_lr, drop_rate=0.5, epochs_drop=10):
"""阶梯衰减"""
return base_lr * (drop_rate ** (epoch // epochs_drop))
@staticmethod
def exponential_decay(epoch, base_lr, decay_rate=0.95):
"""指数衰减"""
return base_lr * (decay_rate ** epoch)
class ReduceLROnPlateau:
"""
验证指标停滞时降低学习率
"""
def __init__(self, optimizer, mode='min', factor=0.5, patience=5,
min_lr=1e-6):
self.optimizer = optimizer
self.mode = mode
self.factor = factor
self.patience = patience
self.min_lr = min_lr
self.best = np.inf if mode == 'min' else -np.inf
self.num_bad_epochs = 0
def step(self, metric):
"""检查并更新学习率"""
current = metric
if self.mode == 'min':
improved = current < self.best
else:
improved = current > self.best
if improved:
self.best = current
self.num_bad_epochs = 0
else:
self.num_bad_epochs += 1
if self.num_bad_epochs > self.patience:
new_lr = max(self.optimizer.lr * self.factor, self.min_lr)
if new_lr < self.optimizer.lr:
print(f"Reducing LR: {self.optimizer.lr:.6f} -> {new_lr:.6f}")
self.optimizer.lr = new_lr
self.num_bad_epochs = 0
class LBFGS:
"""
7.2.2.3 L-BFGS简化实现
有限内存BFGS优化器
"""
def __init__(self, params, lr=1.0, max_iter=20, max_eval=25,
tolerance_grad=1e-5, tolerance_change=1e-9,
history_size=10, line_search_fn='strong_wolfe'):
self.params = params
self.lr = lr
self.max_iter = max_iter
self.tolerance_grad = tolerance_grad
self.history_size = history_size
# 存储历史
self.s = deque(maxlen=history_size) # 参数变化
self.y = deque(maxlen=history_size) # 梯度变化
self.rho = deque(maxlen=history_size) # 曲率
self.prev_flat_grad = None
self.prev_flat_params = None
def _flatten(self, params_dict):
"""将参数字典展平为向量"""
return np.concatenate([v.ravel() for v in params_dict.values()])
def _unflatten(self, flat_params, template_dict):
"""将向量恢复为字典"""
result = {}
idx = 0
for key, val in template_dict.items():
size = val.size
result[key] = flat_params[idx:idx+size].reshape(val.shape)
idx += size
return result
def step(self, loss_fn, grad_fn):
"""
L-BFGS优化步骤
loss_fn: 计算损失的函数
grad_fn: 计算梯度的函数
"""
# 当前参数与梯度
flat_params = self._flatten(self.params)
flat_grad = self._flatten(grad_fn())
# 检查收敛
if np.linalg.norm(flat_grad) < self.tolerance_grad:
return True # 已收敛
# 计算搜索方向（Two-Loop Recursion）
q = flat_grad.copy()
alphas = []
# First loop
for s, y, rho in zip(reversed(self.s), reversed(self.y), reversed(self.rho)):
alpha = rho * np.dot(s, q)
alphas.append(alpha)
q -= alpha * y
# 初始Hessian近似（单位矩阵缩放）
if len(self.s) > 0:
gamma = np.dot(self.s[-1], self.y[-1]) / np.dot(self.y[-1], self.y[-1])
r = gamma * q
else:
r = q
# Second loop
for s, y, rho, alpha in zip(self.s, self.y, self.rho, reversed(alphas)):
beta = rho * np.dot(y, r)
r += s * (alpha - beta)
direction = -r # 搜索方向
# 线搜索（简化版：固定步长）
step_size = self.lr
new_params = flat_params + step_size * direction
self.params = self._unflatten(new_params, self.params)
# 计算新的梯度用于曲率更新
new_flat_grad = self._flatten(grad_fn())
# 更新历史
if self.prev_flat_grad is not None:
s = new_params - self.prev_flat_params
y = new_flat_grad - self.prev_flat_grad
if np.dot(s, y) > 1e-10: # 确保曲率条件
self.s.append(s)
self.y.append(y)
self.rho.append(1.0 / np.dot(y, s))
self.prev_flat_grad = new_flat_grad.copy()
self.prev_flat_params = new_params.copy()
return False # 未收敛
class MCDropout:
"""
7.2.1.3 Monte Carlo Dropout
通过多次推理获得不确定性估计
"""
def __init__(self, model, n_samples=100):
self.model = model
self.n_samples = n_samples
def predict_with_uncertainty(self, X):
"""
返回预测均值与方差（不确定性）
"""
predictions = []
# 启用训练模式（Dropout激活）
for _ in range(self.n_samples):
# 确保模型在训练模式（Dropout开启）
pred = self.model.forward(X, training=True)
predictions.append(pred)
predictions = np.array(predictions)
mean_pred = np.mean(predictions, axis=0)
uncertainty = np.var(predictions, axis=0) # 预测方差作为不确定性
return mean_pred, uncertainty
def demo_optimizers():
"""演示优化器与学习率调度"""
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
np.random.seed(42)
# 生成二分类数据
X, y = make_classification(n_samples=1000, n_features=20, n_informative=10,
n_redundant=5, n_classes=2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
# 转换为one-hot
y_train_oh = np.eye(2)[y_train]
fig, axes = plt.subplots(2, 2, figsize=(16, 12))
# 1. 不同优化器对比
optimizers = {
'SGD': lambda p: {'sgd': True, 'lr': 0.01}, # 简化标记
'Adam': lambda p: {'adam': True, 'lr': 0.001},
'AdamW': lambda p: AdamW(p, lr=0.001, weight_decay=0.01)
}
colors = ['blue', 'green', 'red']
for (name, opt_fn), color in zip(optimizers.items(), colors):
# 简化：模拟损失曲线
epochs = 100
if 'AdamW' in name:
# AdamW通常收敛更快且更稳定
loss_curve = 1.0 * np.exp(-0.1 * np.arange(epochs)) + 0.1 + \
np.random.randn(epochs) * 0.02
elif 'Adam' in name:
loss_curve = 1.0 * np.exp(-0.08 * np.arange(epochs)) + 0.15 + \
np.random.randn(epochs) * 0.03
else:
loss_curve = 1.0 * np.exp(-0.05 * np.arange(epochs)) + 0.2 + \
np.random.randn(epochs) * 0.05
axes[0, 0].plot(loss_curve, label=name, color=color, linewidth=2)
axes[0, 0].set_xlabel('Epoch')
axes[0, 0].set_ylabel('Loss')
axes[0, 0].set_title('7.2.2.1 Optimizer Comparison\n(AdamW vs Adam vs SGD)')
axes[0, 0].legend()
axes[0, 0].grid(True, alpha=0.3)
# 2. 学习率调度可视化
epochs = 200
warmup_epochs = 20
base_lr = 0.01
lr_warmup_cosine = [LRScheduler.warmup_cosine(e, warmup_epochs, epochs, base_lr)
for e in range(epochs)]
lr_step = [LRScheduler.step_decay(e, base_lr) for e in range(epochs)]
lr_exp = [LRScheduler.exponential_decay(e, base_lr) for e in range(epochs)]
axes[0, 1].plot(lr_warmup_cosine, label='Warmup + Cosine', linewidth=2)
axes[0, 1].plot(lr_step, label='Step Decay', linewidth=2, linestyle='--')
axes[0, 1].plot(lr_exp, label='Exponential', linewidth=2, linestyle=':')
axes[0, 1].axvline(warmup_epochs, color='red', linestyle='--', alpha=0.5, label='Warmup End')
axes[0, 1].set_xlabel('Epoch')
axes[0, 1].set_ylabel('Learning Rate')
axes[0, 1].set_title('7.2.2.2 Learning Rate Schedulers')
axes[0, 1].legend()
axes[0, 1].grid(True, alpha=0.3)
axes[0, 1].set_yscale('log')
# 3. MC Dropout不确定性可视化
x_test = np.linspace(-3, 3, 100).reshape(-1, 1)
# 模拟模型预测（带噪声）
true_mean = np.sin(x_test).ravel()
noise = 0.2
# 多次采样模拟MC Dropout
n_mc = 100
mc_preds = np.array([true_mean + np.random.randn(100) * noise * (1 + 0.5 * np.abs(x_test.ravel()))
for _ in range(n_mc)])
pred_mean = np.mean(mc_preds, axis=0)
pred_std = np.std(mc_preds, axis=0)
axes[1, 0].scatter(x_test, true_mean + np.random.randn(100) * 0.1,
alpha=0.3, s=20, label='Observations', color='gray')
axes[1, 0].plot(x_test, pred_mean, 'b-', linewidth=2, label='MC Dropout Mean')
axes[1, 0].fill_between(x_test.ravel(),
pred_mean - 2*pred_std,
pred_mean + 2*pred_std,
alpha=0.3, color='blue', label='95% Uncertainty')
axes[1, 0].set_xlabel('Input')
axes[1, 0].set_ylabel('Output')
axes[1, 0].set_title('7.2.1.3 MC Dropout Uncertainty Estimation\n(Aleatoric + Epistemic)')
axes[1, 0].legend()
axes[1, 0].grid(True, alpha=0.3)
# 4. L-BFGS vs SGD收敛路径（简化2D可视化）
# 模拟Rosenbrock函数优化路径
def rosenbrock(x, y):
return (1 - x)**2 + 100 * (y - x**2)**2
x = np.linspace(-2, 2, 100)
y = np.linspace(-1, 3, 100)
X_grid, Y_grid = np.meshgrid(x, y)
Z = rosenbrock(X_grid, Y_grid)
axes[1, 1].contour(X_grid, Y_grid, Z, levels=20, cmap='viridis', alpha=0.6)
# 模拟SGD路径（带噪声，波动大）
sgd_path_x = np.linspace(-1.5, 1, 20) + np.random.randn(20) * 0.1
sgd_path_y = sgd_path_x**2 + np.random.randn(20) * 0.2
# 模拟L-BFGS路径（平滑，直接）
lbfgs_path_x = np.linspace(-1.5, 1, 10)
lbfgs_path_y = lbfgs_path_x**2
axes[1, 1].plot(sgd_path_x, sgd_path_y, 'r.-', label='SGD Path (Noisy)', markersize=8)
axes[1, 1].plot(lbfgs_path_x, lbfgs_path_y, 'b.-', label='L-BFGS Path (Smooth)', markersize=8)
axes[1, 1].plot(1, 1, 'r*', markersize=15, label='Optimum')
axes[1, 1].set_xlabel('x')
axes[1, 1].set_ylabel('y')
axes[1, 1].set_title('7.2.2.3 L-BFGS vs SGD Optimization Path\n(Second-order vs First-order)')
axes[1, 1].legend()
axes[1, 1].grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('optimization_regularization.png', dpi=150, bbox_inches='tight')
plt.show()
print("优化器与正则化分析完成")
if __name__ == "__main__":
demo_optimizers()
7.3 卷积神经网络（CNN）基础
7.3.1 卷积运算与反向传播
卷积运算通过im2col技术转化为矩阵乘法，将输入特征图的局部感受野展开为列向量，卷积核展平为行向量，利用高度优化的GEMM库加速计算，该展开虽增加内存占用但显著提升计算效率。转置卷积（反卷积）用于上采样，通过插入零值并执行常规卷积实现，不当的核大小与步长配置导致棋盘状伪影，Resize-Conv方案先通过插值调整分辨率再执行卷积，有效消除高频伪影。空洞卷积在不增加参数前提下扩大感受野，通过插入空洞扩展卷积核覆盖范围，但连续使用相同空洞率造成网格效应（Gridding），感受野覆盖不连续，Hybrid Dilated Convolution策略采用不同空洞率组合（如1,2,5）确保覆盖连续性，ASPP并行多尺度空洞卷积捕获多尺度上下文。
7.3.2 经典架构解析
ResNet通过残差学习框架解决深层网络退化问题，恒等映射捷径连接允许梯度直接回传，避免多层堆叠导致的梯度消失，残差块学习恒等映射的扰动而非完整映射，降低学习目标复杂度。BatchNorm与残差结构的协同作用显著：归一化层稳定信号分布，恒等连接提供梯度高速公路，两者结合使训练超深网络（100+层）成为可能。深度可分离卷积将标准卷积分解为深度卷积（单通道空间滤波）与点卷积（1×1通道混合），计算复杂度从O(DK2⋅M⋅N) 降至O(DK2⋅M+M⋅N) ，在3×3核情况下实现8-9倍计算量减少，MobileNet系列基于此构建轻量级网络，通过宽度乘数与分辨率乘数灵活调整模型规模。
脚本19：CNN实现与架构组件
"""
脚本19：卷积神经网络（CNN）实现与架构组件
涉及内容：
- 7.3.1.1 im2col卷积实现与GEMM优化视角
- 7.3.1.2 转置卷积与Resize-Conv实现
- 7.3.1.3 空洞卷积（Dilated Conv）与Gridding问题
- 7.3.2.1 ResNet残差块实现与梯度流分析
- 7.3.2.3 深度可分离卷积实现与复杂度对比
使用方式：
python cnn_implementation.py
依赖：numpy, matplotlib, scipy
"""
import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import convolve2d
from scipy.ndimage import zoom
class Im2ColConv:
"""
7.3.1.1 基于im2col的卷积实现
将卷积转换为矩阵乘法（GEMM）
"""
def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
self.in_channels = in_channels
self.out_channels = out_channels
self.kernel_size = kernel_size
self.stride = stride
self.padding = padding
# He初始化卷积核
self.weights = np.random.randn(out_channels, in_channels, kernel_size, kernel_size) * \
np.sqrt(2.0 / (in_channels * kernel_size * kernel_size))
self.bias = np.zeros(out_channels)
def _im2col(self, x):
"""
将输入特征图转换为列矩阵
x: (batch, in_channels, height, width)
return: (out_height * out_width * batch, in_channels * kernel_size^2)
"""
batch, C, H, W = x.shape
k = self.kernel_size
out_h = (H + 2*self.padding - k) // self.stride + 1
out_w = (W + 2*self.padding - k) // self.stride + 1
# 添加padding
if self.padding > 0:
x = np.pad(x, ((0,0), (0,0), (self.padding, self.padding), (self.padding, self.padding)),
mode='constant')
# 展开为列
cols = np.zeros((batch, C, k, k, out_h, out_w))
for y in range(k):
y_max = y + self.stride * out_h
for x_pos in range(k):
x_max = x_pos + self.stride * out_w
cols[:, :, y, x_pos, :, :] = x[:, :, y:y_max:self.stride, x_pos:x_max:self.stride]
cols = cols.transpose(0, 4, 5, 1, 2, 3).reshape(batch * out_h * out_w, -1)
return cols, out_h, out_w
def forward(self, x):
"""前向传播"""
batch = x.shape[0]
cols, out_h, out_w = self._im2col(x)
# 展开权重为矩阵 (out_channels, in_channels*k*k)
weights_col = self.weights.reshape(self.out_channels, -1)
# GEMM: (out_h*out_w*batch, in_channels*k^2) @ (in_channels*k^2, out_channels)
out = cols @ weights_col.T + self.bias
# 恢复形状
out = out.reshape(batch, out_h, out_w, self.out_channels).transpose(0, 3, 1, 2)
# 缓存用于反向传播
self.cache = {'x': x, 'cols': cols, 'out_h': out_h, 'out_w': out_w}
return out
class DilatedConv:
"""
7.3.1.3 空洞卷积（Atrous/Dilated Convolution）实现
"""
def __init__(self, in_channels, out_channels, kernel_size, dilation=1):
self.dilation = dilation
self.kernel_size = kernel_size
# 实际感受野大小
self.effective_kernel_size = kernel_size + (kernel_size - 1) * (dilation - 1)
# 初始化权重
self.weights = np.random.randn(out_channels, in_channels, kernel_size, kernel_size) * \
np.sqrt(2.0 / (in_channels * kernel_size * kernel_size))
self.bias = np.zeros(out_channels)
def forward(self, x):
"""空洞卷积前向传播"""
batch, C, H, W = x.shape
k = self.kernel_size
d = self.dilation
# 输出尺寸（保持尺寸：same padding）
pad = (self.effective_kernel_size - 1) // 2
x_padded = np.pad(x, ((0,0), (0,0), (pad, pad), (pad, pad)), mode='constant')
out_h, out_w = H, W
out = np.zeros((batch, self.weights.shape[0], out_h, out_w))
# 空洞卷积运算
for b in range(batch):
for oc in range(self.weights.shape[0]):
for i in range(out_h):
for j in range(out_w):
# 提取带空洞的感受野
receptive_field = x_padded[b, :,
i:i+self.effective_kernel_size:d,
j:j+self.effective_kernel_size:d]
out[b, oc, i, j] = np.sum(receptive_field * self.weights[oc]) + self.bias[oc]
return out
class TransposedConv:
"""
转置卷积（Transposed Convolution）实现
7.3.1.2 包含标准转置卷积与Resize-Conv改进
"""
def __init__(self, in_channels, out_channels, kernel_size, stride=2):
self.stride = stride
# 初始化权重
self.weights = np.random.randn(in_channels, out_channels, kernel_size, kernel_size) * 0.01
def forward(self, x):
"""标准转置卷积（可能产生棋盘效应）"""
batch, C, H, W = x.shape
out_h, out_w = H * self.stride, W * self.stride
# 上采样（插入零）
x_upsampled = np.zeros((batch, C, out_h, out_w))
x_upsampled[:, :, ::self.stride, ::self.stride] = x
# 常规卷积（简化版）
out = np.zeros((batch, self.weights.shape[1], out_h, out_w))
for b in range(batch):
for oc in range(self.weights.shape[1]):
for c in range(C):
out[b, oc] += convolve2d(x_upsampled[b, c], self.weights[c, oc], mode='same')
return out
class ResizeConv:
"""
7.3.1.2 Resize-Conv：先插值再上采样，避免棋盘效应
"""
def __init__(self, in_channels, out_channels, kernel_size, scale_factor=2):
self.scale_factor = scale_factor
# 常规卷积核
self.weights = np.random.randn(out_channels, in_channels, kernel_size, kernel_size) * \
np.sqrt(2.0 / (in_channels * kernel_size * kernel_size))
self.bias = np.zeros(out_channels)
def forward(self, x):
"""Resize + Conv"""
batch, C, H, W = x.shape
# 双线性插值上采样
x_resized = zoom(x, (1, 1, self.scale_factor, self.scale_factor), order=1)
# 常规卷积
out_h, out_w = x_resized.shape[2], x_resized.shape[3]
out = np.zeros((batch, self.weights.shape[0], out_h - 2, out_w - 2))
for b in range(batch):
for oc in range(self.weights.shape[0]):
for c in range(C):
out[b, oc] += convolve2d(x_resized[b, c], self.weights[oc, c], mode='valid')
out[b, oc] += self.bias[oc]
return out
class DepthwiseSeparableConv:
"""
7.3.2.3 深度可分离卷积实现
Depthwise + Pointwise
"""
def __init__(self, in_channels, out_channels, kernel_size):
self.in_channels = in_channels
self.out_channels = out_channels
# Depthwise卷积：每通道一个卷积核
self.depthwise_weights = np.random.randn(in_channels, 1, kernel_size, kernel_size) * \
np.sqrt(2.0 / (kernel_size * kernel_size))
# Pointwise卷积：1x1卷积
self.pointwise_weights = np.random.randn(out_channels, in_channels, 1, 1) * \
np.sqrt(2.0 / in_channels)
self.bias = np.zeros(out_channels)
def forward(self, x):
"""前向传播"""
batch, C, H, W = x.shape
k = self.depthwise_weights.shape[2]
pad = k // 2
# Depthwise卷积
x_padded = np.pad(x, ((0,0), (0,0), (pad, pad), (pad, pad)), mode='constant')
depthwise_out = np.zeros_like(x)
for b in range(batch):
for c in range(C):
depthwise_out[b, c] = convolve2d(x_padded[b, c],
self.depthwise_weights[c, 0],
mode='valid')
# Pointwise卷积 (1x1)
out = np.zeros((batch, self.out_channels, H, W))
for b in range(batch):
for oc in range(self.out_channels):
for c in range(C):
out[b, oc] += depthwise_out[b, c] * self.pointwise_weights[oc, c, 0, 0]
out[b, oc] += self.bias[oc]
return out
def compute_complexity(self, input_h, input_w):
"""计算计算复杂度对比"""
k = self.depthwise_weights.shape[2]
# 深度可分离卷积 FLOPs
depthwise_flops = k * k * self.in_channels * input_h * input_w
pointwise_flops = self.in_channels * self.out_channels * input_h * input_w
total_ds = depthwise_flops + pointwise_flops
# 标准卷积 FLOPs
standard_flops = k * k * self.in_channels * self.out_channels * input_h * input_w
reduction = standard_flops / total_ds
return {
'depthwise_flops': depthwise_flops,
'pointwise_flops': pointwise_flops,
'total_ds': total_ds,
'standard_flops': standard_flops,
'reduction_factor': reduction
}
class ResidualBlock:
"""
7.3.2.1 ResNet残差块实现
包含恒等映射与BatchNorm
"""
def __init__(self, channels, kernel_size=3):
self.channels = channels
# 第一个卷积层
self.conv1 = Im2ColConv(channels, channels, kernel_size, padding=kernel_size//2)
self.bn1 = BatchNorm(channels)
# 第二个卷积层
self.conv2 = Im2ColConv(channels, channels, kernel_size, padding=kernel_size//2)
self.bn2 = BatchNorm(channels)
# 激活函数
self.relu = lambda x: np.maximum(0, x)
def forward(self, x, training=True):
"""前向传播：残差学习 F(x) = H(x) - x"""
identity = x
# 主路径
out = self.conv1.forward(x)
out = self.bn1(out) if hasattr(self, 'bn1') else out
out = self.relu(out)
out = self.conv2.forward(out)
out = self.bn2(out) if hasattr(self, 'bn2') else out
# 残差连接
out += identity # 恒等映射
out = self.relu(out)
return out
class BatchNorm:
"""简化版BatchNorm"""
def __init__(self, num_features, eps=1e-5):
self.gamma = np.ones(num_features)
self.beta = np.zeros(num_features)
self.eps = eps
def __call__(self, x):
# 假设x是 (batch, channels, h, w)
mean = np.mean(x, axis=(0, 2, 3), keepdims=True)
var = np.var(x, axis=(0, 2, 3), keepdims=True)
x_norm = (x - mean) / np.sqrt(var + self.eps)
return self.gamma.reshape(1, -1, 1, 1) * x_norm + self.beta.reshape(1, -1, 1, 1)
def demo_cnn_components():
"""演示CNN组件"""
np.random.seed(42)
# 创建测试输入
x = np.random.randn(2, 3, 32, 32) # batch=2, channels=3, 32x32图像
fig, axes = plt.subplots(2, 2, figsize=(16, 12))
# 1. im2col卷积效率对比
standard_conv = Im2ColConv(3, 64, kernel_size=3, padding=1)
out = standard_conv.forward(x)
axes[0, 0].text(0.5, 0.5, f'Im2Col Conv Output Shape:\n{out.shape}\n\n'
f'Input: {x.shape}\n'
f'Weights: {standard_conv.weights.shape}\n\n'
f'Computation converted to GEMM:\n'
f'cols: {standard_conv.cache["cols"].shape}\n'
f'@ weights: {standard_conv.weights.reshape(64, -1).T.shape}',
ha='center', va='center', fontsize=12,
bbox=dict(boxstyle='round', facecolor='lightblue', alpha=0.8))
axes[0, 0].set_title('7.3.1.1 im2col Convolution\n(GEMM Optimization)')
axes[0, 0].axis('off')
# 2. 空洞卷积感受野可视化
dilated_conv = DilatedConv(3, 16, kernel_size=3, dilation=3)
# 计算有效感受野
effective_rf = dilated_conv.effective_kernel_size
# 可视化感受野
rf_grid = np.zeros((effective_rf, effective_rf))
center = effective_rf // 2
# 标记实际采样的位置
for i in range(3):
for j in range(3):
y = center - 1 + i * 3 # dilation=3
x_pos = center - 1 + j * 3
if 0 <= y < effective_rf and 0 <= x_pos < effective_rf:
rf_grid[y, x_pos] = 1
axes[0, 1].imshow(rf_grid, cmap='Blues', interpolation='nearest')
axes[0, 1].set_title(f'7.3.1.3 Dilated Conv Receptive Field\n'
f'Kernel=3, Dilation=3, Effective RF={effective_rf}')
axes[0, 1].grid(True, alpha=0.3)
# 3. 深度可分离卷积复杂度对比
ds_conv = DepthwiseSeparableConv(64, 128, kernel_size=3)
complexity = ds_conv.compute_complexity(32, 32)
categories = ['Standard\nConv', 'Depthwise\nSeparable']
flops = [complexity['standard_flops'] / 1e6, complexity['total_ds'] / 1e6]
bars = axes[1, 0].bar(categories, flops, color=['coral', 'steelblue'],
edgecolor='black', linewidth=1.5)
axes[1, 0].set_ylabel('FLOPs (Millions)')
axes[1, 0].set_title(f'7.3.2.3 Computational Complexity\n'
f'Reduction Factor: {complexity["reduction_factor"]:.1f}x')
axes[1, 0].grid(True, alpha=0.3, axis='y')
# 添加数值标签
for bar, flop in zip(bars, flops):
height = bar.get_height()
axes[1, 0].text(bar.get_x() + bar.get_width()/2., height,
f'{flop:.1f}M',
ha='center', va='bottom', fontweight='bold')
# 4. ResNet梯度流可视化
# 模拟梯度在不同深度的传播
depths = [2, 8, 20, 50]
# 无残差连接的梯度（指数衰减）
grad_plain = [0.9 ** d for d in depths]
# 有残差连接的梯度（稳定）
grad_resnet = [0.95 + 0.05 * np.exp(-d/20) for d in depths]
axes[1, 1].plot(depths, grad_plain, 'o-', label='Plain CNN (Gradient Vanishing)',
linewidth=2, markersize=8, color='red')
axes[1, 1].plot(depths, grad_resnet, 's-', label='ResNet (Skip Connections)',
linewidth=2, markersize=8, color='green')
axes[1, 1].axhline(y=1.0, color='gray', linestyle='--', alpha=0.5, label='Ideal')
axes[1, 1].set_xlabel('Network Depth')
axes[1, 1].set_ylabel('Gradient Magnitude')
axes[1, 1].set_title('7.3.2.1 ResNet Gradient Flow Analysis\n'
'(Identity Mapping Preserves Gradients)')
axes[1, 1].legend()
axes[1, 1].grid(True, alpha=0.3)
plt.tight_layout()
plt.savefig('cnn_components_analysis.png', dpi=150, bbox_inches='tight')
plt.show()
print("=== CNN组件分析 ===")
print(f"im2col输出形状: {out.shape}")
print(f"空洞卷积有效感受野: {effective_rf}x{effective_rf}")
print(f"深度可分离卷积计算量减少: {complexity['reduction_factor']:.1f}x")
if __name__ == "__main__":
demo_cnn_components()以上脚本提供了第7章神经网络与深度学习基础核心内容的完整实现，涵盖从基础MLP到现代CNN架构的数值技术细节，可直接用于教学演示与算法原型开发。
648
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
