梯度下降：AI是怎么"学习"的？

山顶（损失最大）
           ↗ ↖
         ↗     ↖
       ↗   ★你   ↖          ★：当前位置
     ↗     ↓      ↖         ↓：负梯度方向
   ↗       ↓          ↖
 ↗         ↓              ↖
├───────────┴────────────────→
          山谷（损失最小）

# 梯度下降的伪代码
for epoch in range(num_epochs):
    # 1. 前向传播：用当前参数做预测
    predictions = model(data)

    # 2. 计算损失
    loss = loss_function(predictions, targets)

    # 3. 计算梯度（反向传播）
    gradients = compute_gradients(loss, parameters)

    # 4. 更新参数
    for param, grad in zip(parameters, gradients):
        param = param - learning_rate * grad

Loss
 ↑
 |  ×                        ×
 |    ×                    ×
 |      ×                ×
 |        ×            ×
 |          ×        ×
 |            ×    ×
 |              ×
 └──────────────────────────→ epoch
       学习率太大：在谷底来回蹦

Loss
 ↑
 ×
 | ×
 |  ×
 |   ×
 |    ×
 |     ×
 |      ×
 |       ×
 |        ×
 |         ×  ← 还在慢慢降，训练10000步了......
 └──────────────────────────→ epoch
       学习率太小：能收敛，但太慢了

Loss
 ↑
 ×
 |  ×
 |     ×
 |         ×
 |              ×
 |                    ×
 |                          × ← 平稳收敛
 └──────────────────────────→ epoch
       合适的学习率：快速且稳定下降

学习率
 ↑
 │        ╱───────────── 余弦衰减
 │      ╱
 │    ╱  ← Warmup阶段
 │  ╱
 └─────────────────────→ 训练步数

import numpy as np
import matplotlib.pyplot as plt

def f(x):
    """一个简单的碗形函数：f(x) = x^2"""
    return x ** 2

def grad_f(x):
    """梯度：f'(x) = 2x"""
    return 2 * x

def gradient_descent_1d(start, lr, steps):
    """一维梯度下降"""
    x = start
    history = [x]
    for _ in range(steps):
        x = x - lr * grad_f(x)
        history.append(x)
    return history

# 三种学习率
fig, axes = plt.subplots(1, 3, figsize=(15, 4))

configs = [
    (0.01, 'Too Small (lr=0.01)', 50),
    (0.1,  'Just Right (lr=0.1)', 50),
    (0.9,  'Too Large (lr=0.9)', 50),
]

for ax, (lr, title, steps) in zip(axes, configs):
    history = gradient_descent_1d(start=3.0, lr=lr, steps=steps)

    # 画函数曲线
    x_range = np.linspace(-4, 4, 100)
    ax.plot(x_range, f(x_range), 'b-', alpha=0.3, label='f(x)=x²')

    # 画优化路径
    for i in range(min(len(history)-1, 20)):
        ax.annotate('', xy=(history[i+1], f(history[i+1])),
                    xytext=(history[i], f(history[i])),
                    arrowprops=dict(arrowstyle='->', color='red', lw=1.5))
    ax.plot(history[0], f(history[0]), 'go', markersize=10, label='start')
    ax.plot(history[-1], f(history[-1]), 'r*', markersize=15, label='end')

    ax.set_title(title)
    ax.set_xlabel('x')
    ax.set_ylabel('f(x)')
    ax.legend(fontsize=8)
    ax.set_ylim(-1, 12)

plt.tight_layout()
plt.savefig('learning_rate_comparison.png', dpi=150)
plt.show()

Batch GD的路径：          SGD的路径：
      ×                       ×
       ↘                     ↙ ↘
        ↘                   ↗    ↘
         ↘                ↙       ↗
          ↘              ↗        ↘
           ★            ↙          ★
     稳稳地走直线          醉汉走路，但最终也能到

Loss
 ↑
 |    ×
 |   × ×          ×
 |  ×   ×        × ×
 | ×     ×      ×   ×
 |        ×    ×     ×
 |         × ×        ×
 |          ★           ★ ← 全局最小值
 |    局部最小值 ↗
 └────────────────────────→ w

y方向看：一个山谷
     ×     ×
      ×   ×
       × ×
        ★  ← 鞍点
       × ×
      ×   ×
     ×     ×

x方向看：一个山顶
        ★  ← 同一个鞍点
       × ×
      ×   ×
     ×     ×

无动量：       有动量：
  ↓             ↓
  ↓             ↓
  ↓             ↓←←←←←← 惯性冲过去！
  ★             ★
  停在鞍点      继续往前走

import numpy as np
import matplotlib.pyplot as plt

# 生成模拟数据
np.random.seed(42)
n_samples = 100

# 真实关系：score = 15 * hours + 20 + 噪声
hours = np.random.uniform(0, 10, n_samples)       # 学习时间：0~10小时
noise = np.random.normal(0, 5, n_samples)         # 随机噪声
scores = 15 * hours + 20 + noise                  # 考试分数

print(f"数据示例（前5条）：")
for i in range(5):
    print(f"  学习 {hours[i]:.1f} 小时 → 得分 {scores[i]:.1f}")

# 画散点图
plt.figure(figsize=(8, 5))
plt.scatter(hours, scores, alpha=0.6, label='data')
plt.xlabel('study hours')
plt.ylabel('exam score')
plt.title('study hours vs exam score')
plt.legend()
plt.show()

class LinearRegression:
    """从零手写的线性回归模型"""

    def __init__(self):
        # 随机初始化参数
        self.w = np.random.randn()  # 权重
        self.b = np.random.randn()  # 偏置

    def predict(self, x):
        """前向传播：ŷ = wx + b"""
        return self.w * x + self.b

    def loss(self, x, y):
        """均方误差损失：L = (1/N) * Σ(ŷ - y)²"""
        y_pred = self.predict(x)
        return np.mean((y_pred - y) ** 2)

    def gradients(self, x, y):
        """
        手动计算梯度

        L = (1/N) * Σ(wx + b - y)²

        ∂L/∂w = (2/N) * Σ(wx + b - y) * x
        ∂L/∂b = (2/N) * Σ(wx + b - y)
        """
        y_pred = self.predict(x)
        error = y_pred - y  # 预测值 - 真实值
        n = len(x)

        dw = (2 / n) * np.sum(error * x)  # 损失对w的偏导
        db = (2 / n) * np.sum(error)       # 损失对b的偏导

        return dw, db

def train(model, x, y, lr=0.01, epochs=100, print_every=10):
    """
    梯度下降训练

    Parameters:
        model: 线性回归模型
        x, y: 训练数据
        lr: 学习率
        epochs: 训练轮数
    """
    history = {
        'loss': [],
        'w': [],
        'b': []
    }

    print(f"初始参数: w={model.w:.4f}, b={model.b:.4f}")
    print(f"初始损失: {model.loss(x, y):.4f}")
    print(f"目标参数: w≈15, b≈20")
    print("-" * 50)

    for epoch in range(epochs):
        # 1. 计算梯度
        dw, db = model.gradients(x, y)

        # 2. 更新参数（核心：梯度下降公式）
        model.w = model.w - lr * dw
        model.b = model.b - lr * db

        # 3. 记录
        current_loss = model.loss(x, y)
        history['loss'].append(current_loss)
        history['w'].append(model.w)
        history['b'].append(model.b)

        # 4. 打印进度
        if (epoch + 1) % print_every == 0:
            print(f"Epoch {epoch+1:4d} | Loss: {current_loss:8.4f} | "
                  f"w={model.w:7.4f} | b={model.b:7.4f}")

    print("-" * 50)
    print(f"最终参数: w={model.w:.4f}, b={model.b:.4f}")
    print(f"真实参数: w=15.0000, b=20.0000")
    print(f"最终损失: {model.loss(x, y):.4f}")

    return history

# 创建模型并训练
model = LinearRegression()
history = train(model, hours, scores, lr=0.01, epochs=200, print_every=20)

初始参数: w=0.4967, b=-0.1383
初始损失: 8914.7523
目标参数: w≈15, b≈20
--------------------------------------------------
Epoch   20 | Loss: 156.2847 | w= 13.5124 | b= 15.3782
Epoch   40 | Loss:  33.6215 | w= 14.6432 | b= 18.7361
Epoch   60 | Loss:  26.1893 | w= 14.8954 | b= 19.6824
Epoch   80 | Loss:  25.4817 | w= 14.9627 | b= 19.9345
Epoch  100 | Loss:  25.3982 | w= 14.9807 | b= 20.0012
Epoch  120 | Loss:  25.3889 | w= 14.9854 | b= 20.0190
Epoch  140 | Loss:  25.3879 | w= 14.9866 | b= 20.0237
Epoch  160 | Loss:  25.3878 | w= 14.9870 | b= 20.0250
Epoch  180 | Loss:  25.3878 | w= 14.9871 | b= 20.0253
Epoch  200 | Loss:  25.3878 | w= 14.9871 | b= 20.0254
--------------------------------------------------
最终参数: w=14.9871, b=20.0254
真实参数: w=15.0000, b=20.0000
最终损失: 25.3878

fig, axes = plt.subplots(1, 3, figsize=(15, 4))

# 图1：损失下降曲线
axes[0].plot(history['loss'])
axes[0].set_xlabel('epoch')
axes[0].set_ylabel('loss (MSE)')
axes[0].set_title('training loss')
axes[0].set_yscale('log')  # 对数坐标看得更清楚

# 图2：参数w的学习过程
axes[1].plot(history['w'], label='learned w')
axes[1].axhline(y=15, color='r', linestyle='--', label='true w=15')
axes[1].set_xlabel('epoch')
axes[1].set_ylabel('w')
axes[1].set_title('weight convergence')
axes[1].legend()

# 图3：拟合结果
x_line = np.linspace(0, 10, 100)
y_line = model.predict(x_line)
axes[2].scatter(hours, scores, alpha=0.4, label='data')
axes[2].plot(x_line, y_line, 'r-', linewidth=2, label=f'fit: y={model.w:.1f}x+{model.b:.1f}')
axes[2].set_xlabel('study hours')
axes[2].set_ylabel('exam score')
axes[2].set_title('final fit')
axes[2].legend()

plt.tight_layout()
plt.savefig('training_process.png', dpi=150)
plt.show()

fig, axes = plt.subplots(2, 2, figsize=(12, 10))
learning_rates = [0.001, 0.01, 0.05, 0.1]

for ax, lr in zip(axes.flat, learning_rates):
    model_test = LinearRegression()
    # 固定初始值，公平对比
    model_test.w = 0.5
    model_test.b = -0.1

    losses = []
    for epoch in range(200):
        dw, db = model_test.gradients(hours, scores)
        model_test.w -= lr * dw
        model_test.b -= lr * db
        losses.append(model_test.loss(hours, scores))

    ax.plot(losses)
    ax.set_title(f'lr = {lr}')
    ax.set_xlabel('epoch')
    ax.set_ylabel('loss')
    ax.set_ylim(0, 500)

    # 标注最终loss
    final_loss = losses[-1]
    final_w = model_test.w
    ax.text(0.95, 0.95, f'final loss: {final_loss:.1f}\nw={final_w:.2f}',
            transform=ax.transAxes, ha='right', va='top',
            fontsize=9, bbox=dict(boxstyle='round', facecolor='wheat'))

plt.suptitle('effect of learning rate', fontsize=14)
plt.tight_layout()
plt.savefig('lr_comparison.png', dpi=150)
plt.show()

class MomentumOptimizer:
    """带动量的梯度下降"""

    def __init__(self, lr=0.01, momentum=0.9):
        self.lr = lr
        self.momentum = momentum
        self.v_w = 0  # w的速度
        self.v_b = 0  # b的速度

    def step(self, model, x, y):
        dw, db = model.gradients(x, y)

        # 更新速度：新速度 = 动量 × 旧速度 + 当前梯度
        self.v_w = self.momentum * self.v_w + dw
        self.v_b = self.momentum * self.v_b + db

        # 更新参数：用速度而不是梯度
        model.w -= self.lr * self.v_w
        model.b -= self.lr * self.v_b

class AdamOptimizer:
    """Adam优化器"""

    def __init__(self, lr=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):
        self.lr = lr
        self.beta1 = beta1
        self.beta2 = beta2
        self.epsilon = epsilon
        self.t = 0      # 时间步
        self.m_w = 0     # w的一阶矩（方向）
        self.m_b = 0
        self.v_w = 0     # w的二阶矩（大小）
        self.v_b = 0

    def step(self, model, x, y):
        self.t += 1
        dw, db = model.gradients(x, y)

        # 更新一阶矩（梯度的移动平均）
        self.m_w = self.beta1 * self.m_w + (1 - self.beta1) * dw
        self.m_b = self.beta1 * self.m_b + (1 - self.beta1) * db

        # 更新二阶矩（梯度平方的移动平均）
        self.v_w = self.beta2 * self.v_w + (1 - self.beta2) * dw**2
        self.v_b = self.beta2 * self.v_b + (1 - self.beta2) * db**2

        # 偏差修正（初始阶段的补偿）
        m_w_hat = self.m_w / (1 - self.beta1**self.t)
        m_b_hat = self.m_b / (1 - self.beta1**self.t)
        v_w_hat = self.v_w / (1 - self.beta2**self.t)
        v_b_hat = self.v_b / (1 - self.beta2**self.t)

        # 更新参数
        model.w -= self.lr * m_w_hat / (np.sqrt(v_w_hat) + self.epsilon)
        model.b -= self.lr * m_b_hat / (np.sqrt(v_b_hat) + self.epsilon)

def train_with_optimizer(optimizer_name, x, y, epochs=200):
    """用不同优化器训练，返回loss历史"""
    model = LinearRegression()
    model.w, model.b = 0.5, -0.1  # 统一起点

    if optimizer_name == 'SGD':
        lr = 0.01
    elif optimizer_name == 'Momentum':
        opt = MomentumOptimizer(lr=0.01, momentum=0.9)
    elif optimizer_name == 'Adam':
        opt = AdamOptimizer(lr=0.1)  # Adam的lr可以大一些

    losses = []
    for epoch in range(epochs):
        if optimizer_name == 'SGD':
            dw, db = model.gradients(x, y)
            model.w -= lr * dw
            model.b -= lr * db
        else:
            opt.step(model, x, y)
        losses.append(model.loss(x, y))

    return losses, model

# 对比
plt.figure(figsize=(10, 6))
for name in ['SGD', 'Momentum', 'Adam']:
    losses, final_model = train_with_optimizer(name, hours, scores)
    plt.plot(losses, label=f'{name} (final w={final_model.w:.2f})')

plt.xlabel('epoch')
plt.ylabel('loss')
plt.title('SGD vs Momentum vs Adam')
plt.legend()
plt.yscale('log')
plt.savefig('optimizer_comparison.png', dpi=150)
plt.show()

梯度下降：    w = w - η · ∇L(w)

动量：       v = β·v + ∇L
             w = w - η·v

Adam：       m = β₁·m + (1-β₁)·g        (方向)
             v = β₂·v + (1-β₂)·g²       (大小)
             w = w - η · m̂/(√v̂ + ε)     (自适应更新)

Mini-Batch：  每步用m条数据(32~256)计算梯度