本文目录导读：
我来详细介绍如何使用Scikit-learn绘制验证曲线（Validation Curve）来分析模型参数的影响。
验证曲线的基本概念
验证曲线展示不同参数值下模型的训练集和验证集性能,帮助我们：
- 判断模型是否过拟合或欠拟合
- 选择合适的超参数值
- 分析参数对模型性能的影响
基础示例：SVM的RBF核参数gamma验证曲线
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import validation_curve
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
# 1. 生成数据
X, y = make_classification(
n_samples=1000,
n_features=20,
n_informative=10,
n_redundant=5,
random_state=42
)
# 2. 创建管道（包含标准化和SVM）
pipeline = Pipeline([
('scaler', StandardScaler()),
('svm', SVC(kernel='rbf'))
])
# 3. 定义要测试的参数范围
param_range = np.logspace(-3, 3, 7) # 从0.001到1000，7个值
# 4. 计算验证曲线
train_scores, test_scores = validation_curve(
pipeline, # 估计器
X, y, # 数据
param_name='svm__gamma', # 参数名称（注意管道中的前缀）
param_range=param_range, # 参数范围
cv=5, # 5折交叉验证
scoring='accuracy', # 评估指标
n_jobs=-1 # 使用所有CPU核心
)
# 5. 计算平均分和标准差
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
# 6. 绘制验证曲线
plt.figure(figsize=(10, 6))
plt.semilogx(param_range, train_mean, 'o-', color='blue', label='Training score')
plt.fill_between(param_range,
train_mean - train_std,
train_mean + train_std,
alpha=0.25, color='blue')
plt.semilogx(param_range, test_mean, 's-', color='red', label='Cross-validation score')
plt.fill_between(param_range,
test_mean - test_std,
test_mean + test_std,
alpha=0.25, color='red')
plt.xlabel('Parameter gamma')
plt.ylabel('Score')
plt.legend(loc='best')'Validation Curve with SVM (RBF kernel)')
plt.grid(True, alpha=0.3)
plt.show()
# 7. 打印结果
print("Gamma值验证结果:")
for i, gamma in enumerate(param_range):
print(f"gamma={gamma:.4f}: Train={train_mean[i]:.4f} (+/- {train_std[i]:.4f}), "
f"Test={test_mean[i]:.4f} (+/- {test_std[i]:.4f})")
更完整的示例：随机森林的树数量验证
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
# 1. 使用更复杂的数据集
from sklearn.datasets import load_digits
data = load_digits()
X, y = data.data, data.target
# 2. 创建随机森林模型
rf = RandomForestClassifier(random_state=42)
# 3. 测试不同数量的树
param_range = [10, 50, 100, 200, 500]
# 4. 使用分层K折交叉验证
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
# 5. 计算验证曲线（使用更多评估指标）
from sklearn.metrics import make_scorer, accuracy_score, f1_score
# 同时计算准确率和F1分数
train_scores_acc, test_scores_acc = validation_curve(
rf, X, y,
param_name='n_estimators',
param_range=param_range,
cv=cv,
scoring='accuracy',
n_jobs=-1
)
train_scores_f1, test_scores_f1 = validation_curve(
rf, X, y,
param_name='n_estimators',
param_range=param_range,
cv=cv,
scoring='f1_macro',
n_jobs=-1
)
# 6. 绘制双指标对比图
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
# 准确率曲线
for ax, scores, metric in [(ax1, (train_scores_acc, test_scores_acc), 'Accuracy'),
(ax2, (train_scores_f1, test_scores_f1), 'F1 Score')]:
train_mean = np.mean(scores[0], axis=1)
train_std = np.std(scores[0], axis=1)
test_mean = np.mean(scores[1], axis=1)
test_std = np.std(scores[1], axis=1)
ax.plot(param_range, train_mean, 'o-', color='blue', label='Training')
ax.fill_between(param_range,
train_mean - train_std,
train_mean + train_std,
alpha=0.25, color='blue')
ax.plot(param_range, test_mean, 's-', color='red', label='Cross-validation')
ax.fill_between(param_range,
test_mean - test_std,
test_mean + test_std,
alpha=0.25, color='red')
ax.set_xlabel('Number of trees')
ax.set_ylabel(metric)
ax.set_title(f'Validation Curve ({metric})')
ax.legend(loc='best')
ax.grid(True, alpha=0.3)
ax.set_xscale('log')
plt.tight_layout()
plt.show()
优化计算：使用学习曲线对比
from sklearn.model_selection import learning_curve
# 有时候验证曲线不够直观，可以结合学习曲线
def plot_validation_and_learning_curves(estimator, X, y, param_name, param_range):
"""绘制验证曲线和学习曲线的组合图"""
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
# 1. 验证曲线
train_sizes, train_scores, test_scores = validation_curve(
estimator, X, y,
param_name=param_name,
param_range=param_range,
cv=5,
scoring='accuracy',
n_jobs=-1
)
# 绘制验证曲线
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
ax1.plot(param_range, train_mean, 'o-', color='blue', label='Training score')
ax1.fill_between(param_range, train_mean - train_std,
train_mean + train_std, alpha=0.1, color='blue')
ax1.plot(param_range, test_mean, 's-', color='red', label='Cross-validation score')
ax1.fill_between(param_range, test_mean - test_std,
test_mean + test_std, alpha=0.1, color='red')
ax1.set_xlabel('Parameter value')
ax1.set_ylabel('Score')
ax1.set_title('Validation Curve')
ax1.legend(loc='best')
ax1.grid(True, alpha=0.3)
# 2. 学习曲线（使用最佳参数）
best_idx = np.argmax(test_mean)
estimator.set_params(**{param_name: param_range[best_idx]})
train_sizes_lc, train_scores_lc, test_scores_lc = learning_curve(
estimator, X, y,
train_sizes=np.linspace(0.1, 1.0, 10),
cv=5,
scoring='accuracy',
n_jobs=-1
)
train_mean_lc = np.mean(train_scores_lc, axis=1)
train_std_lc = np.std(train_scores_lc, axis=1)
test_mean_lc = np.mean(test_scores_lc, axis=1)
test_std_lc = np.std(test_scores_lc, axis=1)
ax2.plot(train_sizes_lc, train_mean_lc, 'o-', color='blue', label='Training score')
ax2.fill_between(train_sizes_lc, train_mean_lc - train_std_lc,
train_mean_lc + train_std_lc, alpha=0.1, color='blue')
ax2.plot(train_sizes_lc, test_mean_lc, 's-', color='red', label='Cross-validation score')
ax2.fill_between(train_sizes_lc, test_mean_lc - test_std_lc,
test_mean_lc + test_std_lc, alpha=0.1, color='red')
ax2.set_xlabel('Training examples')
ax2.set_ylabel('Score')
ax2.set_title(f'Learning Curve (best {param_name}={param_range[best_idx]:.4f})')
ax2.legend(loc='best')
ax2.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
return best_idx, param_range[best_idx]
# 使用示例
print("分析SVM的gamma参数...")
best_idx, best_param = plot_validation_and_learning_curves(
pipeline, X, y, 'svm__gamma', param_range
)
print(f"最佳gamma值: {best_param:.4f}")
批量分析多个参数
from sklearn.model_selection import ParameterGrid
def analyze_multiple_params(estimator, X, y, param_grid, scoring='accuracy'):
"""批量分析多个参数的验证曲线"""
results = {}
for param_name, param_range in param_grid.items():
print(f"分析参数: {param_name}")
train_scores, test_scores = validation_curve(
estimator, X, y,
param_name=param_name,
param_range=param_range,
cv=5,
scoring=scoring,
n_jobs=-1
)
results[param_name] = {
'param_range': param_range,
'train_mean': np.mean(train_scores, axis=1),
'train_std': np.std(train_scores, axis=1),
'test_mean': np.mean(test_scores, axis=1),
'test_std': np.std(test_scores, axis=1),
'best_param': param_range[np.argmax(np.mean(test_scores, axis=1))]
}
# 绘制多个参数的验证曲线
n_params = len(results)
fig, axes = plt.subplots(1, n_params, figsize=(5*n_params, 5))
if n_params == 1:
axes = [axes]
for ax, (param_name, result) in zip(axes, results.items()):
ax.plot(result['param_range'], result['train_mean'], 'o-', label='Training')
ax.fill_between(result['param_range'],
result['train_mean'] - result['train_std'],
result['train_mean'] + result['train_std'],
alpha=0.2)
ax.plot(result['param_range'], result['test_mean'], 's-', label='Cross-validation')
ax.fill_between(result['param_range'],
result['test_mean'] - result['test_std'],
result['test_mean'] + result['test_std'],
alpha=0.2)
ax.set_xlabel(param_name)
ax.set_ylabel(scoring)
ax.set_title(f'Validation Curve: {param_name}')
ax.legend()
ax.grid(True, alpha=0.3)
# 标记最佳值
best_val = result['best_param']
ax.axvline(x=best_val, color='green', linestyle='--', alpha=0.5)
ax.annotate(f'Best: {best_val:.4f}',
xy=(best_val, ax.get_ylim()[1]),
xytext=(best_val, ax.get_ylim()[1]*0.9),
arrowprops=dict(arrowstyle='->'))
plt.tight_layout()
plt.show()
return results
# 使用示例
param_grid = {
'svc__C': [0.001, 0.01, 0.1, 1, 10, 100],
'svc__gamma': [0.001, 0.01, 0.1, 1, 10, 100]
}
# 注意：需要调整管道定义
pipeline2 = Pipeline([
('scaler', StandardScaler()),
('svc', SVC(kernel='rbf'))
])
# 由于验证曲线一次只能分析一个参数，我们分别分析
analysis_results = analyze_multiple_params(
pipeline2, X, y,
{'svc__C': [0.001, 0.01, 0.1, 1, 10, 100]}
)
实际应用案例：房价预测
from sklearn.datasets import fetch_california_housing
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, r2_score
# 加载数据
housing = fetch_california_housing()
X, y = housing.data, housing.target
# 创建GBDT模型
gbr = GradientBoostingRegressor(random_state=42)
# 分析学习率参数
param_range = [0.01, 0.05, 0.1, 0.2, 0.5, 1.0]
train_scores, test_scores = validation_curve(
gbr, X, y,
param_name='learning_rate',
param_range=param_range,
cv=5,
scoring='r2',
n_jobs=-1
)
# 绘制详细的结果图
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.semilogx(param_range, train_mean, 'o-', color='blue', label='Training R²')
plt.fill_between(param_range,
train_mean - train_std,
train_mean + train_std,
alpha=0.2, color='blue')
plt.semilogx(param_range, test_mean, 's-', color='red', label='Cross-validation R²')
plt.fill_between(param_range,
test_mean - test_std,
test_mean + test_std,
alpha=0.2, color='red')
plt.xlabel('Learning Rate')
plt.ylabel('R² Score')'Validation Curve: Learning Rate Effect')
plt.legend(loc='best')
plt.grid(True, alpha=0.3)
plt.subplot(1, 2, 2)
# 计算过拟合程度
overfitting = train_mean - test_mean
plt.semilogx(param_range, overfitting, 'd-', color='purple', linewidth=2)
plt.axhline(y=0, color='gray', linestyle='--', alpha=0.5)
plt.xlabel('Learning Rate')
plt.ylabel('Train - Test Score (Overfitting)')'Overfitting Analysis')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print(f"最佳学习率: {param_range[np.argmax(test_mean)]:.3f}")
print(f"最佳R²分数: {np.max(test_mean):.4f}")
注意事项和最佳实践
# 1. 参数范围选择
def suggest_param_range(estimator, X, y, param_name):
"""自动建议参数范围"""
# 这里只是一个示例，实际需要根据模型类型调整
base_params = estimator.get_params()
default_val = base_params.get(param_name, 1.0)
# 建议从默认值的1/10到10倍
if isinstance(default_val, (int, float)):
if default_val > 0:
return np.logspace(
np.log10(default_val) - 1,
np.log10(default_val) + 1,
10
)
return np.linspace(0.1, 10, 10)
# 2. 并行计算优化
from joblib import parallel_backend
def optimized_validation_curve(estimator, X, y, param_name, param_range):
"""使用并行优化的验证曲线"""
with parallel_backend('threading', n_jobs=-1):
train_scores, test_scores = validation_curve(
estimator, X, y,
param_name=param_name,
param_range=param_range,
cv=5,
scoring='accuracy',
n_jobs=-1
)
return train_scores, test_scores
# 3. 结果保存和比较
def save_validation_results(results, filename='validation_results.npy'):
"""保存验证曲线结果"""
np.save(filename, results)
print(f"Results saved to {filename}")
def load_validation_results(filename='validation_results.npy'):
"""加载验证曲线结果"""
return np.load(filename, allow_pickle=True).item()
# 4. 自动选择最佳参数
def find_optimal_param(estimator, X, y, param_name, param_range,
metric='accuracy', verbose=True):
"""自动找到使验证分数最高的参数"""
train_scores, test_scores = validation_curve(
estimator, X, y,
param_name=param_name,
param_range=param_range,
cv=5,
scoring=metric,
n_jobs=-1
)
test_mean = np.mean(test_scores, axis=1)
best_idx = np.argmax(test_mean)
best_param = param_range[best_idx]
if verbose:
print(f"Best {param_name}: {best_param}")
print(f"Best {metric}: {test_mean[best_idx]:.4f}")
print(f"Training {metric}: {np.mean(train_scores, axis=1)[best_idx]:.4f}")
# 检查过拟合
overfit = np.mean(train_scores, axis=1)[best_idx] - test_mean[best_idx]
if overfit > 0.1:
print(f"Warning: Possible overfitting (difference: {overfit:.4f})")
return best_param, test_mean[best_idx]
通过以上案例,你可以：
- 快速理解验证曲线的作用和用法
- 掌握不同模型的参数调优方法
- 学会分析过拟合和欠拟合情况
- 自动化参数选择过程
验证曲线是机器学习模型优化的重要工具,建议在实际项目中经常使用！
