# 过拟合的症状：
# 训练集准确率: 99%
# 测试集准确率: 65%  ← 差距巨大，过拟合了

# 欠拟合的症状：
# 训练集准确率: 65%
# 测试集准确率: 63%  ← 差距很小，但两个都很低，欠拟合了

# 理想状态：
# 训练集准确率: 92%
# 测试集准确率: 89%  ← 差距小，且两个都高

# 错误做法：在训练集上评估
model.fit(X, y)
score = model.score(X, y)  # 这是训练集准确率，没有意义！

from sklearn.model_selection import train_test_split

# 第一次分割：留出测试集（永远不碰，最后评估用）
X_temp, X_test, y_temp, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# 第二次分割：训练集和验证集
X_train, X_val, y_train, y_val = train_test_split(
    X_temp, y_temp, test_size=0.25, random_state=42
    # 0.25 × 0.8 = 0.2，即总数据的20%用于验证
)

print(f"训练集: {len(X_train)} 样本")
print(f"验证集: {len(X_val)} 样本")
print(f"测试集: {len(X_test)} 样本")

from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_iris

iris = load_iris()
X, y = iris.data, iris.target

rf = RandomForestClassifier(n_estimators=100, random_state=42)

# 5折交叉验证：把数据分成5份，轮流用每份做测试
cv_scores = cross_val_score(rf, X, y, cv=5, scoring='accuracy')
print(f"5折交叉验证准确率: {cv_scores}")
print(f"均值: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")

5折交叉验证准确率: [0.9667 0.9333 0.9667 0.9667 1.0000]
均值: 0.9667 ± 0.0219

              预测正例  预测负例
实际正例  |    TP    |    FN   |
实际负例  |    FP    |    TN   |

TP（True Positive）：真正例，预测为正且实际为正
FP（False Positive）：假正例，预测为正但实际为负（误报）
FN（False Negative）：假负例，预测为负但实际为正（漏报）
TN（True Negative）：真负例，预测为负且实际为负

from sklearn.metrics import (
    accuracy_score, precision_score, recall_score,
    f1_score, classification_report
)
import numpy as np

# 示例：不平衡数据集（99%正常，1%欺诈）
y_true = np.array([0]*990 + [1]*10)  # 真实标签
y_pred_lazy = np.zeros(1000)          # 全预测为0（偷懒模型）
y_pred_smart = np.array([0]*985 + [1]*15)  # 有一定识别能力

print("=== 偷懒模型（全预测正常）===")
print(f"准确率: {accuracy_score(y_true, y_pred_lazy):.4f}")  # 99%！但没用
print(f"召回率: {recall_score(y_true, y_pred_lazy):.4f}")    # 0，没发现任何欺诈

print("\n=== 正常模型 ===")
print(f"准确率: {accuracy_score(y_true, y_pred_smart):.4f}")
print(f"精确率: {precision_score(y_true, y_pred_smart):.4f}")
print(f"召回率: {recall_score(y_true, y_pred_smart):.4f}")
print(f"F1分数: {f1_score(y_true, y_pred_smart):.4f}")

from sklearn.metrics import roc_curve, auc
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import load_breast_cancer

data = load_breast_cancer()
X, y = data.data, data.target
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)

# 获取预测概率（不是直接预测类别，而是预测每个类别的概率）
y_prob = rf.predict_proba(X_test)[:, 1]

# 计算ROC曲线
fpr, tpr, thresholds = roc_curve(y_test, y_prob)
roc_auc = auc(fpr, tpr)

print(f"AUC: {roc_auc:.4f}")
# 越接近1越好，0.95以上是优秀

from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier

# 方法1：网格搜索（穷举所有参数组合）
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [5, 10, None],
    'min_samples_split': [2, 5, 10]
}
# 一共 3×3×3 = 27 种组合，每种做5折交叉验证 = 135次训练

grid_search = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid,
    cv=5,
    scoring='f1',
    n_jobs=-1,
    verbose=1
)
grid_search.fit(X_train, y_train)

print(f"最佳参数: {grid_search.best_params_}")
print(f"最佳F1: {grid_search.best_score_:.4f}")

from sklearn.model_selection import learning_curve
import numpy as np
import matplotlib.pyplot as plt

def plot_learning_curve(model, X, y, title):
    train_sizes, train_scores, val_scores = learning_curve(
        model, X, y, cv=5,
        train_sizes=np.linspace(0.1, 1.0, 10),
        scoring='accuracy'
    )

    train_mean = train_scores.mean(axis=1)
    val_mean = val_scores.mean(axis=1)

    plt.figure(figsize=(10, 6))
    plt.plot(train_sizes, train_mean, label='训练集')
    plt.plot(train_sizes, val_mean, label='验证集')
    plt.xlabel('训练样本数')
    plt.ylabel('准确率')
    plt.title(title)
    plt.legend()
    plt.show()

# 如何判断：
# 训练集和验证集分数都低 → 欠拟合（需要更复杂的模型或更多特征）
# 训练集高、验证集低 → 过拟合（需要正则化或更多数据）
# 两者都高且接近 → 理想状态