数据分析01

数据分析 01

import pandas as pd
from sklearn.preprocessing import OneHotEncoder
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split, GridSearchCV

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score, classification_report

import joblib

## 加载数据

df = pd.read_csv(r'C:\Users\78731\Desktop\Artificial Intelligence Trainer Competition\01_data analysis\credit_with_missing_values.csv')

# # 打印数据的前几行
# print(df.head())
# # 显示数据的基本描述性统计
# print(df.describe())
# # 显示数据集的信息，包括每个字段的类型和空值情况
# print(df.info())

## 数据空值处理  平均值mean()  中值median() 众数mode()

df['income'] = df['income'].fillna(df['income'].mean())
df['limit'] = df['limit'].fillna(df['limit'].mean())
df['age'] = df['age'].fillna(df['age'].median())
df['default_fre'] = df['default_fre'].fillna(df['default_fre'].mode()[0])

df['default'] = df['default'].fillna(df['default'].mode()[0])

df['gender'] = df['gender'].fillna(df['gender'].mode()[0])
df['province'] = df['province'].fillna(df['province'].mode()[0])

# print(df.head())

gender_mapping = {'male':0, 'female':1}
df['gender'] = df['gender'].map(gender_mapping)

# 使用 OneHotEncoder 来对省份进行 one-hot 编码
onehot_encoder = OneHotEncoder(sparse=False, dtype=int)  # 使用整数类型
province_encoded = onehot_encoder.fit_transform(df[['province']])  # 必须是二维数组形式

# 将one-hot编码转换为单列，每行是编码的字符串
df['province'] = [''.join(map(str, row)) for row in province_encoded]


q_low = df['income'].quantile(0.01)
q_high = df['income'].quantile(0.99)
df['income'] = np.where(df['income'] < q_low, q_low, df['income'])
df['income'] = np.where(df['income'] > q_high, q_high, df['income'])

# 处理 limit 字段的异常值
q_low = df['limit'].quantile(0.01)
q_high = df['limit'].quantile(0.99)
df['limit'] = np.where(df['limit'] < q_low, q_low, df['limit'])
df['limit'] = np.where(df['limit'] > q_high, q_high, df['limit'])


# sns.set(style="whitegrid")

# 散点图：Income vs Age，颜色表示是否违约
# plt.figure(figsize=(10, 6))
# sns.scatterplot(x='age', y='income', hue='default', data=df, style='default', palette='coolwarm', alpha=0.6)
# plt.title('Income vs Age by Default Status')
# plt.xlabel('Age')
# plt.ylabel('Income')
# plt.legend(title='Default')
# plt.show()

# # 箱线图：Income 分布按性别
# plt.figure(figsize=(10, 6))
# sns.boxplot(x='gender', y='income', hue='default', data=df, palette='coolwarm')
# plt.title('Income Distribution by Gender and Default Status')
# plt.xlabel('Gender (0: Male, 1: Female)')
# plt.ylabel('Income')
# plt.legend(title='Default')
# plt.show()

# # 直方图：Age 分布按违约情况
# plt.figure(figsize=(10, 6))
# sns.histplot(data=df, x='age', hue='default', element='step', palette='coolwarm', bins=30)
# plt.title('Age Distribution by Default Status')
# plt.xlabel('Age')
# plt.ylabel('Count')
# plt.show()


# 选择特征和目标变量
X = df.drop('default', axis=1)  # 特征: 去除 'default' 列
y = df['default']  # 目标变量: 'default' 列

# 划分训练集和测试集，比例为 80% 训练，20% 测试
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 查看结果
print("训练集样本数: ", X_train.shape[0])
print("测试集样本数: ", X_test.shape[0])


model = RandomForestClassifier(n_estimators=100, random_state=42)

# 设置参数网格
param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [None, 10, 20, 30],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4]
}

# 创建 GridSearchCV 对象
grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=5, scoring='accuracy', verbose=2)
# 执行网格搜索
grid_search.fit(X_train, y_train)

# 找到最佳模型
best_model = grid_search.best_estimator_
# 使用最佳模型对测试集进行预测
y_pred = best_model.predict(X_test)


# 保存模型
joblib.dump(best_model, r'C:\Users\78731\Desktop\Artificial Intelligence Trainer Competition\01_data analysis\final_model.pkl')

# # 训练模型
# model.fit(X_train, y_train)
# # 预测测试集
# y_pred = model.predict(X_test)
# 评估模型
accuracy = accuracy_score(y_test, y_pred)
print("Accuracy:", accuracy)
print(classification_report(y_test, y_pred))

# 查看更新后的数据集
# print(df.head())

# df.to_csv(r'C:\Users\78731\Desktop\Artificial Intelligence Trainer Competition\01_data analysis\credit_new.csv', index=False)