Kaggle实战（三）： 数据集的特征工程

本节主要介绍对于所给的数据，进行特征变换以及构造新的特征

import numpy as np
import pandas as pd
from sklearn.feature_extraction.text import  CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy import sparse
from sklearn.preprocessing import LabelEncoder
from sklearn.cluster import KMeans
from nltk.metrics import distance as distance
from sklearn.model_selection import StratifiedKFold
from MeanEncoder import MeanEncoder

dpath = 'F:/Python_demo/XGBoost/data/'
train = pd.read_json(dpath +"RentListingInquries_train.json")
test = pd.read_json(dpath+"RentListingInquries_test.json")
train.head()

1.将分类标签编码为数字

y_map = {'low': 2, 'medium': 1, 'high': 0}
train['interest_level'] = train['interest_level'].apply(lambda x: y_map[x])  # lambda相当于定义一个表达式，x为变量，y_map[x]为对应的数值
#y_train = train.interest_level.values
y_train = train.interest_level
train = train.drop(['listing_id', 'interest_level'], axis=1)  # 删除列

listing_id = test.listing_id.values
test = test.drop('listing_id', axis=1)  # 删除列

ntrain = train.shape[0]
train_test = pd.concat((train, test), axis=0).reset_index(drop=True)  # 将训练集、测试集拼接起来一同进行特征变换

2.去除噪点、异常值

# 去除杂点
#ulimit = np.percentile(tr