kaggle常用笔记

1.data process

# data read
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
# data type and number
train.dtypes.value_counts()
# 分别展示类型和数值
train['subcat1'].value_counts().index.values.astype('str')
train['subcat1'].value_counts().values
# number of unique 
train['subcat1'].nunique()
#data describe
#train.describe/describe()/info()/columns/shape
#data append
full =  data_train.append( data_test, ignore_index=True )
# data append and checking for missing data
NAs = pd.concat([train.isnull().sum(), test.isnull().sum()], axis=1, keys=['Train', 'Test'])
NAs[NAs.sum(axis=1) > 0]
#data type change:astype
result = pd.DataFrame({'Id':test['Id'].as_matrix(),'SalePrice':predictions.astype(np.float64) })

2. plot

# 1. bar/hist 柱状图 直方图
import matplotlib.pyplot as plt
fig = plt.figure()
fig.set(alpha=0.2)
# kind=bar/hist
train.MSSubClass.value_counts().plot(kind = 'bar',title='MSSubClass')
#plt.title (u'MSSubClass')
plt.show() 
# 2. scatter 散点图
plt.scatter(train.YearBuilt, train.LotArea)
or
data = pd.concat([train['SalePrice'],train['GrLivArea']], axis=1)
data.plot.scatter(x='GrLivArea', y='SalePrice')
plt.title(u'year and area')
plt.show()
# 3. norm 直方图带偏移线
import seaborn as sns
sns.distplot(train['SalePrice'],fit=norm )
plt.show()
# 偏移度
print("skewness: %f" %train['SalePrice'].skew())
print("Kurtosis: ",train['SalePrice'].kurt())
# 4. pairplot 同时观察多个变量之间关系
sns.set()
cols = ['SalePrice', 'OverallQual', 'GrLivArea', 'GarageCars', 'TotalBsmtSF', 'FullBath', 'YearBuilt']
sns.pairplot(train[cols],size=2.5)
plt.show()
# 5. boxplot 盒图
data = pd.concat([train['SalePrice'], train['OverallQual']], axis=1)
#f, ax = plt.subplots(figsize=(8, 6),)
fig = sns.boxplot(x='OverallQual', y="SalePrice", data=data)
#fig.axis(ymin=0, ymax=800000)
plt.xticks(rotation=90) # x rotation （x轴label旋转）
plt.show()
# 6. heatmap 查看数据中各个变量间相关性
corrmat = train.corr()
f, ax = plt.subplots(figsize=(12, 9))
sns.heatmap(corrmat, vmax=0.8, square=True)
plt.show()
# heatmap 拓展，查看相关度最高的前k个问题，此处k=10
k = 10
#cols = corrmat.nlargest(k, 'SalePrice')['SalePrice'].index
cols = corrmat.nlargest(k, 'SalePrice').index
cm = np.corrcoef(train[cols].values.T)
sns.set(font_scale=1.25)
hm = sns.heatmap(cm,cbar=True,annot=True,square=True, fmt='.2f',annot_kws={'size': 8}, yticklabels=cols.values, xticklabels=cols.values)
plt.show()
# 7. data missing percent 数据缺失率
total = train.isnull().sum().sort_values(ascending=False)
percent = (train.isnull().sum()/train.isnull().count()).sort_values(ascending=False)
missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
train.isnull().sum()
# 8. 两种属性下数据特别统计
pd.crosstab(df.Fireplaces, df.FireplaceQu)
# 9. 两种变量对比作图，此处为对比有无'entralAir'对SalePrice 影响
sns.factorplot('HeatingQC', 'SalePrice', hue = 'CentralAir', estimator = np.mean, data = train, 
             size = 4.5, aspect = 1.4)
plt.show()
# 10. 子图设置
fig, ax = plt.subplots(1, 2, figsize = (10, 4))
sns.boxplot(x = 'Street', y = 'SalePrice', data = df, ax = ax[0])
sns.boxplot(x = 'Alley', y = 'SalePrice', data = df, ax = ax[1])
plt.tight_layout()
###########################################################
plt.figure(figsize = (12,6))   #figure size图片大小设定
fig, ax = plt.subplots(2, 1, figsize = (10, 6)) #多图  

3. function

# 测试模型最佳参数
GridSearchCV