pandas

---

一、Series

# Series 一维，带标签数组
# DataFrame 二维，Series容器
import pandas as pd

p1 = pd.Series([1, 2, 31, 12, 3, 4])
print(p1)
print(type(p1))
# 指定索引
p2 = pd.Series([1, 6, 8, 14, 5], index=["a", "b", "c", "d", "e"])
print(p2)
print(p2.dtype)
# 字典创建 顺序会变
temp_dic = {"name": "mrb", "age": 30, "tel": 10086}
p3 = pd.Series(temp_dic)
print(p3)
print(p3.dtype)
print(p2.astype(float))
print(p3["age"])
print(p3[1])
# 前1行
print(p3[:1])
# 二三行
print(p3[[1, 2]])
# 没有这个键就变成nan
print(p3[["age", "name", "tel"]])
# 值大于10
print(p1[p1 > 10])
for i in p3.index:
    print(i)
# 两个数组
print(type(p3.index))
print(list(p3.index))
print(type(p3.values))
# where方法不同

---

二、读取数据dataframe

import numpy as np
import pandas as pd

# DataFrame讲解
df = pd.read_csv("./dogname2.csv")
print(df)
# index和column
# 每一行代表一组数据
t3 = pd.DataFrame(np.arange(12).reshape(3, 4), index=list("abc"), columns=list("WXYZ"))
print(t3)
d1 = {"name": ["mrb", "xz"], "age": [20, 32], "tel": [10086, 10010]}
t1 = pd.DataFrame(d1)
# 列索引变成键 行索引还是数字
print(t1)
# 多个字典构成的列表
d2 = [{"name": "xiaohong", "age": 32, "tel": 10010}, {"name": "xiaogang", "tel": 10000},
      {"name": "xiaowang", "age": 22}]
t2 = pd.DataFrame(d2)
print(t2)
# 缺失就是nan
print(t2.index)
print(t2.columns)
print(t2.values)
print(t2.shape)
print(t2.dtypes)  # 列数据类型
print(t2.ndim)  # 维度
print(t2.head(1))  # 只显示第一行
print(t2.tail(2))  # 只显示后两行
print(t2.info())  # 显示信息
print(t2.describe())  # 统计信息 1/4 1/2 3/4 中位数
# 排降序
df = df.sort_values(by="Count_AnimalName", ascending=False)
print(df)
print(df[:20])  # 前20行
print(df[:20]["Row_Labels"])  # 取某列
print(type(df[:20]["Row_Labels"]))  # Series类型 因为已经仅1列了
print(t3.loc["a", "Z"])  # 行列查找
print(t3.loc["a"])
print(t3.loc["a", :])  # a行同上
print(t3.loc[:, "Y"])  # Y列
print(t3.loc[["a", "c"]])  # ac行
print(t3.loc[["a", "c"], :])  # ac行
print(t3.loc[["a", "b"], ["W", "Z"]])  # 特殊切片冒号全选中ABC=A:C
# iloc 通过位置获取
print(t3.iloc[1])  # 第“1”行
print(t3.iloc[:, [0, 2]])  # 02两列
print(t3.iloc[[0, 2], [1, 2]])
t3.iloc[1:, :2] = 30
print(t3)
t3.iloc[1:, :2] = np.nan
print(t3)
print(df[(60 < df["Count_AnimalName"]) & (df["Count_AnimalName"] < 80)])
# print(df[(60 < df["Count_AnimalName"]) | (df["Count_AnimalName"] < 80)])
# 字符串长度大于
print(df[df["Row_Labels"].str.len() > 11])
# .str.split("/")将切割成列表
# 缺失数据处理
# 将缺失位置画上True
print(pd.isnull(t3))
# print(pd.notnull(t3))
# 选择W列不为nan的行
print(t3[pd.notnull(t3["W"])])
# 删除含有nan行
print(t3.dropna(axis=0))
# print(t3.dropna(axis=0, how="any"))
# 全为nan行删除
print(t3.dropna(axis=0, how="all"))
# 替换
# t3.dropna(axis=0, how="any", inplace=True)
# print(t3)
# 填充数据
print(t2.fillna(100))
# 均值填充
print(t2.fillna(t2.mean()))
# # 填充某列
# t2["age"] = t2["age"].fillna(t2["age"].mean())
# print(t2)
# padas 中nan不参与计算
# 0的处理 t[t==0]=np.nan

---

三、直方图

import pandas as pd
from matplotlib import pyplot as plt
# 取条形图绘制
file_path = "./IMDB-Movie-Data.csv"
df = pd.read_csv(file_path)
print(df.head(1))
print(df.info())
# runtime_data = df["Runtime (Minutes)"].values
runtime_data = df["Rating"].values
max_runtime = runtime_data.max()
min_runtime = runtime_data.min()
print(max_runtime-min_runtime)
# 组数
# num_bin = (max_runtime - min_runtime)//5
num_bin = (max_runtime - min_runtime)//0.5
plt.figure(figsize=[20, 8], dpi=80)
# plt.xticks(range(min_runtime, max_runtime+5, 5))
_x = [min_runtime]
i = min_runtime
while i <= max_runtime+0.5:
    i = i + 0.5
    _x.append(i)
# plt.hist(runtime_data, num_bin)
num_bin = num_bin.astype(int)
print(num_bin)
plt.hist(runtime_data, num_bin)
plt.xticks(_x)
plt.show()

---

四、统计方法（字符离散）

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
# 取条形图绘制
file_path = "./IMDB-Movie-Data.csv"
df = pd.read_csv(file_path)
print(df.head(1))
print(df.info())
# runtime_data = df["Runtime (Minutes)"].values
runtime_data = df["Rating"].values
max_runtime = runtime_data.max()
min_runtime = runtime_data.min()
print(max_runtime-min_runtime)
# 组数
# num_bin = (max_runtime - min_runtime)//5
num_bin = (max_runtime - min_runtime)//0.5
plt.figure(figsize=[20, 8], dpi=80)
# plt.xticks(range(min_runtime, max_runtime+5, 5))
_x = [min_runtime]
i = min_runtime
while i <= max_runtime+0.5:
    i = i + 0.5
    _x.append(i)
# plt.hist(runtime_data, num_bin)
num_bin = num_bin.astype(int)
print(num_bin)
plt.hist(runtime_data, num_bin)
plt.xticks(_x)
plt.show()
# 获取平均分
print(df["Rating"].mean())
# 导演人数 tolist数组转化为列表 set集合去重
print(len(set(df["Director"].tolist())))
# unique去重列表
print(len(df["Director"].unique()))
# 提取演员
temp_actors_list = df["Actors"].str.split(",").tolist()
actors_list = [i for j in temp_actors_list for i in j]
# 一般不用numpy处理字符
# actors_list = np.array(temp_actors_list).flatten().tolist()
actors_num = len(set(actors_list))
print(actors_num)
# 统计分类 一个电影可能有多个分类
# 构造全0数组 列名为分类 如果某一条数据中分类出现过 0变1
# 统计分类列表
temp_list = df["Genre"].str.split(",").tolist()
genre_list = list(set(i for j in temp_list for i in j))
zeros_df = pd.DataFrame(np.zeros((df.shape[0],len(genre_list))),columns=genre_list)
# print(zeros_df)
for i in range(df.shape[0]):
    zeros_df.loc[i, temp_list[i]] = 1
print(zeros_df)
# 求和
genre_count = zeros_df.sum(axis=0)
genre_count = genre_count.sort_values()
print(genre_count)

---

五、统计方法 分组与聚合

# join合并行索引相同 调用join方法会加进来 空闲nan
import numpy as np
import pandas as pd

df1 = pd.DataFrame(np.ones((2, 4)), index=["A", "B"], columns=list("abcd"))
df2 = pd.DataFrame(np.zeros((3, 3)), index=["A", "B", "C"], columns=list("xyz"))
print(df1)
print(df2)
print(df1.join(df2))
print(df2.join(df1))
df3 = pd.DataFrame(np.arange(9).reshape((3, 3)), columns=list("fax"))
print(df3)
# on找相同的字段
print(df1.merge(df3, on="a", how="inner"))
print(df1.merge(df3, on="a", how="outer"))
print(df1.merge(df3, on="a", how="left"))
print(df1.merge(df3, on="a", how="right"))
# 分组与聚合

# 分组与聚合
import numpy as np
import pandas as pd

file_path = "./starbucks_store_worldwide.csv"
df = pd.read_csv(file_path)
print(df.head(1))
print(df.info())
grouped = df.groupby(by="Country")
print(grouped)
for i, j in grouped:
    print(i)
    print("-" * 100)
    print(j, type(j))
    print("*" * 100)
print(df[df["Country"] == "US"])
# 调用聚合方法
country_count = grouped["Brand"].count()
print(grouped.count())
print(country_count["US"])
print(country_count["CN"])
# 统计中国各个省份
china_data = df[df["Country"] == "CN"]
grouped_china = china_data.groupby(by = "State/Province").count()["Brand"]
# grouped返回DataFrameGroupBy可迭代 每个元素是个元组
print(grouped_china)
# 多条件分组 结果多索引
grouped_mul = df["Brand"].groupby(by=[df["Country"], df["State/Province"]]).count()
print(grouped_mul)
# 双方框变Dataframe 而不是Series
grouped_mul1 = df[["Brand"]].groupby(by=[df["Country"], df["State/Province"]]).count()
grouped_mul2 = df.groupby(by=[df["Country"], df["State/Province"]])[["Brand"]].count()
grouped_mul3 = df.groupby(by=[df["Country"], df["State/Province"]]).count()[["Brand"]]
print(grouped_mul1)
# 复合索引
print(grouped_mul1.index)
# 改变索引
df1 = pd.DataFrame(np.ones((2, 4)), index=["A", "B"], columns=list("abcd"))
print(df1)
df1.index = ["a", "b"]
print(df1)
# 有的能取到 没有取不到
df1_reindex = df1.reindex(["a", "f"])
print(df1_reindex)
# a这一列设为索引
df1_set = df1.set_index("a")
print(df1_set)
# 作为索引这一列还保存
df1_set_drop = df1.set_index("a", drop=False)
print(df1_set_drop)
# 降重 索引可以重复
print(df1["a"].unique())
print(df1.set_index("b").index.unique())
# 双索引
df1_double = df1.set_index(["c", "b"], drop=False)
print(df1_double)
print(df1_double.index)
a = pd.DataFrame({'a':range(7),'b':range(7,0,-1),'c':['one','one','one','two','two','two','two'],'d':list("hjklmno")})
print(a)
b = a.set_index(["c", "d"])
print(b)
c = b["a"]
print(c["one"]["j"])
print(c["one"])
d = a.set_index(["d", "c"])["a"]
# 可以交换d.swaplevel() tip.必须先取外层索引
print(d)
print(b.loc["one"].loc["h"])

---

六、统计方法 分组绘制条形图

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from matplotlib import font_manager

my_font = font_manager.FontProperties(fname="/usr/share/fonts/truetype/arphic/uming.ttc")

file_path = "./starbucks_store_worldwide.csv"
df = pd.read_csv(file_path)
# # 绘制店铺总数前十国家 统计国家栏目次数
# data1 = df.groupby(by="Country").count()["Brand"].sort_values(ascending=False)[:10]
# _x = data1.index
# _y = data1.values
# plt.figure(figsize=(20, 8), dpi=80)
# plt.bar(range(len(_x)), _y)
# plt.xticks(range(len(_x)), _x)
# plt.show()
# 绘制中国各个城市
df = df[df["Country"] == "CN"]
data2 = df.groupby(by="City").count()["Brand"].sort_values(ascending=False)[:25]
_x = data2.index
_y = data2.values
plt.figure(figsize=(20, 8), dpi=80)
# plt.bar(range(len(_x)), _y, width=0.3, color="orange")
plt.barh(range(len(_x)), _y, height=0.3, color="orange")
# plt.xticks(range(len(_x)), _x, fontproperties=my_font)
plt.yticks(range(len(_x)), _x, fontproperties=my_font)
plt.show()

---

七、统计方法 分组绘制折线图

import pandas as pd
from matplotlib import pyplot as plt

file_path = "./books.csv"
df = pd.read_csv(file_path)
# print(df.head(1))
# print(df.info())
# # 不同年份书的数量
# data1 = df[pd.notnull(df["original_publication_year"])]
# grouped = data1.groupby(by="original_publication_year").count()["title"]
# print(grouped)
# 不同年份书的平均评分
# 去除original_publication_year中nan的行
data2 = df[pd.notnull(df["original_publication_year"])]
# 按年份分组
grouped = data2["average_rating"].groupby(by=data2["original_publication_year"]).mean()
print(grouped)
_x = grouped.index
_y = grouped.values
plt.figure(figsize=(20, 8), dpi=80)
plt.plot(range(len(_x)), _y)
# 别忘改第二个x
plt.xticks(list(range(len(_x)))[::10], _x[::10].astype(int))
plt.show()

---

# 八、统计方法 统计次数

import pandas as pd
import numpy as np
from matplotlib import pyplot as plt

df = pd.read_csv("./911.csv")
print(df.head())
print(df.info())
# 统计不同类型紧急电话次数 按title前面字段分类
# print(df["title"].str.split(":"))
# 拿到前面逗号前
temp_list = df["title"].str.split(": ").tolist()
# cate_list = list(set([i[0] for i in temp_list]))
cate_list = [i[0] for i in temp_list]
# print(cate_list)
# # 构造全0数组
# zeros_df = pd.DataFrame(np.zeros((df.shape[0], len(cate_list))), columns=cate_list)
# for cate in cate_list:
#     zeros_df[cate][df["title"].str.contains(cate)] = 1
# print(zeros_df)
# print(zeros_df.sum(axis=0))
df["cate"] = pd.DataFrame(np.array(cate_list).reshape((df.shape[0], 1)))
print(df.groupby(by="cate").count()["title"])
# 统计不同月份不同类型

---

九、时间序列

import pandas as pd
from matplotlib import pyplot as plt

# # 生成一段时间 以天Date为频率
# print(pd.date_range(start="20171230", end="20180131", freq="D"))
# print(pd.date_range(start="20171230", end="20180131", freq="10D"))
# # 生成10个月份
# print(pd.date_range(start="20171230", periods=10, freq="M"))
# # 生成10个小时
# print(pd.date_range(start="20171230", periods=10, freq="H"))
# # 生成10个第一工作日
# print(pd.date_range(start="20171230", periods=10, freq="MS"))
# 时间序列索引
# 重采样：升降采样
# 统计不同月份电话次数变化情况
df = pd.read_csv("./911.csv")
df["timeStamp"] = pd.to_datetime(df["timeStamp"])
# 时间索引并原地修改
df.set_index("timeStamp", inplace=True)
# print(df.head())
# 月份重采样
count_by_month = df.resample("M").count()["title"]
# print(count_by_month)
_x = count_by_month.index
_x = [i.strftime("%Y%m%d") for i in _x]
_y = count_by_month.values
print(_x)
print(_y)
plt.figure(figsize=[20, 8], dpi=80)
plt.plot(range(len(_x)), _y)
plt.xticks(range(len(_x)), _x, rotation=45)
plt.show()

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
# 统计不同月份不同类型电话次数变化情况

df = pd.read_csv("./911.csv")
df["timeStamp"] = pd.to_datetime(df["timeStamp"])
# 时间索引并原地修改
temp_list = df["title"].str.split(": ").tolist()
cate_list = [i[0] for i in temp_list]
df["cate"] = pd.DataFrame(np.array(cate_list).reshape((df.shape[0], 1)))
plt.figure(figsize=[20, 8], dpi=80)
df.set_index("timeStamp", inplace=True)

for group_name, group_data in df.groupby(by="cate"):
    count_by_month = group_data.resample("M").count()["title"]
    # print(count_by_month)
    _x = count_by_month.index
    _x = [i.strftime("%Y%m%d") for i in _x]
    _y = count_by_month.values
    # print(_x)
    # print(_y)
    plt.plot(range(len(_x)), _y, label=group_name)

plt.xticks(range(len(_x)), _x, rotation=45)
plt.legend(loc="best")
plt.show()

---

十、案例

import pandas as pd
from matplotlib import pyplot as plt
file_path = "./PM2.5/BeijingPM20100101_20151231.csv"
df = pd.read_csv(file_path)
# print(df.head())
# print(df.info())
# 几列不同年月日小时放在一起
# DatetimeIndex时间戳
# PeriodIndex时间段
period = pd.PeriodIndex(year=df["year"], month=df["month"], day=df["day"], hour=df["hour"], freq="H")
# print(period)
df["datetime"] = period
# print(df.head(10))
df.set_index("datetime", inplace=True)
# 5个城市PM2.5随着时间变化
# 降采样
df = df.resample("7D").mean()
# 缺失数据处理
print(df["PM_US Post"])
data = df["PM_US Post"]
data_china = df["PM_Dongsi"]
# print(data)
_x = data.index
_x = [i.strftime("%Y%m%d") for i in _x]
_x_china = data_china.index
_x_china = [i.strftime("%Y%m%d") for i in _x_china]

_y = data.values
_y_china = data_china.values
plt.figure(figsize=[20, 8], dpi=80)
plt.plot(range(len(_x)), _y, label="US", alpha=0.7)
plt.plot(range(len(_x_china)), _y_china, label="CN", alpha=0.7)
plt.xticks(range(0, len(_x_china), 10), list(_x_china)[::10], rotation=45)
plt.legend(loc = "best")
plt.show()