卡方检验两分类实现

最新推荐文章于 2025-05-20 09:57:35 发布

dataastron

最新推荐文章于 2025-05-20 09:57:35 发布

阅读量2k

点赞数

CC 4.0 BY-SA版权

分类专栏： python

本文链接：https://blog.youkuaiyun.com/dataastron/article/details/79445434

python 专栏收录该内容

28 篇文章

订阅专栏

该博客介绍了如何使用Python进行卡方检验，用于比较两个分类数据集中特定词汇的分布差异。博主首先纠正了之前算法的错误，然后通过读取Excel文件获取数据，使用jieba进行分词，并定义了计算卡方值的函数。最后，运行示例数据并输出显著差异的词汇及其卡方值。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import jieba
import numpy as np
import xlrd
import re
import json

# 数据加载
# jieba词库设置

#读取文本，读取其中1个
#统计包含a的单词和不包含a的单词
#统计主题
def chisquare(datasetTrue, datasetFlase):
dicta ={}
dictb ={}
for i in datasetTrue:
for word in i:
if word not in dicta:
dicta[word] = 0
else:
dicta[word] = dicta[word] + 1
for i in datasetFlase:
if word in i:
if word not in dictb:
dictb[word] = 0
else:
dictb[word] = dictb[word] + 1

keyset = set()
for i in dicta:
keyset.add(i)
for i in dictb:
keyset.add(i)

dictres = {}
for w in keyset:
if w not in dicta:
a = 0
else:
a = dicta[w]
if w not in dictb:
b = 0
else:
b = dictb[w]

c = len(datasetTrue)-a
d = len(datasetFlase)-b
#print(a,b,c,d)
if (a>0 or b>0):
chi = (a * d - b * c) ** 2 / ((a + c) * (a + b) * (b + d) * (c + d))
print(w,chi)
dictres[w] = chi
return dictres
pass

def data_gen():
xlsfile = r'E:\work\chisquare\file\test0111.xlsx'
book = xlrd.open_workbook(xlsfile) # 得到Excel文件的book对象
sheet0 = book.sheet_by_index(0) # 通过sheet索引获得sheet对象
nrows = sheet0.nrows
jiahuoTrue = []
jiahuoFlase = []
for rownum in range(1, nrows):
rowvalue = sheet0.cell_value(rownum, 2)
rowflag = sheet0.cell_value(rownum, 1)
if (rowflag == 1):
jiahuoTrue.append(rowvalue)
elif (rowflag == 0):
jiahuoFlase.append(rowvalue)
return jiahuoTrue,jiahuoFlase

def getwordlist(datasetTrue,datasetFlase):
s = set()
for i in datasetTrue:
for w in i:
s.add(w)
for i in datasetFlase:
for w in i:
s.add(w)
return s

# 运行评分过程
def run():
jt,jf =data_gen()
datasetTrue = [list(set(jieba.lcut(d))) for d in jt]
datasetFlase = [list(set(jieba.lcut(d))) for d in jf]
dict = chisquare(datasetTrue, datasetFlase)
import pandas as pd
arr = list(dict.items())
df = pd.DataFrame(list(dict.items()), columns=['word', 'score'])
return df
pass

if __name__ == '__main__':
df1=run()
df1.sort_values("score").to_csv("res2")

print("end")

之前的老版本一个词一个词计算，算法是错误的，要跑几分钟

import jieba
import numpy as np
import xlrd
import re
import json

# 数据加载
# jieba词库设置

#读取文本，读取其中1个
#统计包含a的单词和不包含a的单词
#统计主题
def chisquare(datasetTrue, datasetFlase, word):
a=b=c=d=0
for i in datasetTrue:
if word in i:
a+=1
else:
c+=1
for i in datasetFlase:
if word in i:
b+=1
else:
d+=1
chi = (a*d-b*c)**2/((a+c)*(a+b)*(b+d)*(c+d))
#print(chi)
return chi
pass

def data_gen():
xlsfile = r'E:\work\chisquare\file\test0111.xlsx'
book = xlrd.open_workbook(xlsfile) # 得到Excel文件的book对象
sheet0 = book.sheet_by_index(0) # 通过sheet索引获得sheet对象
nrows = sheet0.nrows
jiahuoTrue = []
jiahuoFlase = []
for rownum in range(1, nrows):
rowvalue = sheet0.cell_value(rownum, 2)
rowflag = sheet0.cell_value(rownum, 1)
if (rowflag == 1):
jiahuoTrue.append(rowvalue)
elif (rowflag == 0):
jiahuoFlase.append(rowvalue)
return jiahuoTrue,jiahuoFlase

def getwordlist(datasetTrue,datasetFlase):
s = set()
for i in datasetTrue:
for w in i:
s.add(w)
for i in datasetFlase:
for w in i:
s.add(w)
return s

# 运行评分过程
def run():
jt,jf =data_gen()
datasetTrue = [jieba.lcut(d) for d in jt]
datasetFlase = [jieba.lcut(d) for d in jf]
word = "物流"
s = getwordlist(datasetTrue, datasetFlase)
dict = {}
for word in s:

dict[word] = chisquare(datasetTrue, datasetFlase, word)
if dict[word]>0.01:
print(word,dict[word])
pass

if __name__ == '__main__':
run()
print("end")