SVM支持向量机文本分析实例

 我使用的数据集是从网上下载的，下载地址和具体内容直接看网址吧

Home Page for 20 Newsgroups Data Set (qwone.com)

这里是下载，然后解压就可以用了 

import os
import sys
# 加载数据
from sklearn import datasets
twenty_train = datasets.load_files("C:/Users/Rick/Desktop/20news-bydate/20news-bydate-train")
twenty_test = datasets.load_files("C:/Users/Rick/Desktop/20news-bydate/20news-bydate-test")
#打印一下文件数量
print(len(twenty_train.target_names),len(twenty_train.data),len(twenty_train.filenames),len(twenty_test.data))
#计算词频
from sklearn.feature_extraction.text import CountVectorizer
count_vect = CountVectorizer(stop_words="english",decode_error='ignore')
X_train_counts = count_vect.fit_transform(twenty_train.data)
X_train_counts.shape
#使用TF-IDF进行特征提取
from sklearn.feature_extraction.text import TfidfTransformer
tf_transformer = TfidfTransformer(use_idf = False).fit(X_train_counts)
X_train_tf = tf_transformer.transform(X_train_counts)
X_train_tf.shape
tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
X_train_tfidf.shape
from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB().fit(X_train_tfidf,twenty_train.target)
docs_new = ['God is love','OpenGL on the GPU is fast']
X_new_counts = count_vect.transform(docs_new)
X_new_tfidf = tfidf_transformer.transform(X_new_counts)
predicted = clf.predict(X_new_tfidf)
for doc,category in zip(docs_new,predicted):
    print(("%r => %s") %(doc,twenty_train.target_names[category]))
#建立管道
from sklearn.pipeline import Pipeline
text_clf = Pipeline([('vect',CountVectorizer(stop_words="english",decode_error='ignore')),
                    ('tfidf',TfidfTransformer()),
                    ('clf',MultinomialNB()),
                    ])
text_clf = text_clf.fit(twenty_train.data,twenty_train.target)
#朴素贝叶斯分类器
import numpy as np
docs_test = twenty_test.data
predicted = text_clf.predict(docs_test)
np.mean(predicted == twenty_test.target)
#线性核支持向量机
from sklearn.linear_model import SGDClassifier
text_clf_2 = Pipeline([('vect',CountVectorizer(stop_words='english',decode_error='ignore')),
                      ('tfidf',TfidfTransformer()),
                      ('clf',SGDClassifier(loss = 'hinge',penalty = 'l2',
                                          alpha = 1e-3,n_iter_no_change = 5, random_state = 42)),
                      ])
_ = text_clf_2.fit(twenty_train.data,twenty_train.target)
predicted = text_clf_2.predict(docs_test)
np.mean(predicted == twenty_test.target)

通过朴素贝叶斯和向量机两种方式进行分类结果如下 

 

可以看到向量机是优于朴素贝叶斯分类的结果的