新闻抓取爬虫 PPT

不用多言，请见附件 ：）

上slideshare view -> [url]http://www.slideshare.net/edisonlz/ppt-4395676[/url]

补充，不使用分词的，很简单，但效果一般的排重算法如下：

#encoding = utf8
import re 
import sys,os
import re

chars = []
def generate_chars(strList):
    """
    创建字典
    """
    for sr in strList:
        for char in sr:
            if char not in chars:
                chars.append(char)


def vector(sr):
    """
    将字符串转换为向量空间
    """
    vec = []
    for char in chars:
        if sr.find(char) !=-1:
            vec.append(1)
        else:
            vec.append(0)
    return vec

def intersaction(str1,str2):
    """
    计算交集
    """
    v1 =vector(str1)
    v2= vector(str2)
    rv = 0
    for i in xrange(len(v1)):
        rv += v1[i] * v2[i]
    return rv

def detect_repeat(strList):
    """
    检测字符串字符匹配
    """
    chars = generate_chars(strList)

    #临界值
    critical = 3
    results = []
    for st in strList:
        if all(intersaction(st,rt) < critical  for rt in results):
            results.append(st)

    #返回检测结果
    return results


if __name__ =="__main__":
    strLists = [u"大家好啊",u"大家好",u"张靓颖新专辑",u"张靓颖专辑"]
    results = detect_repeat(strLists)
    print "result " + " > " * 20
    for ru in results:
        print ru.encode("utf-8")