新手入门爬虫lxml+Requests+MongoDB

测试爬取一加社区

import  requests
from lxml import etree
import  pymongo
import proxyIP
import time

def get_UrlInfos(url,proxyIp):
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3493.3 Safari/537.36'
    }
    response = requests.get(url,proxies = proxyIp,headers = header).text
    html = etree.HTML(response)
    items = html.xpath('//tbody')
    for item in items:
        info = {
            'type':item.xpath('tr/th/div[2]/span/em/a')[0].text.strip() if len(item.xpath('tr/th/div[2]/span/em/a'))>0 else '',
            'title':item.xpath('tr/th/div[2]/a/text()')[0],
            'author':item.xpath('tr/th/div[2]/div/em[2]/a/text()')[0].strip(),
            'time': item.xpath('tr/th/div[2]/div/em[3]/span/text()')[0].strip() 
            if len(item.xpath('tr/th/div[2]/div/em[3]/span/text()'))>0 else item.xpath('tr/th/div[2]/div/em[3]/span/span/@title')[0],
            'view' :int(item.xpath('tr/th/div[2]/div/em[1]/text()')[0].split(':')[1]),
            'reply':int(item.xpath('tr/th/div[2]/div/em[1]/a/text()')[0].strip())
        }
        yijia.insert_one(info)
                
if __name__ == '__main__':
    start = time.time()
    mongoclient = pymongo.MongoClient('127.0.0.1',27017)
    mydb = mongoclient['mydb']
    yijia = mydb['yijia']
    proxyIp = proxyIP.getIp()
    urls = ['http://www.oneplusbbs.com/forum-116-{}.html'.format(i) for  i in range(2,1000)]
    for url in  urls:
        get_UrlInfos(url,proxyIp)
    end = time.time()
    print("单线程完成耗时：%d"%(end-start))

多线程可以自己用multiprocessing玩一下