DEBUG: Filtered offsite request to

在做爬虫项目时，出现了一个问题，解析一个网站二次爬取时没有获取到数据，就写了一个测试程序试了下，测试程序如下

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule




class ZhenaiSpider(CrawlSpider):
    name = 'zhenai'
    allowed_domains = ['www.zhenai.com']
    start_urls = ['http://www.zhenai.com/zhenghun/beijing/1']


    rules = (
        Rule(LinkExtractor(allow=r'http://www.zhenai.com/zhenghun/beijing/\d+'), callback='parse_item', follow=False),
    )


    def parse_item(self, response):
        a_list = response.xpath('//div[@class="content"]/table//tr/th/a')
        for a in a_list:
            item = {}
            title = a.xpath('./text()').extract_first()
            item['title'] = title