cats_miao---python/java

用命令在终端创建一个项目: scrapy startproject myvipspider进入到myvipspider项目下运行命令: scrapy genspider weipin "vip.com"项目下有这几个文件,当settings.py文件设置:# -*- coding: utf-8 -*-# Scrapy settings for weipinhui project## For...

import pymysqlclass MysqlPipeline(object): def __init__(self): self.conn = None self.cur = None def open_spider(self, spider): self.conn = pymysql.connect( ...

 import loggingimport pymysql     logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG)     class MysqlPipeline(object):   def __init__(self): self.conn = None self.cur = None   def open...

我的源码地址: https://github.com/miaochaoqing123/taobaoMeishiSpider   from time import sleepimport re from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.suppo...

爬取猫眼电影网的排行榜单(进程池)    import reimport json import requests from multiprocessing import Pool # 导入捕获异常库 from requests.exceptions import RequestException   # 获取网页数据 def get_one_page(url): headers = { "Use...

from time import sleepimport jsonimport csvfrom selenium import webdriverfrom bs4 import BeautifulSoupimport urllib.parseimport urllib.requestclass SpiderVip(object):    def __init__(self,url,start_pa...

import urllib.requestimport urllib.parsefrom bs4 import BeautifulSoupimport jsonclass LianJiaSpider(object):    def __init__(self,url,start,end):        self.url = url        self.start = start       ...

# 如有错的地址,请大神们指定,谢谢import randomimport reimport urllib.requestlistQqAll = []listNum = []for i in range(1,50): # 循环49次爬取 xx通讯录 共49页 urlPath = "xx通讯录网址" # 就不显示了 下面的变量名也改了Xx,我自己是带拼音的,...

# 思路:# 1.遍历目录# 2.如果是目录接着遍历# 3.如果是文件# 1.以只读的形式打开文件# 2.一行一行的读# 3.提取邮箱类型# 4.判断该类型对应的目录是否存在# 存在# 不存在---> 创建目录# 5.写# 关闭文件import osdef mailType(sour...

接上一编weipin.py文件的代码 :# -*- coding: utf-8 -*-import scrapyfrom weipinhui.items import WeipinhuiItemimport urllib.parseclass WeipinSpider(scrapy.Spider): name = 'weipin' allowed_domains = ['...

# coding: utf-8from urllib.request import urlretrieveimport requestsimport osdef hero_imgs_download(url, header): req = requests.get(url=url, headers=header).json() hero_num = l...