Fredreck1919的博客

# 爬取糗百信息import threading from queue import Queue import requests from lxml import etree import time # 最大开启采集线程数（并发数） concurrent = 3 # 解析并发数 conparse = 3 class Crawl(threading.Thread): def __init...

（78）--用scrapy框架简单爬取赶集网信息# main.pyfrom scrapy import cmdline cmdline.execute('scrapy crawl spider_ganji'.split()) # spider_ganji.py# -*- coding: utf-8 -*- import scrapy from ..items import WwwGanjiIte...

# 用框架爬取博客园信息并保存到数据库# cnlogs_itload.py# -*- coding: utf-8 -*- import scrapy import re from ..items import CnblogItem,CnblogItemLoader from datetime import datetime from w3lib.html import remove_tags f...

# 多线程爬取腾讯招聘职位信息并存入数据库 # mydb.pyimport pymysql class Mydb: def __init__(self): try: self.conn = pymysql.connect('127.0.0.1','root','123456','han',charset='utf8') se...

# 用框架爬取交友信息并保存到数据库# yuehui.py# -*- coding: utf-8 -*- import scrapy import json import math from ..items import YuehuiItem,TrueHeartItem import re class YuehuiSpider(scrapy.Spider): name = 'yuehui...

# 用多进程爬取腾讯招聘的文本信息from multiprocessing import Pool import requests from bs4 import BeautifulSoup import time base_url = 'http://hr.tencent.com/position.php?start=%d' headers = { 'User-Agent' : '...

# 二级爬取58交友的名字、年龄、身高、学历、图片信息，并把这些信息保存到数据库中# 首先要在当前目录下建立一个58文件夹，然后通过Navicat连接到数据库# mydb.pyimport pymysql class Mydb: def __init__(self): try: self.conn = pymysql.connect('127.0....

# 用框架爬取腾讯招聘信息并保存到数据库# main.pyfrom scrapy import cmdline cmdline.execute('scrapy crawl tencent_new'.split())# tencent_new.py# -*- coding: utf-8 -*- import scrapy from urllib import request,parse from ...

# 爬取兄弟连老师信息，先建一个teacher文件夹from bs4 import BeautifulSoup import requests from urllib import request import json base_url = 'http://www.itxdl.cn/activity/teacher/teacher_lieibiao/' response = requests....

# 用框架爬取腾讯招聘首页# tencent.py# -*- coding: utf-8 -*- import scrapy from ..items import JobItem from datetime import datetime class TencentSpider(scrapy.Spider): name = 'tencent' allowed_domains =...

#  爬取兄弟连网页信息，包括详细信息和图片import requests from lxml import etree import json from urllib import request def getUrl(): base_url = 'http://www.itxdl.cn/html/php/phparticles/' response = requests.ge...

# 用框架爬取兄弟连老师姓名# xdl.py# -*- coding: utf-8 -*- import scrapy class XdlSpider(scrapy.Spider): name = 'xdl' # allowed_domains = ['www.xdl.cn'] start_urls = ['http://www.itxdl.cn/activity/te...

# 用JS代码自动运行程序from selenium import webdriver import time chrome = webdriver.Chrome(executable_path=r'E:\Python\python爬虫\chromedriver.exe') base_url = 'https://image.baidu.com/search/index?ct=201326592&...

# 爬取拉勾网招聘信息from selenium import webdriver import time from lxml import etree dc = { 'phantomjs.page.customHeaders.User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM...

# 爬取哦漫画图片并下载到相应文件夹from selenium import webdriver import time import requests from bs4 import BeautifulSoup import re from urllib import request,parse import os # 1.获取漫画章节链接 phantom = webdriver.Phant...

#  selenium的简单应用from selenium import webdriver import time browser = webdriver.Chrome(executable_path=r'E:\Python\python爬虫\chromedriver.exe') base_url = 'http://www.baidu.com' browser.get(base_url) ...

# 增加搜索功能import re import urllib.request import json def InsertDict(D, key, value): D.setdefault(key) D[key] = value def GetHtml(url): response = urllib.request.urlopen(url) text = re...