qq_43308639的博客

明确目标目标url：https://www.ryjiaoyu.com/tag/details/7。静态网页。需要进入详情页爬取信息。scrapy框架建项目新建scrapy项目：1.创建爬虫项目，命令：scrapy startproject 项目名称。例如scrapy startproject RenYou2.创建爬虫文件，命令：scrapy genspider 文件名称 域名例如scrapy genspider ry https://www.ryjiaoyu.com/tag/det

from selenium import webdriveroption = webdriver.ChromeOptions()option.add_argument('headless')from selenium.webdriver.common.action_chains import ActionChainsimport csvimport timedef jddata(): commts=[] for i in range(1,11,1): cmts =

import requestsfrom lxml import etreeimport csv#爬取58同城二手房if __name__ == "__main__": headers = { "user-agent": "Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 80.0.3987.162Safari / 537.36" }

import requestsimport jsonimport csvif __name__ == "__main__": headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36' } fp = open('D:/Python/

import requestsimport pymysqlfrom lxml import etreeimport csvdef getHtml(url): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ...

import requestsfrom bs4 import BeautifulSoupimport csvdef getHtml(url):headers = {‘User-Agent’: ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 S...