python完成网页下载（包括图片和css）

最新推荐文章于 2025-05-15 12:04:46 发布

renzha2010

最新推荐文章于 2025-05-15 12:04:46 发布

阅读量3.1k

点赞数

CC 4.0 BY-SA版权

分类专栏：爬虫文章标签： python 爬虫

本文链接：https://blog.youkuaiyun.com/renzha2010/article/details/9050677

爬虫专栏收录该内容

1 篇文章

订阅专栏

该博客介绍了一个Python程序，用于下载网页内容并同时抓取和保存页面中的CSS文件和图片。通过BeautifulSoup库解析HTML，找到链接的CSS和图片资源，然后替换为本地路径，最后将整个页面内容写入到一个HTML文件中。此外，程序还创建了必要的文件夹来存储下载的CSS和图片资源。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import urllib2
import os
from bs4 import BeautifulSoup 
import re
siteUrls = " "

url = "http://www.sina.com.cn"
def getContent(url):
	content = urllib2.urlopen(url).read()
	content = writeCss(url,content)
	content = writefileName(url,content)
	fileNames = re.findall(r'/[^\?]*\?([^/|^\?]*)$',url)
	fileName = fileNames[0]
	print fileName
	f = file(fileName+".html",'w')
	f.write(content)
	f.close()

def writeCss(url,content):
	soup = BeautifulSoup(content)
	csss = soup.findAll('link',attrs={'type':'text/css'})
	css_pat = re.compile('.*/(.*)\.css')
	fileNames = re.findall(r'/[^\?]*\?([^/|^\?]*)$',url)
	fileName = fileNames[0]
	print fileName
	for css in csss:
		cssnames = re.findall(r'.*/(.*)\.css',str(css))
		cssurls = re.findall(r'.*href=\"([^\"]*)\"',str(css))
#		print cssnames[0]
#		print cssurls[0]
		cssurl = "http://review.artintern.net/" + cssurls[0]
#		print cssurl
		content = content.replace(cssurls[0],fileName + "/" + cssnames[0]+".css")
		print os.path.isdir(fileName)
		if not os.path.isdir(fileName):
			os.mkdir(fileName)
		csscontent = urllib2.urlopen(cssurl).read()
		cssNewName = fileName+"/"+cssnames[0]+".css"
		cssfile = file(cssNewName,'w')
		cssfile.write(csscontent)
		cssfile.close()
	return content

def writefileName(url,content):
	soup = BeautifulSoup(content)
	imgs = soup.findAll('img')
	img_pat = re.compile('.*/(.*)\.[jpg|gif]')
	fileNames = re.findall(r'/[^\?]*\?([^/|^\?]*)$',url)
	fileName = fileNames[0]
	for img in imgs:
		imgNames = re.findall(r'.*/(.*)\.[jpg|gif]',str(img))
		imgType = re.findall(r'.*/.*\.([^ ]*)"',str(img))
		imgUrls = re.findall(r'.*src=\"([^\"]*)\"',str(img))
#		print imgNames[0]
#		print imgType[0]
#		print imgUrls[0]
		imgUrl = "http://review.artintern.net/" + imgUrls[0]
#		print imgUrl
		content = content.replace(imgUrls[0],fileName+"/"+imgNames[0]+"."+imgType[0])
		if not os.path.isdir(fileName):
			os.mkdir(fileName)
		imgContent = urllib2.urlopen(imgUrl).read()
		imgNewName = fileName+"/"+imgNames[0]+"."+imgType[0]
		imgfile = file(imgNewName,'w')
		imgfile.write(imgContent)
		imgfile.close()
	return content

getContent(url)