Python 爬取单个网页所需要加载的URL地址和CSS、JS文件地址

直接上代码：

脱敏后自用的py采集代码，

#!/usr/bin/env python
# -*- coding:utf-8 -*-
"""
@author:Andy
@file:xxx.py
@time:下午05:50
@desc:采集的文章数据进博客
"""
import os
import re
import time
import requests
from bs4 import BeautifulSoup, SoupStrainer
from requests.exceptions import RequestException
from hashlib import md5
from urllib.parse import urlparse
import urllib

headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'
}


def get_content():
    url = 'http://ask.xxxx.com/question/xxxx'  # url
    response = requests.get(url, headers=headers).text.replace('<i class="fa fa-paperclip"></i>', '')
    soup = BeautifulSoup(response, 'lxml')
    # div = soup.select('#aw-mod-body ueditor-p-reset')
    pattern = re.compile('<a\shref="(http://ask.apelearn.com/file.*?)".*?>(.*?)</a>', re.S)
    p = soup.find_all('a')
    for item in p:
        # print(str(item))
        result = re.findall(pattern, str(item))
        if result:
            # print(result)
            for i in result:
                url, name = i
                # print(i)
                yield {
                    'url': url,
                    'name': name
                }


def mkdir(path):
    # 去除首位空格
    path=path.strip()
    # 去除尾部 \ 符号
    path=path.rstrip("\\")
    # 判断路径是否存在
    # 存在     True
    # 不存在   False
    isExists=os.path.exists(path)
    # 判断结果
    if not isExists:
        # 如果不存在则创建目录
        # 创建目录操作函数
        os.makedirs(path)
        print(path+' 创建成功')
        return True
    else:
        # 如果目录存在则不创建，并提示目录已存在
        print(path+' 目录已存在')
        return False

def getUrl(html):
    #patterncss = '<link href="(.*?)"'
    patternjs = '<script src="(.*?)"'
    patternimg = '<img src="(.*?)"'
    #href = re.compile(patterncss, re.S).findall(html)
    href = re.compile(patternimg, re.S).findall(html)
    href += re.compile(patternjs, re.S).findall(html)
    return href

def getCssUrl(html):
    patterncss = '<link href="(.*?)"'
    href = re.compile(patterncss, re.S).findall(html)
    return href

# 下载网页
def download_html(root_path, url):
    a = urlparse(url)
    file_path = a.path
    file_name = os.path.basename(file_path)
    _, file_suffix = os.path.splitext(file_name)
    if file_suffix != '.html':
        file_name_real = file_name + '.html'
    else:
        file_name_real = file_name
    file_path_real = file_path.replace(file_name, '')
    file_path_reals = file_path_real.replace('/', "\\")
    all_file_path_real = root_path + file_path_reals + file_name_real
    headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
    re = requests.get(url, headers = headers)
    re.encoding = "utf-8"

    itemurl = getUrl(re.text)
    for item1 in itemurl:
        download_commonimgjs(root_path, item1)

    itemcssurl = getCssUrl(re.text)
    for item2 in itemcssurl:
        download_css(root_path, item2)

    new_text = re.text.replace('https://www.xxxxxx.com', 'http://www.xxxxx.com')
    new_texts = new_text.replace('xxxxxx.com', '3cinno.shanhubei.com')
    with open(all_file_path_real, "w+", encoding="utf-8") as html_file:
        html_file.write(new_texts)

def download_commonimgjs(root_path, url):
    if str(url[:1]) == r"/":
        imgurl = "https://www.xxxxxx.com" + url
    else:
        imgurl = url
    a = urlparse(imgurl)
    file_path = a.path
    file_name = os.path.basename(file_path)
    _, file_suffix = os.path.splitext(file_name)
    # print(os.path.curdir(file_path))
    match_url = file_path.replace(file_name, '')
    match_url_new = match_url.replace('/', "\\")
    newmkpath = root_path + match_url_new
    if os.path.isfile(newmkpath + file_name):
        return
    # 调用函数
    mkdir(newmkpath)
    try:
        opener = urllib.request.build_opener()
        opener.addheaders = [('User-agent',
                              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36')]
        urllib.request.install_opener(opener)
        urllib.request.urlretrieve(imgurl, newmkpath + file_name)
    except urllib.error.HTTPError:
        print('error')



def download_img(root_path, url):
    if str(url[:1]) == r"/":
        imgurl = "https://www.xxxxxx.com" + url
    else:
        imgurl = url
    a = urlparse(imgurl)
    file_path = a.path
    file_name = os.path.basename(file_path)
    _, file_suffix = os.path.splitext(file_name)
    # print(os.path.curdir(file_path))
    match_url = file_path.replace(file_name, '')
    match_url_new = match_url.replace('/', "\\")
    newmkpath = root_path + match_url_new
    # 调用函数
    mkdir(newmkpath)
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36')]
    urllib.request.install_opener(opener)
    urllib.request.urlretrieve(imgurl, newmkpath + file_name)

def download_js(root_path, url):
    if str(url[:1]) == r"/":
        imgurl = "https://www.xxxxxx.com" + url
    else:
        imgurl = url
    a = urlparse(imgurl)
    file_path = a.path
    file_name = os.path.basename(file_path)
    _, file_suffix = os.path.splitext(file_name)
    # print(os.path.curdir(file_path))
    match_url = file_path.replace(file_name, '')
    match_url_new = match_url.replace('/', "\\")
    newmkpath = root_path + match_url_new
    # 调用函数
    mkdir(newmkpath)
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent',
                          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36')]
    urllib.request.install_opener(opener)
    urllib.request.urlretrieve(imgurl, newmkpath + file_name)

def download_css(root_path, url):
    if str(url[:1]) == r"/":
        imgurl = "https://www.xxxxxx.com" + url
    else:
        imgurl = url
    a = urlparse(imgurl)
    file_path = a.path
    file_name = os.path.basename(file_path)
    _, file_suffix = os.path.splitext(file_name)
    if file_suffix != '.css':
        return
    # print(os.path.curdir(file_path))
    match_url = file_path.replace(file_name, '')
    match_url_new = match_url.replace('/', "\\")
    newmkpath = root_path + match_url_new
    if os.path.isfile(newmkpath + file_name):
        return
    # 调用函数
    mkdir(newmkpath)
    try:
        opener = urllib.request.build_opener()
        opener.addheaders = [('User-agent',
                              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36')]
        urllib.request.install_opener(opener)
        urllib.request.urlretrieve(imgurl, newmkpath + file_name)
    except urllib.error.HTTPError:
        print('error')

def get_xml():
    url = 'https://www.xxxxxx.com/sitemap-1.xml'
    headers = {'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
    res = requests.get(url, headers=headers)
    res.encoding = "utf-8"
    # 根据你的文章链接格式写正则匹配，可能与我的不一样
    r = re.compile(r'https://www.xxxxxx.com/\S*?')
    big = re.findall(r, res.text)
    for i in big:
        print(i)


def main():
    # get_content()
    # url = r'https://www.xxxxxx.com/news/xxxx-proje-20711498'
    url = r'https://www.xxxxxx.com/uploads/20218080/logo202107221507387902092.png'
    # 定义要创建的目录
    root_path = "F:\\Project-cz\\shanhubei\\3cinno"
    #download_img(root_path, url)

    #htmlurl = r'https://www.xxxxxx.com/3d-clear-led-dmx-ball'
    #download_html(root_path, htmlurl)

    cssurl = r'https://www.xxxxxx.com/images/m184/black/style.css'
    #download_css(root_path, cssurl)

    #demourl = 'https://www.xxxxxx.com/Content/kcim/js/layim-public.js?t=20190404'
    #demo(demourl)

    get_xml()


def demo(url):
    a = urlparse(url)
    file_path = a.path
    print(a.scheme)
    print(a.hostname)
    print('a.file_path=' + file_path)
    file_name = os.path.basename(file_path)
    print('file_name=' +file_name)
    _, file_suffix = os.path.splitext(file_name)
    print('a.file_suffix=' + file_suffix)





if __name__ == '__main__':
    main()

 来源：http://www.shanhubei.com/archives/2491.html