2021.03.31 代理的使用

2021.03.31

代理的使用

import requests
import time
from re import findall
import csv
from selenium.webdriver import Chrome
from selenium import webdriver


# 获取代理ip
def get_ips():
    url = 'http://piping.mogumiao.com/proxy/api/get_ip_bs?appKey=6226c130427f487385ad7b5235bc603c&count=2&expiryDate=0&format=2&newLine=3'
    response = requests.get(url)
    if response.text[0] == '{':
        return None
    result = [x for x in response.text.split('\n') if x]
    return result


def save_data(datas, file):
    with open(file, 'w', encoding='utf-8', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(['排名', '电影名称', '评分'])
        writer.writerows(datas)


def get_douban1():
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
    }

    # 不断获取代理ip只带取到ip位置
    while True:
        ips = get_ips()
        if ips:
            break
        time.sleep(11)

    print(ips)
    # 添加代理
    proxies = {'http': ips[0], 'https': ips[1]}
    response = requests.get('https://movie.douban.com/top250', headers=headers, proxies=proxies)
    if response.status_code == 200:
        re_str = r'(?s)<li>\s*<div class="item">.+?alt="(.+?)".+?"v:average">(.+?)</span>.+?</li>'
        result = findall(re_str, response.text)
        new_result = [[index+1, *result[index]] for index in range(len(result))]
        save_data(new_result, '豆瓣电影.csv')
    else:
        print('获取数据失败')


def get_douban2():
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
    }
    start = 0
    while True:
        url = f'https://movie.douban.com/top250?start={start}'
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            start += 25
        if start == 275:
            break

selenium设置代理

from selenium.webdriver import Chrome
from selenium.webdriver import ChromeOptions
import requests


# 获取代理ip
def get_ips():
    url = 'http://piping.mogumiao.com/proxy/api/get_ip_bs?appKey=6226c130427f487385ad7b5235bc603c&count=2&expiryDate=0&format=2&newLine=3'
    response = requests.get(url)
    if response.text[0] == '{':
        return None
    result = [x for x in response.text.split('\n') if x]
    return result


ips = get_ips()
if ips:
    options = ChromeOptions()
    # 添加代理
    # option.add_argument('--proxy-server=http://代理ip:端口')
    options.add_argument(f'--proxy-server=http://{ips[0]}')
    b = Chrome(options=options)
    b.get('https://movie.douban.com/top250')
else:
    print('获取ip失败!')

selenium控制页面滚动

from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
import time

b = Chrome()
b.get('https://www.jd.com/')
search = b.find_element_by_css_selector('#key')
search.send_keys('手机')
search.send_keys(Keys.ENTER)
time.sleep(1)
# 一步到位，滚动到底部
# b.execute_script('window.scrollTo(0, document.body.scrollHeight)')
# 一点一点的滚动： 500 -> 修改成每次需要滚动的距离，单位是像素;  1000 -> 每次滚动的时间间隔，单位是毫秒
b.execute_script("""
            height = 500
            t = setInterval(function(){
                if (height > document.body.scrollHeight){
                    clearInterval(t)
                }
                window.scrollTo(0, height)
                height += 500
            }, 1000)
""")

bs4数据解析

from bs4 import BeautifulSoup

# BeautifulSoup是基于css选择器的解析库

# 1.准备数据（一般是通过requests或者selenium爬取到的）
with open('data.html', encoding='utf-8') as f:
    content = f.read()

# 2.创建BeautifulSoup解析器对象
# BeautifulSoup(解析对象, 解析器类型)
# 解析对象 - 一般是html格式字符串(一般是通过requests或者selenium爬取到的)
# 解析器类型 - lxml(最常用)、html.parser、xml、html5lib
soup = BeautifulSoup(content, 'lxml')

# 3.获取标签 - 通过css选择器
# 解析器对象.select(css选择器)  -  在整个页面中按照css选择器查找指定标签
# 标签对象.select(css选择器)   -   在当前标签中按照css选择器查找指定标签
# result = soup.select('p')
# print(result)

# result = soup.select('#f2')
# print(result)

# result = soup.select('div>p')
# print(result)

# 嵌套写法: 标签1.标签2.标签3....   ->  获取标签1中第一个标签2中的第一个标签3...
# div = soup.select('div')[0]
# print(div.p)

# ol = soup.select('#o1')[0]
# print(ol)
#
# print(ol.li)
# print(ol.select('li'))


# lis = soup.select('#o1>li')
# print(lis)

# 4.按照指定属性值查找标签
# 获取href属性值是'https://www.jd.com'的标签
result = soup.find_all(attrs={'href': 'https://www.jd.com'})
print(result)

# 获取tag属性是'hot'的所有标签
result = soup.find_all(attrs={'tag': 'hot'})
print(result)

# 获取tag属性是'hot'并且height属性是'100'的所有标签
result = soup.find_all(attrs={'tag': 'hot', 'height': '100'})
print(result)

# 5.获取内容和属性
# 标签.string  - 获取双标签的文字内容（注意：被获取的标签中不能有子标签，否则结果是None）
# 标签.contents - 获取双标签的内容（包括文字内容和子标签）
b1 = soup.select('#f1')[0]
print(b1.string)    # 我是font1

b2 = soup.select('#f2')[0]
print('===:', b2.string)   # ===: None

print(b1.contents)   # ['我是font1']
print(b2.contents, b2.contents[-1].string)   # ['我是font2 ', <a href="#">abc</a>]  abc

print(b1.get_text())    # 我是font1
print(b2.get_text())    # 我是font2 abc

# 6.获取标签属性
# 标签对象.attrs[属性名]
b3 = soup.body.div.a
print(b3.attrs['href'])    # https://www.baidu.com

b4 = soup.find_all(attrs={'title': 't百度'})[0]
print(b4.attrs['src'], b4.attrs['title'])    # https://www.baidu.com/img/PCtm_d9c8750bed0b3c7d089fa7d55720d6cf.png t百度