day5-优快云博客

作业1：

from selenium import webdriver
import time
#获取驱动对象
driver = webdriver.Chrome()
try:
#自动登陆抽屉新热榜
#发送get请求
driver.get('https://dig.chouti.com/ ')
#隐式等待
driver.implicitly_wait(10)
#获取 '登陆' 按钮
send_tag = driver.find_element_by_id('login_btn')
send_tag.click()
#获取手机号输入框
username = driver.find_element_by_class_name('login-phone')
username.send_keys('*******')
time.sleep(1)
#获取密码输入框
password = driver.find_element_by_class_name('pwd-password-input')
password.send_keys('*******')
time.sleep(1)
#获取 '登陆' 按钮
login = driver.find_elements_by_link_text('登录')
login[1].click()
time.sleep(10)
finally:
driver.close()

作业2

import requests
import re
import time
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
}
def get_index(url):
time.sleep(1)
response1 = requests.get(url, headers=headers)
return response1
def parse_index(text):
ip_list1 = re.findall('<tr>.*?<td data-title="IP">(.*?)</td>.*?<td data-title="PORT">(.*?)</td>', text, re.S)
for ip_port in ip_list1:
ip1 = ':'.join(ip_port)
yield ip1
def test_ip(ip2):
print('测试ip: %s' % ip2)
try:
proxies = {'https': ip2}
# ip测试网站
ip_url1 = 'https://www.ipip.net/'
# 使用有效与无效的代理对ip测试站点进行访问，若返回的结果为200则代表当前测试ip正常
response2 = requests.get(ip_url1, headers=headers, proxies=proxies, timeout=1)
if response2.status_code == 200:
return ip
# 若ip代理无效则抛出异常
except Exception as e:
print(e)
# 使用代理爬取nba
def spider_nba(good_ip1):
url = 'https://china.nba.com/'
proxies = {'https': good_ip1}
response3 = requests.get(url, headers=headers, proxies=proxies)

print(response3.status_code)
print(response3.text)

if __name__ == '__main__':
base_url = 'https://www.kuaidaili.com/free/inha/{}/'
for line in range(1, 2905):
ip_url = base_url.format(line)
response = get_index(ip_url)
ip_list = parse_index(response.text)
for ip in ip_list:
good_ip = test_ip(ip)
if good_ip:
spider_nba(good_ip)

'''
今日内容:
1.requests之post请求
2.requests高级用法
3.selenium模块
4.万能破解登录
'''
'''
请求url:
请求方式：post
请求头：
# 上一次请求从哪里来
referer: https://github.com/login
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36
请求体：
commit: Sign in
utf8: ✓
authenticity_token: j+v64L9wj7tLRsCwhFtjVM2Gt3zJnu6ZZZCu9J+oEbxQlIU+vJrH/uga9zGmcA6GWKGiqVZOVG6Opb5lOEYsWw==
login: lyj68
password: 1998223689lyj
webauthn-support: supported
'''

# 1.访问login页获取token
'''
请求url:
https://github.com/login
请求方式: get
响应头:
Set-Cookie
请求头：
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36

'''
import requests
import re
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}
response = requests.get(url='https://github.com/login',headers=headers)
#print(response.text)

authenticity_token = re.findall('<input type="hidden" name="authenticity_token" value="(.*?)" /> ',response.text, re.S)[0]

print(authenticity_token)

# 2.往session发送post请求
headers2 = {
'Referer': 'https://github.com/login',
'User-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36'
}

form_data = {
'commit': 'Sign in',
'utf8': '✓',
'authenticity_token':authenticity_token,
'login': 'lyj68',
'password': '1998223689lyj',
'webauthn-support': 'supported'
}
# 把login页返回的cookies信息转换成字典
login_cookies = response.cookies.get_dict()
# 往session地址发送post请求
# 携带请求头、请求体、login页的cookies信息
response2= requests.post(url='https://github.com/session',data=form_data,headers=headers2,cookies=login_cookies)

print(response2.status_code)
#print(response2.text)
with open('gethub.html','w',encoding='utf-8') as f:
f.write(response2.text)

# response响应
# import requests
# response = requests.get("https://baidu.com")

# #reponse 响应
# print(response.status_code) # 获取响应状态码
# print(response.url) #获取urlz地址
# print(response.encoding) #字符编码
# response.encoding='utf-8'
# print(response.text) # 获取文本
# print(response.content) #获取二进制流
# print(response.headers) #获取页面头部信息
# print(response.history) #上一次跳转的地址
# # 1.返回cookies字典 2.返回cookies对象
# print(response.cookies) #获取cookies信息
# print(response.cookies.get_dict()) #获取cookies信息转换成字典
# print(response.cookies.items()) #获取cookies信息转换成字典
# print(response.encoding)
# print(response.elapsed) #访问时间

#证书验证(大部分网站都是https)
# import requests
# 如果是ssl请求,首先检查证书是否合法,不合法则报错,程序终端
# response = requests.get('https://www.xiaohuar.com')
# print(response.status_code)
#
# # 改进1:去掉报错,但是会报警告
# import requests
# response = requests.get('https://www.xiaohuar.com', verify=False)
# # 不验证证书,报警告,返回200
# print(response.status_code)
#
# # 改进2:去掉报错,并且去掉警报信息
# import requests
# import urllib3
# urllib3.disable_warnings() # 关闭警告
# response = requests.get('https://www.xiaohuar.com', verify=False)
# print(response.status_code)

# 改进3:加上证书
# 很多网站都是https,但是不用证书也可以访问,大多数情况都是可以携带也可以不携带证书
# 知乎\百度等都是可带可不带
# 有硬性要求的,则必须带，比如对于定向的用户,拿到证书后才有权限访问某个特定网站

# import urllib3
# # urllib3.disable_warnings() # 关闭警告
# response = requests.get(
# 'https://www.xiaohuar.com',
# # verify=False,
# cert=('/path/server.crt', '/path/key'))
# print(response.status_code)

# 官网链接: http://docs.python-requests.org/en/master/user/advanced/#proxies

# 代理设置:先发送请求给代理,然后由代理帮忙发送(封ip是常见的事情)
import requests

proxies={
# 带用户名密码的代理,@符号前是用户名与密码
# 'http':'http://tank:123@localhost:9527',
# 'http':'http://localhost:9527',
'https':'https://localhost:9527'
}
response=requests.get('https://www.12306.cn',proxies=proxies)
print(response.status_code)

# # 支持socks代理,安装:pip install requests[socks]
# import requests
# proxies = {
# 'http': 'socks5://user:pass@host:port',
# 'https': 'socks5://user:pass@host:port'
# }
# respone=requests.get('https://www.12306.cn',
# proxies=proxies)
#
# print(respone.status_code)

import requests
import re
import time

HEADERS = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36',
}

def get_index(url):
time.sleep(1)
response = requests.get(url, headers=HEADERS)
return response

def parse_index(text):
ip_list = re.findall('<tr class="odd">.*?<td>(.*?)</td>.*?<td>(.*?)</td>', text, re.S)
for ip_port in ip_list:
ip = ':'.join(ip_port)
yield ip

def test_ip(ip):
print('测试ip: %s' % ip)
try:
proxies = {
'https': ip
}

# ip测试网站
ip_url = 'https://www.ipip.net/'

# 使用有效与无效的代理对ip测试站点进行访问，若返回的结果为200则代表当前测试ip正常
response = requests.get(ip_url, headers=HEADERS, proxies=proxies, timeout=1)

if response.status_code == 200:
return ip

# 若ip代理无效则抛出异常
except Exception as e:
print(e)

# 使用代理爬取nba
def spider_nba(good_ip):
url = 'https://china.nba.com/'

proxies = {
'https': good_ip
}

response = requests.get(url, headers=HEADERS, proxies=proxies)
print(response.status_code)
print(response.text)

if __name__ == '__main__':
base_url = 'https://www.xicidaili.com/nn/{}'

for line in range(1, 3677):
ip_url = base_url.format(line)

response = get_index(ip_url)

ip_list = parse_index(response.text)
for ip in ip_list:
# print(ip)
good_ip = test_ip(ip)

if good_ip:
# 真是代理，开始测试
spider_nba(good_ip)

'''
selenium模块：
1.什么是selenium？
最初是一个自动化测试工具，可以使用它帮我们驱动浏览器自动去执行某些
自定义好的操作。例如在页面中执行JS代码、跳过登录验证.
2.为什么要使用selenium？
（1）.优点：
使用requests模块登录需要分析大量的复杂通信流程，使用selenium
可以轻松跳过登录验证。
（2）缺点:
浏览器会加载css.js.图片.视频...数据，爬虫效率相比requests模块要低
3.如何使用selenium？
下载selenium模块：
pip3 install -i https://pypi.tuna.tsinghua.edu.cn/simple selenium
下载:
http://npm.taobao.org/mirrors/chromedriver/2.38/

'''

from selenium import webdriver # 用来驱动浏览器的

# 调用得到一个动作链对象，破解滑动验证码的时候用的，可以拖动图片
from selenium.webdriver import ActionChains

# 按照什么方式查找属性，By.ID,By.CSS_SELECTOR,By.Class
from selenium.webdriver.common.by import By

from selenium.webdriver.common.keys import Keys # 键盘按键操作
from selenium.webdriver.support import expected_conditions as EC # 和下面WebDriverWait一起用的
from selenium.webdriver.support.wait import WebDriverWait # 等待页面加载某些元素

#selenium之第一次
from selenium import webdriver
import time
# 通过谷歌浏览器驱动打开谷歌浏览器
# webdriver.Chrome(r'chromedriver.exe的绝对路径')
# chrome = webdriver.Chrome(r'D:\python学习\chromedriver_win32\chromedriver.exe')

# chrome = webdriver.Chrome()
# # 若try出现异常
# try:
# # 往lyj博客主页发送get请求
# # chrome.get('https://www.cnblogs.com/lyj68/')
#
# # 参数1：驱动对象参数2：等待时间
# wait = WebDriverWait(chrome,10)
# #1.访问百度
# chrome.get('http://www.baidu.com')
# # 2.查找input输入框
#
# input_tag = wait.until(
# # 调用EC的presence_of_element_located()
# EC.presence_of_element_located((By.ID,'kw')))
# # 参数1：查找属性的方式 # 参数2：属性的名字
#
# #3.搜索一拳超人
# input_tag.send_keys("一拳超人")
# #4.按键回车键
# input_tag.send_keys(Keys.ENTER)
#
# time.sleep(3)
# finally:
# # 关闭浏览器
# chrome.close()

chrome = webdriver.Chrome()
# 若try出现异常
try:
# 往lyj博客主页发送get请求
# chrome.get('https://www.cnblogs.com/lyj68/')

# 参数1：驱动对象参数2：等待时间
wait = WebDriverWait(chrome,10)
#1.访问百度
chrome.get('https://www.jd.com/')
# 2.查找input输入框

input_tag = wait.until(
# 调用EC的presence_of_element_located()
EC.presence_of_element_located((By.ID,'key')))
# 参数1：查找属性的方式 # 参数2：属性的名字

#3.搜索一拳超人
input_tag.send_keys("唐诗三百首")
#4.根据class属性名称查找标签
search_button = wait.until(EC.presence_of_element_located((By.CLASS_NAME,'button')))
# 5.点击搜索按钮
search_button.click()
time.sleep(3)
finally:
# 关闭浏览器
chrome.close()

# from selenium import webdriver
# import time
#
# driver = webdriver.Chrome()
#
# try:
# driver.get('https://china.nba.com/')
# driver.implicitly_wait(10)
#
# news_tag = driver.find_element_by_class_name("nav-news")
# print(news_tag)
# print(news_tag.tag_name)
# time.sleep(10)
#
# finally:
# driver.close()

from selenium import webdriver
import time

driver = webdriver.Chrome()

try:
driver.get('https://www.baidu.com/')
driver.implicitly_wait(10)
# 1、find_element_by_link_text
# 通过链接文本去找
#发布
# send_tag = driver.find_element_by_link_text('登录')
# send_tag.click()
# time.sleep(1)

# 2、find_element_by_id
# 通过id去找
# 3、find_element_by_class_name
login_tag = driver.find_element_by_class_name('tang-pass-footerBarULogin pass-link')
# login_tag =

# 4、find_element_by_partial_link_text
# 5、find_element_by_name
# 6、find_element_by_css_selector
# 7、find_element_by_tag_name

finally:
driver.close()

# from selenium import webdriver # 用来驱动浏览器的
# import time
#
# '''
# 隐式等待
# '''
# # 获取驱动对象、
# driver = webdriver.Chrome()
#
# try:
# # 显式等待: 等待某个元素加载
# # 参数1: 驱动对象参数2: 等待时间
# # wait = WebDriverWait(chrome, 10)
#
# driver.get('https://china.nba.com/')
#
# # 隐式等待: 等待页面所有元素加载
# driver.implicitly_wait(10)
# news_tag = driver.find_element_by_class_name('nav-news')
# # 获取标签对象
# print(news_tag)
# # 获取标签的名字
# print(news_tag.tag_name)
#
#
# time.sleep(10)
#
# finally:
# driver.close()

from selenium import webdriver # 用来驱动浏览器的
import time

'''
===============所有方法===================
element是查找一个标签
elements是查找所有标签

1、find_element_by_link_text 通过链接文本去找
2、find_element_by_id 通过id去找
3、find_element_by_class_name
4、find_element_by_partial_link_text
5、find_element_by_name
6、find_element_by_css_selector
7、find_element_by_tag_name
'''
# 获取驱动对象、
driver = webdriver.Chrome()

try:

# 往百度发送请求
driver.get('https://www.baidu.com/')
driver.implicitly_wait(10)

# 1、find_element_by_link_text 通过链接文本去找
# 根据登录
# send_tag = driver.find_element_by_link_text('登录')
# send_tag.click()

# 2、find_element_by_partial_link_text 通过局部文本查找a标签
login_button = driver.find_element_by_partial_link_text('登')
login_button.click()
time.sleep(1)

# 3、find_element_by_class_name 根据class属性名查找
login_tag = driver.find_element_by_class_name('tang-pass-footerBarULogin')
login_tag.click()
time.sleep(1)

# 4、find_element_by_name 根据name属性查找
username = driver.find_element_by_name('userName')
username.send_keys('15622792660')
time.sleep(1)

# 5、find_element_by_id 通过id属性名查找
password = driver.find_element_by_id('TANGRAM__PSP_10__password')
password.send_keys('*******')
time.sleep(1)

# 6、find_element_by_css_selector 根据属性选择器查找
# 根据id查找登录按钮
login_submit = driver.find_element_by_css_selector('#TANGRAM__PSP_10__submit')
# driver.find_element_by_css_selector('.pass-button-submit')
login_submit.click()

# 7、find_element_by_tag_name 根据标签名称查找标签
div = driver.find_element_by_tag_name('div')
print(div.tag_name)

time.sleep(10)

finally:
driver.close()

转载于:https://www.cnblogs.com/xybds/p/11066179.html