import requests
from bs4 import BeautifulSoup
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 UBrowser/6.1.2107.204 Safari/537.36'
}
class QSBK:
def get_url(self,url):
response=requests.get(url,headers=header)
return response
def writefile(filename,data):
with open(filename,'a')as f:
f.write(data+'\n')
url='https://www.qiushibaike.com/8hr/page/1/'
soup_1=BeautifulSoup(requests.get(url,headers=header).text,'html.parser')
all_page=soup_1.find_all('span',class_='page-numbers')[-1].text
for i in range(1,int(all_page)+1):
joker=QSBK()
html=joker.get_url('https://www.qiushibaike.com/8hr/page/'+str(i)+'/')
soup=BeautifulSoup(html.text,'html.parser')
all_text=soup.find_all(class_='content')
for a in all_text:
text=a.get_text()
print(text)
writefile('QSBK.txt',text)