# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
import re
import csv
import time
from pymongo import MongoClient
client = MongoClient('localhost',27017)
db = client.admin
db.authenticate("root", "root")
my_db = client.mydb
col = my_db.zwfw_html
link1 = 'http://www.hbzwfw.gov.cn/hbzw/sxcx/itemList/xz_index.do?webId=31&deptid='
def get_html(link):
Cookie = "PHPStat_First_Time_10000011=1480428327337; PHPStat_Cookie_Global_User_Id=_ck16112922052713449617789740328; PHPStat_Return_Time_10000011=1480428327337; PHPStat_Main_Website_10000011=_ck16112922052713449617789740328%7C10000011%7C%7C%7C; VISITED_COMPANY_CODE=%5B%22600064%22%5D; VISITED_STOCK_CODE=%5B%22600064%22%5D; seecookie=%5B600064%5D%3A%u5357%u4EAC%u9AD8%u79D1; _trs_uv=ke6m_532_iw3ksw7h; VISITED_MENU=%5B%228451%22%2C%229055%22%2C%229062%22%2C%229729%22%2C%228528%22%5D"
headers = {
'User-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
'Cookie': Cookie,
'Connection': 'keep-alive',
'Accept': '*/*',
'Accept-Language': 'zh-CN,zh;q=0.8',
'Host': 'query.sse.com.cn',
'Referer': 'http://www.sse.com.cn/assortment/stock/list/share/'
}
r = requests.get(link, headers=headers, timeout=10)
if 200 != r.status_code:
pass
html = r.text
return html
def get_id(link11):
movie_list = []
soup = BeautifulSoup(get_html(link=link11), "lxml")
div_list2 = re.findall(r'href="javascript:changebm(.*)" title=(.*)',soup.decode("utf8", "ignore"))
for i in range(len(div_list2)):
list1 = str(div_list2[i])
list2 = re.findall(r"[\u4e00-\u9fa5]+",list1)
list3 = re.findall(r"\d+",list1)
if len((list3[0]))==9:
movie_list.append(list3[0])
else:
pass
return movie_list
def get_shuju_1():
movie_list2 = get_id(link1)
print(movie_list2)
for n in range(len(movie_list2)):
try:
url_id = movie_list2[n]
except IndexError as e:
pass
for p in range(1,9):
url3 = "http://www.hbzwfw.gov.cn/hbzw/sxcx/itemList/xz_list.do?webId=31&deptid=%s&isone=&isonline=&type=&word=&page_num=%s" % (url_id,p)
soup3 = BeautifulSoup(get_html(link=url3), "lxml")
div_list2 = soup3.select('a')
if len(div_list2) != 0:
print("存在此页" + '' + url3)
div_list = soup3.select('div > div.r3_tit > a')
for m in range(len(div_list)):
div_list_2 = str(div_list[m]).replace('<a href="', '').replace('" target="_blank" title="', '').replace('\r\n\t\t\t\t\t\t\t\t\t</a>', '')
div_list_3 = re.sub(r'">[\u4e00-\u9fa5]+', '', str(div_list_2))
pattern = re.compile(r'^http(.*)html')
div_list_4 = re.findall(pattern, div_list_3)
div_list_5 = re.findall(r"[\u4e00-\u9fa5,(,),、,]+", div_list_3)
time.sleep(0.1)
movie_list3 = ('http'+str(div_list_4[0])+'html')
print(movie_list3, div_list_5[0])
save_contents(movie_list3,div_list_5[0])
dict_url = dict(zip(div_list_5[0],movie_list3))
col.insert(dict_url)
else:
None
def save_contents(shuju,title):
urlist = shuju
titleist = title
try:
with open("二级目录网址.csv",'a+',newline='') as f:
writer = csv.writer(f)
writer.writerows(zip([urlist], [titleist]))
except:
pass
if __name__ == '__main__':
get_html(link=link1)
get_shuju_1()