requests bs4 datetime re json

最新推荐文章于 2025-09-11 10:39:32 发布

转载最新推荐文章于 2025-09-11 10:39:32 发布 · 144 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/leolaosao/p/9064013.html

文章标签：

#json #python

本文介绍了一个使用Python爬取特定新闻网站上文章详细信息的方法，包括标题、来源、发布时间、正文内容及责任编辑等，并获取了相关新闻的评论数量。通过requests、BeautifulSoup等库实现了网页内容的抓取与解析。

requests bs4 datetime re json

import re
import json
import requests
from bs4 import BeautifulSoup
from datetime import datetime
url='https://news.qq.com/a/20180520/008334.htm'
fuck={}
res=requests.get(url)
soup = BeautifulSoup(res.text, 'html.parser')
res.encoding='utf-8'
fuck['title']=soup.select('.hd h1')[0].text
fuck['editor']=soup.select('#QQeditor')[0].text
fuck['origin']=soup.select('.a_source')[0].text
fuck['time']=soup.select('.a_time')[0].text
#timesource = soup.select('.time-source')[0].contents[0].strip()
#time1=soup.select('.a_time')[0].contents[0].strip()
#fuck['dt'] = datetime.strptime(time1,'%Y年%m月%d日%H:%M')
fuck['article']='@'.join([p.text.strip() for p in soup.select('#Cnt-Main-Article-QQ p')[:]])#到哪一步
fuck

#原来的一个案例 2016
import requests
from bs4 import BeautifulSoup
from datetime import datetime
import re
import json

commenturl = 'http://comment5.news.sina.com.cn/page/info?version=1&format=js&channel=gn&newsid=comos-{}&\
group=&compress=0&ie=utf-8&oe=utf-8&page=1&\
page_size=20'

 
def getCommentCounts(newsurl):
    m = re.search('doc-i(.*).shtml', newsurl)
    newsid = m.group(1)
    comments = requests.get(commenturl.format(newsid))
    print(commenturl.format(newsid))
    jd = json.loads(comments.text.strip('var data='))
    return jd['result']['count']['total']


def getNewsDetail(newsurl):
    result = {}
    res = requests.get(newsurl)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text, 'html.parser')
    result['title'] = soup.select('#artibodyTitle')[0].text
    result['newssource'] = soup.select('.time-source span a')[0].text
    timesource = soup.select('.time-source')[0].contents[0].strip()
    result['dt'] = datetime.strptime(timesource,'%Y年%m月%d日%H:%M')
    result['article'] = '@'.join([p.text.strip() for p in soup.select('#artibody p')[:-1]])
    result['editor'] = soup.select('.article-editor')[0].text.strip('责任编辑：')
    result['comments'] = getCommentCounts(newsurl)
    return result

newsurl = 'http://news.sina.com.cn/c/nd/2016-12-18/doc-ifxytqax6457791.shtml' #只要这条代码中的newsurl具体赋值就可以了
print(getNewsDetail(newsurl))