学习xpath,使用lxml+xpath提取内容。
使用xpath提取丁香园论坛的回复内容。
丁香园直通点:http://www.dxy.cn/bbs/thread/626626#626626 。
代码
def getHTMLText(url):
try:
r = requests.get(url)
r.raise_for_status()
return r.text
except:
print("getHTMLText error!")
return ""
def parsePage(text):
htmlInfo = {}
try:
html = etree.HTML(text)
names = html.xpath('//div[@class = "auth"]/a/text()')
contents = html.xpath("//td[@class = 'postbody']")
for i in range(len(names)):
htmlInfo[names[i]] = contents[i].xpath('string()').strip()
except:
print("parsePage error!")
return htmlInfo
def printHTMLInfo(htmlInfo):
tplt = "{:}\t\t{:}"
print(tplt.format("name", "content"))
for key, value in htmlInfo.items():
print(tplt.format(key, value), end='\n\n')
def main():
url = "http://www.dxy.cn/bbs/thread/626626"
text = getHTMLText(url)
htmlInfo = parsePage(text)
printHTMLInfo(htmlInfo)
main()
解析
转载参考:https://blog.youkuaiyun.com/wxl1999/article/details/89116195