html显示request,request-html 简单爬虫

import asyncio

from requests_html import HTMLSession

url = 'http://www.xiaohuar.com/hua/'

session = HTMLSession( browser_args=[

'--no-sand',

'--disable-infobars'

'--user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'

],headless=False)

res = session.request(url=url,method='GET')

script = """

() => {

return {

width: document.documentElement.clientWidth,

height: document.documentElement.clientHeight,

deviceScaleFactor: window.devicePixelRatio,

}

}

"""

try:

res.html.render(keep_page = True)

async def main():

await res.html.page.waitFor(1000)

await res.html.page.setViewport({'width': 1366, 'height': 768})

url_list = await res.html.page.xpath('//div[@]/a')

for url in url_list:

url_link = await (await url.getProperty('href')).jsonValue()

print(url_link)

asyncio.get_event_loop().run_until_complete(main())

except Exception as e:

print(e)

finally:

session.close()