爬虫小技巧京东

最新推荐文章于 2023-12-28 11:03:59 发布

张航瑞

最新推荐文章于 2023-12-28 11:03:59 发布

阅读量2.6k

点赞数

分类专栏： python 文章标签：爬虫京东

本文链接：https://blog.youkuaiyun.com/qq_31899273/article/details/78607791

版权

python 专栏收录该内容

7 篇文章

订阅专栏

该博客介绍了如何使用Python的urllib、re和selenium库来爬取京东网站的图片。首先设置代理和User-Agent，然后通过正则表达式找到图片链接，最后将图片下载到本地。此外，还使用了selenium模拟浏览器行为，获取页面源码。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import urllib.request
import os
import random
import time
import re
from selenium import webdriver
#browser=webdriver.Firefox(executable_path='/home/renpu/pachong/geckodriver')
def url_open(url):
   '''iplist=['218.20.55.185:9999','39.104.13.145:8080']
   a=random.choice(iplist)
   print(a)
   proxy_support=urllib.request.ProxyHandler({'http':a})
   opener = urllib.request.build_opener(proxy_support)
   opener.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0')]
   urllib.request.install_opener(opener)'''
   headers = {'User-Agent':'Mozilla/5.0 (X11; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0'}
   req = urllib.request.Request(url=url, headers=headers)
   respone = urllib.request.urlopen(req)
   return respone.read()
def url_webdriver(url,browser):
   browser.get(url)
   html=browser.page_source.encode().decode('utf-8')
   with open('jingdongweiber.txt','w') as fjingdong:
       fjingdong.write(html)
   img=r'(?:(?://.{3,150}?)\.jpg)'
   imglist=re.findall(img,html)
   img_name=[]
   for i in imglist:
       b = i.split('/')[-1]
       img_name.append(b)
   img_name=list(set(img_name)) # 删除相同文件名
   print(img_name)
   for j in img_name:
       print(j)
'''def get_page(url):
   html = url_open(url).decode('utf-8')
   a=html.find('current-comment-page')+23
   b=html.find(']', a)
   return html[a:b]'''

def find_imgs(url):
   html = url_open(url).decode('ISO-8859-1')
   '''html=url_webdriver(url,browser).decode('utf-8')'''
   print(html)
   img_addrs = []
   print(url)
   a = html.find('img data-src=')
   print(a)
   print('找图片')
   while a != -1:
       b=html.find('.png',a,a+255)
       #print(b)
       if b==-1:
           b=html.find('.jpg',a,a+255)
       if b != -1:
           img_addrs.append(html[a+14:b+4])
       else:
           b = a+14
       a=html.find('img data-src=',b)
   a=html.find('img src=')
   while a != -1:
       b=html.find('.png',a,a+255)
       #print(b)
       if b==-1:
           b=html.find('.jpg',a,a+255)
       if b != -1:
           img_addrs.append(html[a+9:b+4])
       else:
           b = a+9
       a=html.find('img src=',b)
   a=html.find('img data-lazy-img=')
   while a != -1:
       b=html.find('.png',a,a+255)
       #print(b)
       if b==-1:
           b=html.find('.jpg',a,a+255)
       if b != -1:
           img_addrs.append(html[a+19:b+4])
       else:
           b = a+19
       a=html.find('img data-lazy-img',b)
   for each in img_addrs:
       print(each)
   return img_addrs

def save_imgs(folder, img_addrs):
   for each in img_addrs:
       filename = each.split('/')[-1]
       each = 'http:'+each
       print(each)
       with open(filename,'wb') as f:
           img = url_open(each)
           f.write(img)

def download_mm(folder='OOXX'):
   if os.path.exists(folder)==False:
       os.mkdir(folder)
       print('创建成功')
   os.chdir(folder)
   url='https://wt.jd.com'
   '''page_num=int(get_page(url))
   print(page_num)
   for i in range(page):
       page_num -= i
       print(page_num)
       page_url = url+'page-'+str(page_num)+'#comments'
       print(page_url)'''
   img_addrs = find_imgs(url)
   save_imgs(folder,img_addrs)
if __name__ == '__main__':
   #download_mm()
   url='https://wt.jd.com'
   browser=webdriver.Firefox(executable_path='/home/renpu/pachong/geckodriver')
   url_webdriver(url,browser)