爬虫爬html数据库,一次用爬虫爬取数据库的小记

一、需求

所要爬取的数据库是CKB数据库

image.png

点击进入Genes页面

image.png

蓝色的gene名的部分是数据库公布的，可获得的信息

随便点击一个，比如ABL1

image.png

可以看到对应的页面下，存在箭头所指的两个表格

而，需求就是将所有蓝色部分的gene所对应的这两个表格爬取下来

二、所使用的技术路线

1.路线

这里所使用的方法是requests-BeautifulSoup4

requests.get()获取页面内容

BeautifulSoup4解析页面

2.分析

首先看一下下图的网页源代码

image.png

image.png

先简单的找到比如说ABL1相关的信息

image.png

可以看出来ABL1是由

和两个标签包围的

其他的基因也是这样

比如

image.png

image.png

可以看出来class信息是一样的，而href信息可以获得每个gene对用的页面链接信息

因此，可以用soup.find_all,将name设为"a",attrs设为"btn btn-default btn-gene btn-block"来获得所有gene的信息

将gene 和对应的链接以字典形式存储

然后，可以看一下某个gene页面，比如ABL1

先看第一个表，Gene Variants

image.png

第二个表，Gene Level Evidence

image.png

看一下源代码

image.png

先找到和第一个表格相关的地方

image.png

基本上，可以看出来每个Variant对应的信息被包括在tbody以及两个子标签tr和td中

同理找一下第二个表格相关的信息即可

三、全代码

import requests

import bs4

from bs4 import BeautifulSoup

def getHTMLText(url):

try:

r = requests.get(url,timeout = 30)

r.raise_for_status()

r.encoding = r.apparent_encoding

return r.text

except:

return ""

def get_gene_id(url):

html = getHTMLText(url)

soup = BeautifulSoup(html,'html.parser')

gene_id_dict = {}

for a in soup.find_all(name = "a",attrs = "btn btn-default btn-gene btn-block"):

gene_name = a.string.replace("\n","").replace(" ","")

ID = a.attrs['href']

gene_id = "https://ckb.jax.org" + ID

gene_id_dict[gene_name] = gene_id

return gene_id_dict

def gene_variant_list(alist,html):

soup = BeautifulSoup(html,'html.parser')

for tr in soup.find_all("tbody")[1].children:

if isinstance(tr,bs4.element.Tag):

var_des = ''

tds = tr("td")

variant = tds[0].a.string.replace(" ",'').replace("\n",'')

for string in tds[3].strings:

var_des += string

var_des = var_des.replace("\n","")

alist.append([variant,tds[1].string,tds[2].string,var_des,tds[4].string])

def gene_level_evidence_list(blist,html):

soup = BeautifulSoup(html,'html.parser')

num = 0

for name in soup.find_all(name = "a",attrs = "btn btn-default btn-wrap btn-therapy"):

num += 1

count = num*8

eight_list = []

for k in range(count):

string_extract = ""

if (k+1)%8 == 1:

if k != 0 :

blist.append([eight_list[0],eight_list[1],eight_list[2],eight_list[3],eight_list[4],eight_list[5],eight_list[6],eight_list[7]])

eight_list = []

for i in soup.find_all("td")[-count:][k]:

a = i.string.replace("\n","") #string extract

aL = a.split(" ")

aL = list(filter(None,aL))

a = " ".join(aL)

string_extract += " "+ a #delete blank and get string

eight_list.append(string_extract)

if eight_list:

blist.append([eight_list[0],eight_list[1],eight_list[2],eight_list[3],eight_list[4],eight_list[5],eight_list[6],eight_list[7]])

def print_gene_variant_list(alist,gene):

num = len(alist)

for i in range (num):

u = alist[i]

print (gene,u[0],u[1],u[2],u[3],u[4],sep="\t")

def print_gene_level_evidence_list(blist,gene):

num = len(blist)

if num :

for i in range (num):

m = blist[i]

print (gene,m[0],m[1],m[2],m[3],m[4],m[5],m[6],m[7],sep = "\t")

else:

if num == 0:

print(gene)

def main():

url = "https://ckb.jax.org/gene/grid"

gDict = get_gene_id(url)

print("{:^10}\t{:^6}\t{:^10}\t{:^10}\t{:^10}\t{:^10}".format("Gene_Name","Variant","Impact","Protein_Effect","Variant_Description","Associated_with_drug_resistance"))

for name,ID in gDict.items():

ainfo = []

url = ID

gene = name

html = getHTMLText(url)

gene_variant_list(ainfo,html)

print_gene_variant_list(ainfo,gene)

print("{:^10}\t{:^10}\t{:^10}\t{:^10}\t{:^10}\t{:^10}\t{:^10}\t{:^10}\t{:^10}".format("Gene_Name","Molecular_Profile","Indication/Tumour_type","Response_Type","Therapy_Name","Approval_Status","Evidence_Type","Efficacy_Evidence","References"))

for name,ID in gDict.items():

binfo = []

url = ID

gene = name

html = getHTMLText(url)

gene_level_evidence_list(binfo,html)

print_gene_level_evidence_list(binfo,gene)

main()

cat craw_3.xls |tr -d "\r" >new.xls

四、结果展示

image.png

image.png