1 .VQAFeatureDataset
此类是ReGat项目对torch自带的from torch.utils.data import Dataset的重写,是模型运行的时候训练集和测试集的加载，加载的数据是模型forward函数的参数。如下：
regat.forward():
def forward(self, v, b, q, implicit_pos_emb, sem_adj_matrix,
spa_adj_matrix, labels):
"""Forward
v: [batch, num_objs, obj_dim]
b: [batch, num_objs, b_dim]
q: [batch_size, seq_length]
pos: [batch_size, num_objs, nongt_dim, emb_dim]
sem_adj_matrix: [batch_size, num_objs, num_objs, num_edge_labels]
spa_adj_matrix: [batch_size, num_objs, num_objs, num_edge_labels]
return: logits, not probs
"""
w_emb = self.w_emb(q) #问题嵌入
q_emb_seq = self.q_emb.forward_all(w_emb) # [batch, q_len, q_dim]
q_emb_self_att = self.q_att(q_emb_seq) #添加自注意力信息
# [batch_size, num_rois, out_dim]
if self.relation_type == "semantic": #如果关系类型是语义
v_emb = self.v_relation.forward(v, sem_adj_matrix, q_emb_self_att)
elif self.relation_type == "spatial": #如果关系类型是空间
v_emb = self.v_relation.forward(v, spa_adj_matrix, q_emb_self_att)
else: # implicit #如果是隐式关系
v_emb = self.v_relation.forward(v, implicit_pos_emb,
q_emb_self_att)
if self.fusion == "ban": #融合模型1
joint_emb, att = self.joint_embedding(v_emb, q_emb_seq, b)
elif self.fusion == "butd": #融合模型2
q_emb = self.q_emb(w_emb) # [batch, q_dim]
joint_emb, att = self.joint_embedding(v_emb, q_emb)
else: # mutan融合模型3
joint_emb, att = self.joint_embedding(v_emb, q_emb_self_att)
if self.classifier: #分类模型
logits = self.classifier(joint_emb)
else:
logits = joint_emb
return logits, att
VQAFeatureDataset
self中的变量
| 变量名 | 含义 | 来源 | 值 |
|---|---|---|---|
| self.ans2label | 单词-索引表示：字典 | trainval_ans2label.pkl | {‘net’: 0, ‘pitcher’: 1, ‘orange’: 2, ‘yes’: 3, ‘white’: 4,… |
| self.label2ans | 索引-单词表示：列表 | trainval_label2ans.pkl | [‘net’, ‘pitcher’, ‘orange’, ‘yes’, ‘white’,. |
| self.num_ans_candidates | 答案单词候选数：int | len(self.ans2label) | 3129 |
| self.img_id2idx | 图像id-索引表示：字典 | imgid2idx.pkl | {218224: 0, 306670: 1, 208663: 2, 225177: 3, 467257: 4, . |
| self.features | 图像特征：Tensor | hf.get(‘image_features’) | tensor[40504,36,2048] |
| self.normalized_bb | 标准化区域边界框空间位置：Tensor | hf.get(‘spatial_features’) | tensor([40504, 36, 4]) |
| self.bb | 区域边界框位置：Tensor | hf.get(‘image_bb’) | tensor[40504,36,4] |
| self.semantic_adj_matrix | 语义形容词矩阵 | 如果在hf的键中：hf.get(‘semantic_adj_matrix’) ，不在=None | |
| self.spatial_adj_matrix | 空间形容词矩阵 | 如果在hf的键中：hf.get(‘image_adj_matrix’) ，不在=None | |
| self.pos_boxes | None | None | None |
| self.entries | 数据条目，items：list | _load_dataset(dataroot, name, self.img_id2idx,self.label2ans) | 长度214354 |
| self.nongt_dim | self.nongt_dim = nongt_dim | 36 | |
| self.emb_dim | 位置嵌入维度 | pos_emb_dim | 64 |
| self.v_dim | 图像特征嵌入维度 | self.features.size(1 if self.adaptive else 2) | 2048 |
| self.s_dim | 方向维度 | self.normalized_bb.size(1 if self.adaptive else 2) | 6 |
class VQAFeatureDataset(Dataset):
def __init__(self, name, dictionary, relation_type, dataroot='data',
adaptive=False, pos_emb_dim=64, nongt_dim=36):
super(VQAFeatureDataset, self).__init__()
assert name in ['train', 'val', 'test-dev2015', 'test2015']
# 加载annotations.json的预处理后的pkl文件
ans2label_path = os.path.join(dataroot, 'cache', 'trainval_ans2label.pkl')
label2ans_path = os.path.join(dataroot, 'cache', 'trainval_label2ans.pkl')
self.ans2label = pickle.load(open(ans2label_path, 'rb')) #形如{'w1':1,'w2':2,...,'w3129':3129}
self.label2ans = pickle.load(open(label2ans_path, 'rb'))#['w1','w2',...,'w3129']
self.num_ans_candidates = len(self.ans2label) #候选答案单词数目=3129
self.dictionary = dictionary #词典，包含19901个单词，键：idx2word#['w1','w2',...,'w19901'],word2idx{'w1':1,'w2':2,...,'w19901':19901}，padding_idx=19901,ntoken=19901
self.relation_type = relation_type
self.adaptive = adaptive #数据集是否是自适应的10-100个区域的
prefix = '36'
if 'test' in name:
prefix = '_36'
#加载hdf5文件目录
h5_dataroot = dataroot+"/Bottom-up-features-adaptive"\
if self.adaptive else dataroot+"/Bottom-up-features-fixed"
imgid_dataroot = dataroot+"/imgids" #加载图像ids文件：
#加载imgid2idx.pkl文件，保存再self.img_id2idx里：{id1:1,id2:2,...,id40504:40504}
self.img_id2idx = pickle.load(open(os.path.join(imgid_dataroot, '%s%s_imgid2idx.pkl' %
(name, '' if self.adaptive else prefix)), 'rb'))
#加载hdf5文件
h5_path = os.path.join(h5_dataroot, '%s%s.hdf5' %
(name, '' if self.adaptive else prefix))
print('loading features from h5 file %s' % h5_path)
with h5py.File(h5_path, 'r') as hf:
# self.features = np.array(hf.get('image_features'))
self.features = np.array(hf.get('image_features'),dtype='float32')
self.normalized_bb = np.array(hf.get('spatial_features'),dtype='float32')
self.bb = np.array(hf.get('image_bb'),dtype='float32')
print("hdf5数据加载成功！")
if "semantic_adj_matrix" in hf.keys() \
and self.relation_type == "semantic":
self.semantic_adj_matrix = np.array(
hf.get('semantic_adj_matrix'))
print("Loaded semantic adj matrix from file...",
self.semantic_adj_matrix.shape)
else:
self.semantic_adj_matrix = None
print("Setting semantic adj matrix to None...")
if "image_adj_matrix" in hf.keys()\
and self.relation_type == "spatial":
self.spatial_adj_matrix = np.array(hf.get('image_adj_matrix'))#从文件加载空间的形容词矩阵
print("Loaded spatial adj matrix from file...",
self.spatial_adj_matrix.shape)
else:
self.spatial_adj_matrix = None
print("Setting spatial adj matrix to None...")
self.pos_boxes = None
if self.adaptive:
self.pos_boxes = np.array(hf.get('pos_boxes'),dtype='float32')
self.entries = _load_dataset(dataroot, name, self.img_id2idx,
self.label2ans)
self.tokenize()
print("数据加载成功！")
self.tensorize()
self.nongt_dim = nongt_dim
self.emb_dim = pos_emb_dim
self.v_dim = self.features.size(1 if self.adaptive else 2)
self.s_dim = self.normalized_bb.size(1 if self.adaptive else 2)
def tokenize(self, max_length=14):
"""Tokenizes the questions.
This will add q_token in each entry of the dataset.
-1 represent nil, and should be treated as padding_idx in embedding
"""
for entry in self.entries:
tokens = self.dictionary.tokenize(entry['question'], False)
tokens = tokens[:max_length]
if len(tokens) < max_length:
# Note here we pad to the back of the sentence
padding = [self.dictionary.padding_idx] * \
(max_length - len(tokens))
tokens = tokens + padding
utils.assert_eq(len(tokens), max_length)
entry['q_token'] = tokens
def tensorize(self):
self.features = torch.from_numpy(self.features)
self.normalized_bb = torch.from_numpy(self.normalized_bb)
self.bb = torch.from_numpy(self.bb)
if self.semantic_adj_matrix is not None:
self.semantic_adj_matrix = torch.from_numpy(
self.semantic_adj_matrix).double()
if self.spatial_adj_matrix is not None:
self.spatial_adj_matrix = torch.from_numpy(
self.spatial_adj_matrix).double()
if self.pos_boxes is not None:
self.pos_boxes = torch.from_numpy(self.pos_boxes)
for entry in self.entries:
question = torch.from_numpy(np.array(entry['q_token']))
entry['q_token'] = question
answer = entry['answer']
if answer is not None:
labels = np.array(answer['labels'])
scores = np.array(answer['scores'], dtype=np.float32)
if len(labels):
labels = torch.from_numpy(labels)
scores = torch.from_numpy(scores)
entry['answer']['labels'] = labels
entry['answer']['scores'] = scores
else:
entry['answer']['labels'] = None
entry['answer']['scores'] = None
def __getitem__(self, index):
entry = self.entries[index]
raw_question = entry["question"]
image_id = entry["image_id"]
question = entry['q_token']
question_id = entry['question_id']
if self.spatial_adj_matrix is not None:
spatial_adj_matrix = self.spatial_adj_matrix[entry["image"]]
else:
spatial_adj_matrix = torch.zeros(1).double()
if self.semantic_adj_matrix is not None:
semantic_adj_matrix = self.semantic_adj_matrix[entry["image"]]
else:
semantic_adj_matrix = torch.zeros(1).double()
if not self.adaptive:
# fixed number of bounding boxes
features = self.features[entry['image']]
normalized_bb = self.normalized_bb[entry['image']]
bb = self.bb[entry["image"]]
else:
features = self.features[
self.pos_boxes[
entry['image']][0]:self.pos_boxes[entry['image']][1], :]
normalized_bb = self.normalized_bb[
self.pos_boxes[
entry['image']][0]:self.pos_boxes[entry['image']][1], :]
bb = self.bb[
self.pos_boxes[
entry['image']][0]:self.pos_boxes[entry['image']][1], :]
answer = entry['answer']
if answer is not None:
labels = answer['labels']
scores = answer['scores']
target = torch.zeros(self.num_ans_candidates)
if labels is not None:
target.scatter_(0, labels, scores)
return features, normalized_bb, question, target,\
question_id, image_id, bb, spatial_adj_matrix,\
semantic_adj_matrix
else:
return features, normalized_bb, question, question_id,\
question_id, image_id, bb, spatial_adj_matrix,\
semantic_adj_matrix
def __len__(self):
return len(self.entries)
entries
entries是数据的条目，类型是list，共214354条数据，每条数据是一个字典。
每条数据如下：
| 键值 | 含义 | 值 |
|---|---|---|
| question_id | 问题id | 42000 |
| image_id | 图像id | 42 |
| image | 图像 | 37244 |
| question | 问题文字表示 | ‘What color are the gym shoes?’ |
| answer | 答案：label,score | {‘labels’: tensor([ 4, 1594], dtype=torch.int32), ‘scores’: tensor([1.0000, 0.3000])} |
| q_token | 问题索引向量表示 | tensor([ 0, 10, 68, 11, 2618, 225, 19901, 19901, 19901, 19901,19901, 19901, 19901, 19901], dtype=torch.int32) |
2. 模型需要传入的getitem数据返回 ：如果是固定36个区域
| 变量名 | 来源 |
|---|---|
| features | self.features[entry[‘image’]]，此处的image是位置 |
| normalized_bb | self.normalized_bb[entry[‘image’]] |
| question | entry[‘q_token’] |
| target | scatter_(0, labels, scores) |
| question_id | entry[‘question_id’] |
| image_id | entry[“image_id”] |
| bb | self.bb[entry[“image”]] |
| spatial_adj_matrix | self.spatial_adj_matrix[entry[“image”]] |
| semantic_adj_matrix | self.semantic_adj_matrix[entry[“image”]] |
403
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
