位置编码的一种实现

位置编码的一种实现

import torch
import math
if __name__ == '__main__':
    embedding_dim = 12
    num_embeddings = 12
    
    half_dim = embedding_dim // 2
    emb = math.log(10000) / (half_dim - 1)  # 10000的i次幂数量级太大，转换为log
    print('emb=',emb)
    emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb)
    print('emb=', emb)

    emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0)  # 生成二维向量
    print('emb=', emb)
    emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1)  # 按列拼接5列变10列
    print('emb=', emb)

import math
import torch
def make_pos(max_len, dim):
	pe = torch.zeros(max_len, dim)                                              # [max_len, dim]
	position = torch.arange(0, max_len).unsqueeze(1)                            # [max_len, 1]
	div_term = torch.exp((torch.arange(0, dim, 2, dtype=torch.float) *
                              -(math.log(10000.0) / dim)))
	pe[:, 0::2] = torch.sin(position.float() * div_term)                        # 从0开始，间隔两个取值
	pe[:, 1::2] = torch.cos(position.float() * div_term)                        # 从1开始，间隔两个取值
	pe = pe.unsqueeze(0)                                                        # [1, max_len, dim]