tensorflow实现Word2Vec(找到目标英文单词的相近词)

最新推荐文章于 2021-10-23 17:04:15 发布

煊琰

最新推荐文章于 2021-10-23 17:04:15 发布

阅读量565

点赞数

本文介绍了一种从压缩文件中读取文本数据的方法，并将其转换为可用于机器学习模型的格式。通过使用Python标准库及TensorFlow，实现了文本数据的下载、解压、清洗与词汇表构建等功能。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

根据自己的理解写的读书笔记。

import collections
import math
import os
import random
import zipfile
import urllib
import numpy as np
import tensorflow as tf

#定义下载文本数据的函数
# url = 'http://mattmahoney.net/dc/'
#
# def maybe_download(filename,expected_bytes):
# if not os.path.exists(filename):
# filename,_ = urllib.request.urlretrieve(url + filename,filename)
# statinfo = os.stat(filename) #访问一个文件的详细信息。
# if statinfo.st_size == expected_bytes: #文件大小(以字节为单位)
# print('Found and verified(验证）',filename)
# else:
# print(statinfo.st_size)
# raise Exception('Failed to verify(验证）' + filename + 'Can you get to it with a browser(浏览器)?')
# return filename
#
# filename = maybe_download('text8.zip',31344016)

filename = './text8.zip'

我有几张阿里云幸运券分享给你，用券购买或者升级阿里云相应产品会有特惠惊喜哦！把想要买的产品的幸运券都领走吧！快下手，马上就要抢光了。

#解压文件，并将数据转化成单词的列表
def read_data(filename):
with zipfile.ZipFile(filename) as f:
#获得名字列表，读取成字符串，编码成'utf-8'，最后进行分割
data = tf.compat.as_str(f.read(f.namelist()[ 0 ])).split()
return data

words = read_data(filename)
# print('Data size',len(words))
# print(words)

#创建词汇表，将出现最多的50000个单词作为词汇表，放入字典中。
vocabulary_size = 50000

def build_dataset(words):
count = [[ 'UNK' ,- 1 ]]
count.extend(collections.Counter(words).most_common(vocabulary_size - 1 ))
# c=collections.Counter(words).most_common(10)
# print(c)
# count.extend(c)
# print(count) #[['UNK', -1], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764), ('in', 372201), ('a', 325873), ('to', 316376), ('zero', 264975), ('nine', 250430), ('two', 192644)]
dictionary = dict () #新建空字典
for word,_ in count:
dictionary[word] = len (dictionary)
# print(dictionary) #{'UNK': 0, 'the': 1, 'of': 2, 'and': 3, 'one': 4, 'in': 5, 'a': 6, 'to': 7, 'zero': 8, 'nine': 9, 'two': 10}
data = list ()
unk_count = 0 #未知单词数量
for word in words: #单词索引，不在字典中，则索引为0
if word in dictionary:
index = dictionary[word]
else :
index = 0
unk_count += 1
data.append(index)
count[ 0 ][ 1 ] = unk_count
reverse_dictionary = dict ( zip (dictionary.values(),dictionary.keys()))

return data,count,dictionary,reverse_dictionary

原文链接