统计SQuAD的词汇得到word2id 并把词都转成id的python代码

最新推荐文章于 2024-10-04 12:17:36 发布

转载最新推荐文章于 2024-10-04 12:17:36 发布 · 1.8k 阅读

NLP/DeepLearning 专栏收录该内容

319 篇文章

订阅专栏

本文介绍了一种从JSON文件中读取数据并进行词频统计的方法。通过对训练数据集的解析，提取了所有段落标题、上下文、问题及答案中的词汇，并统计了这些词汇出现的频率。此外，还展示了如何将文本转换为数值ID，以便于后续的机器学习应用。

import json
import collections

json_file = open("train-v1.1.json")
data = json.load(json_file)

all_words = []

for paragraphs_title in data["data"]:
    all_words.extend(paragraphs_title["title"].split())
    paragraphs = paragraphs_title["paragraphs"]
    for context_qas in paragraphs:
        all_words.extend(context_qas["context"].split())
        qas = context_qas["qas"]
        for answers_question in qas:
            answers = answers_question["answers"]
            all_words.extend(answers_question["question"].split())
            if len(answers)>1:
                print(answers)
            for answerstart_text in answers:
                all_words.extend(answerstart_text["text"].split())

counter = collections.Counter(all_words)
count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))

words, _ = list(zip(*count_pairs))
word_to_id = dict(zip(words, range(len(words))))

data_vec = []
for paragraphs_title in data["data"]:
    title = paragraphs_title["title"]
    paragraphs = paragraphs_title["paragraphs"]
    paragraphs_title = []
    data_vec.append(paragraphs_title)
    for context_qas in paragraphs:
        paragraphs_vec = []
        paragraphs_title.append(paragraphs_vec)
        context_vec = []
        questions_answers = []
        paragraphs_vec.append(context_vec)
        paragraphs_vec.append(questions_answers)
        for word in context_qas["context"].split():
            context_vec.append(word_to_id[word])
        qas = context_qas["qas"]
        for answers_question in qas:
            question_answer = []
            questions_answers.append(question_answer)
            question_vec = []
            answer_vec = []
            question_answer.append(question_vec)
            question_answer.append(answer_vec)
            answers = answers_question["answers"]
            for word in answers[0]["text"].split():
                answer_vec.append(word_to_id[word])
            for word in answers_question["question"].split():
                question_vec.append(word_to_id[word])
print("!")