python划分train、dev、test

转载已于 2022-03-09 08:47:54 修改 · 911 阅读

1 ·

CC 4.0 BY-SA版权

原文链接：https://blog.youkuaiyun.com/allan2222/article/details/99672868

文章标签：

#python #开发语言

于 2020-04-23 20:01:28 首次发布

python编程专栏收录该内容

143 篇文章

订阅专栏

本文介绍了一种使用Python实现的数据集划分方法，通过随机抽样将原始数据集分为训练集和验证测试集，比例为1:1。代码详细展示了如何读取原始文件，进行数据划分，并将结果写入新的文件中。

数据格式：一行为一句话。

import random

def split(all_list, shuffle=False, ratio=0.5):
    num = len(all_list)
    offset = int(num * ratio)
    if num == 0 or offset < 1:
        return [], all_list
    if shuffle:
        random.shuffle(all_list)
    train = all_list[:offset]
    dev_test = all_list[offset:]

    return train, dev_test

def write_split(film, train, dev_test):
    infilm = open(film, 'r', encoding='utf-8')
    trainfilm = open(train, 'w', encoding='utf-8')
    dev_testfilm = open(dev_test, 'w', encoding='utf-8')
    list = []
    for datas in infilm.readlines():
        datas = datas.replace('\n','')
        list.append(datas)
    traindatas, dev_testdatas = split(list, shuffle=True, ratio=0.5)
    for traindata in traindatas:
        trainfilm.write(traindata + '\n')
    for dev_testdata in dev_testdatas:
        dev_testfilm.write(dev_testdata + '\n')

    infilm.close()
    trainfilm.close()
    dev_testfilm.close()
if __name__ == '__main__':
    #分割train、dev、test
    film='./data/RenminNER/BIOES/test.txt'
    train = './data/RenminNER/BIOES/new_dev.txt'
    dev_test = './data/RenminNER/BIOES/new_test.txt'
    write_split(film, train, dev_test)