python3删除文本文件中的重复数据

删除文本中的重复数据
# 描述：删除数据集中的重复行。
def remove_duplicates_text(file_path):
    seen_lines = set()
    unique_lines = []

    # 读取文件内容
    with open(file_path, 'r', encoding='utf-8') as file:
        for line in file:
            line = line.strip()  # 去除行末的换行符和空白字符
            if line not in seen_lines:
                seen_lines.add(line)
                unique_lines.append(line)

    # 写回文件
    with open(file_path, 'w', encoding='utf-8') as file:
        for line in unique_lines:
            file.write(line + '\n')

# 使用示例
remove_duplicates_text('../openpyxl/output.txt')