本文目录导读：
Python 实现（推荐）
基础去重（保留首次出现顺序）
def remove_duplicates(input_file, output_file):
seen = set()
with open(input_file, 'r', encoding='utf-8') as fin, \
open(output_file, 'w', encoding='utf-8') as fout:
for line in fin:
line_stripped = line.rstrip('\n')
if line_stripped not in seen:
seen.add(line_stripped)
fout.write(line)
print(f"去重完成！原始行数: 未知, 去重后行数: {len(seen)}")
# 使用示例
remove_duplicates('input.txt', 'output.txt')
统计信息版
def dedup_with_stats(input_file, output_file):
seen = set()
duplicate_count = 0
total_count = 0
with open(input_file, 'r', encoding='utf-8') as fin, \
open(output_file, 'w', encoding='utf-8') as fout:
for line in fin:
total_count += 1
line_stripped = line.rstrip('\n')
if line_stripped not in seen:
seen.add(line_stripped)
fout.write(line)
else:
duplicate_count += 1
print(f"总行数: {total_count}")
print(f"重复行数: {duplicate_count}")
print(f"去重后行数: {len(seen)}")
print(f"去重完成！")
dedup_with_stats('input.txt', 'output.txt')
Shell 命令实现
基础去重（排序后去重）
# 排序后去重（不保留原始顺序） sort input.txt | uniq > output.txt # 统计去重信息 sort input.txt | uniq -c > output_with_count.txt
保留顺序去重
# 使用 awk 保留第一次出现的顺序 awk '!seen[$0]++' input.txt > output.txt # 或者使用 cat -n 配合 sort cat -n input.txt | sort -k2 -k1n | uniq -f1 | sort -nk1 | cut -f2-
批量文件处理
import os
from pathlib import Path
def batch_dedup(folder_path, extension='.txt'):
"""批量处理文件夹中的所有文件"""
folder = Path(folder_path)
for file_path in folder.glob(f'*{extension}'):
print(f"处理文件: {file_path.name}")
output_path = file_path.parent / f"{file_path.stem}_dedup{extension}"
remove_duplicates(str(file_path), str(output_path))
# 使用示例
batch_dedup('./data', '.txt')
高级去重选项
def advanced_dedup(input_file, output_file, ignore_case=False,
strip_whitespace=True, remove_empty=True):
"""高级去重，支持更多选项"""
seen = set()
with open(input_file, 'r', encoding='utf-8') as fin, \
open(output_file, 'w', encoding='utf-8') as fout:
for line in fin:
# 处理空白字符
if strip_whitespace:
line_processed = line.strip()
else:
line_processed = line.rstrip('\n')
# 忽略空行
if remove_empty and not line_processed:
continue
# 忽略大小写
if ignore_case:
line_processed = line_processed.lower()
if line_processed not in seen:
seen.add(line_processed)
fout.write(line)
print(f"去重完成！共处理 {len(seen)} 条唯一记录")
# 使用示例
advanced_dedup('input.txt', 'output.txt', ignore_case=True)
大文件处理（内存优化）
def dedup_large_file(input_file, output_file, chunk_size=10000):
"""处理超大文件的分块去重"""
import tempfile
import heapq
# 分块排序
temp_files = []
with open(input_file, 'r', encoding='utf-8') as fin:
chunk = []
for i, line in enumerate(fin, 1):
chunk.append(line)
if i % chunk_size == 0:
chunk.sort()
temp = tempfile.NamedTemporaryFile(mode='w', delete=False,
encoding='utf-8')
temp.writelines(chunk)
temp.close()
temp_files.append(temp.name)
chunk = []
if chunk:
chunk.sort()
temp = tempfile.NamedTemporaryFile(mode='w', delete=False,
encoding='utf-8')
temp.writelines(chunk)
temp.close()
temp_files.append(temp.name)
# 合并去重
files = [open(f, 'r', encoding='utf-8') for f in temp_files]
with open(output_file, 'w', encoding='utf-8') as fout:
last_line = None
for line in heapq.merge(*files):
if line != last_line:
fout.write(line)
last_line = line
# 清理临时文件
for f in files:
f.close()
for f in temp_files:
os.unlink(f)
# 使用示例
dedup_large_file('large_file.txt', 'output.txt')
注意事项
- 编码问题：根据文件实际编码调整
encoding参数 - 内存管理：大文件建议使用分块处理
- 换行符：不同操作系统换行符不同，Python 会自动处理
- 性能考虑：
- 小文件使用 Python set 方法最快
- 大文件使用分块排序合并更省内存
选择哪种方法取决于你的具体需求：
- 文件大小
- 是否要保留原始顺序
- 是否需要统计信息
- 处理文件的批量程度
