本文目录导读：
使用 Shell 命令（最简单）
# 合并多个文件并去重 cat file1.txt file2.txt file3.txt | sort -u > merged_unique.txt # 如果不想排序，保持原始顺序 cat file1.txt file2.txt file3.txt | awk '!seen[$0]++' > merged_unique.txt
Python 脚本（最灵活）
基础版本
#!/usr/bin/env python3
import argparse
def merge_and_deduplicate(input_files, output_file):
"""合并多个文件并去重"""
seen = set()
with open(output_file, 'w', encoding='utf-8') as out:
for filename in input_files:
try:
with open(filename, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip() # 去除空白字符
if line and line not in seen: # 跳过空行和重复行
seen.add(line)
out.write(line + '\n')
except FileNotFoundError:
print(f"警告：文件 {filename} 不存在，已跳过")
print(f"处理完成！输出了 {len(seen)} 行唯一内容")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='合并文件并去重')
parser.add_argument('files', nargs='+', help='输入文件列表')
parser.add_argument('-o', '--output', default='merged.txt', help='输出文件名')
args = parser.parse_args()
merge_and_deduplicate(args.files, args.output)
增强版本（支持多种去重策略）
#!/usr/bin/env python3
import re
import argparse
class FileMerger:
def __init__(self, ignore_case=False, ignore_spaces=False):
self.ignore_case = ignore_case
self.ignore_spaces = ignore_spaces
self.seen = set()
def normalize(self, line):
"""标准化行内容用于比较"""
text = line.strip()
if self.ignore_case:
text = text.lower()
if self.ignore_spaces:
text = re.sub(r'\s+', ' ', text)
return text
def merge_files(self, input_files, output_file):
"""合并文件并去重"""
total_lines = 0
unique_count = 0
with open(output_file, 'w', encoding='utf-8') as out:
for filename in input_files:
try:
with open(filename, 'r', encoding='utf-8') as f:
for line in f:
total_lines += 1
key = self.normalize(line)
if line.strip() and key not in self.seen:
self.seen.add(key)
out.write(line)
unique_count += 1
except FileNotFoundError:
print(f"跳过不存在的文件: {filename}")
print(f"处理完成:")
print(f"- 总行数: {total_lines}")
print(f"- 唯一行数: {unique_count}")
print(f"- 重复行数: {total_lines - unique_count}")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='智能文件合并去重工具')
parser.add_argument('files', nargs='+', help='输入文件列表')
parser.add_argument('-o', '--output', default='merged.txt', help='输出文件')
parser.add_argument('-i', '--ignore-case', action='store_true', help='忽略大小写')
parser.add_argument('-s', '--ignore-spaces', action='store_true', help='忽略多余空格')
args = parser.parse_args()
merger = FileMerger(args.ignore_case, args.ignore_spaces)
merger.merge_files(args.files, args.output)
处理大文件（内存优化版）
#!/usr/bin/env python3
import hashlib
def merge_large_files(input_files, output_file, chunk_size=8192):
"""处理大文件，使用MD5减少内存占用"""
seen_hashes = set()
with open(output_file, 'w', encoding='utf-8') as out:
for filename in input_files:
try:
with open(filename, 'r', encoding='utf-8') as f:
for line in f:
line_content = line.strip()
if not line_content:
continue
# 使用MD5哈希来判断重复
line_hash = hashlib.md5(line_content.encode()).hexdigest()
if line_hash not in seen_hashes:
seen_hashes.add(line_hash)
out.write(line)
except FileNotFoundError:
print(f"跳过: {filename}")
# 使用示例
merge_large_files(['file1.txt', 'file2.txt'], 'merged.txt')
使用方式
Bash 命令
# 简单使用 python merge_files.py file1.txt file2.txt file3.txt # 指定输出文件 python merge_files.py -o result.txt file*.txt # 忽略大小写 python merge_files.py -i -o unique_result.txt data/*.txt
批量处理
#!/bin/bash
# 批量处理多个目录
for dir in data/*/; do
python merge_files.py "$dir"/*.txt -o "${dir}merged.txt"
done
高级特性
# 支持 CSV 和 TSV 格式
import csv
def merge_csv_files(input_files, output_file, delimiter=','):
"""合并CSV文件并去重"""
seen_rows = set()
with open(output_file, 'w', newline='', encoding='utf-8') as out:
writer = csv.writer(out, delimiter=delimiter)
for filename in input_files:
with open(filename, 'r', encoding='utf-8') as f:
reader = csv.reader(f, delimiter=delimiter)
for row in reader:
row_tuple = tuple(row)
if row_tuple not in seen_rows:
seen_rows.add(row_tuple)
writer.writerow(row)
选择建议
- 小文件：直接使用
sort -u或 shell 命令 - 中等文件：使用 Python 基础脚本
- 大文件：使用内存优化版或 MD5 哈希方案
- 需要复杂逻辑：使用增强版本，自定义去重规则
这样就能轻松实现文件内容的合并去重了！
