本文目录导读：
基于字符相似度的模糊对比
import difflib
def fuzzy_compare(text1, text2, threshold=0.6):
"""
模糊对比两个文本内容
"""
# 计算相似度
similarity = difflib.SequenceMatcher(None, text1, text2).ratio()
# 获取不同之处
diff = list(difflib.unified_diff(
text1.splitlines(keepends=True),
text2.splitlines(keepends=True),
n=0
))
return {
'similarity': similarity,
'is_match': similarity >= threshold,
'diffs': diff
}
# 使用示例
with open('file1.txt', 'r') as f1, open('file2.txt', 'r') as f2:
result = fuzzy_compare(f1.read(), f2.read())
print(f"相似度: {result['similarity']*100:.2f}%")
基于行级别的模糊匹配
def line_based_fuzzy_match(file1, file2, threshold=0.8):
"""
逐行进行模糊匹配
"""
with open(file1, 'r', encoding='utf-8') as f1, \
open(file2, 'r', encoding='utf-8') as f2:
lines1 = f1.read().splitlines()
lines2 = f2.read().splitlines()
matches = []
for i, line1 in enumerate(lines1):
for j, line2 in enumerate(lines2):
similarity = difflib.SequenceMatcher(None, line1, line2).ratio()
if similarity >= threshold:
matches.append({
'file1_line': i + 1,
'file2_line': j + 1,
'similarity': similarity,
'content1': line1,
'content2': line2
})
return matches
使用N-gram进行模糊对比
from collections import defaultdict
class NGramMatcher:
def __init__(self, n=3):
self.n = n
def extract_ngrams(self, text):
"""提取N-gram特征"""
words = text.lower().split()
ngrams = set()
for i in range(len(words) - self.n + 1):
ngram = ' '.join(words[i:i+self.n])
ngrams.add(ngram)
return ngrams
def compare(self, text1, text2):
"""基于N-gram的相似度对比"""
ngrams1 = self.extract_ngrams(text1)
ngrams2 = self.extract_ngrams(text2)
if not ngrams1 or not ngrams2:
return 0
intersection = ngrams1 & ngrams2
union = ngrams1 | ngrams2
return len(intersection) / len(union)
# 使用示例
matcher = NGramMatcher(n=3)
with open('file1.txt') as f1, open('file2.txt') as f2:
similarity = matcher.compare(f1.read(), f2.read())
print(f"N-gram相似度: {similarity*100:.2f}%")
综合模糊对比工具
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
class ComprehensiveFuzzyComparator:
def __init__(self):
self.methods = {
'exact': self.exact_match,
'word': self.word_similarity,
'ngram': self.ngram_similarity,
'cosine': self.cosine_similarity
}
def preprocess(self, text):
"""文本预处理"""
# 去除标点符号和空格
text = re.sub(r'[^\w\s]', ' ', text)
text = re.sub(r'\s+', ' ', text)
return text.strip().lower()
def exact_match(self, text1, text2):
"""精确匹配"""
return text1 == text2
def word_similarity(self, text1, text2):
"""基于词集的相似度"""
words1 = set(text1.split())
words2 = set(text2.split())
if not words1 or not words2:
return 0
intersection = len(words1 & words2)
union = len(words1 | words2)
return intersection / union
def ngram_similarity(self, text1, text2, n=3):
"""N-gram相似度"""
def get_ngrams(text, n):
chars = list(text.replace(' ', ''))
return set(''.join(chars[i:i+n]) for i in range(len(chars)-n+1))
ngrams1 = get_ngrams(text1, n)
ngrams2 = get_ngrams(text2, n)
if not ngrams1 or not ngrams2:
return 0
intersection = len(ngrams1 & ngrams2)
union = len(ngrams1 | ngrams2)
return intersection / union
def cosine_similarity(self, text1, text2):
"""TF-IDF余弦相似度"""
vectorizer = TfidfVectorizer()
try:
tfidf_matrix = vectorizer.fit_transform([text1, text2])
similarity = (tfidf_matrix * tfidf_matrix.T).A[0, 1]
return similarity
except:
return 0
def compare_files(self, file1_path, file2_path, methods=None):
"""对比两个文件"""
with open(file1_path, 'r', encoding='utf-8') as f1, \
open(file2_path, 'r', encoding='utf-8') as f2:
text1 = self.preprocess(f1.read())
text2 = self.preprocess(f2.read())
results = {}
methods = methods or ['word', 'ngram', 'cosine']
for method in methods:
if method in self.methods:
results[method] = self.methods[method](text1, text2)
# 计算综合分数
if results:
results['combined'] = sum(results.values()) / len(results)
return results
# 使用示例
comparator = ComprehensiveFuzzyComparator()
results = comparator.compare_files('file1.txt', 'file2.txt')
for method, score in results.items():
if isinstance(score, float):
print(f"{method}: {score*100:.2f}%")
else:
print(f"{method}: {score}")
实用的模糊对比脚本
#!/usr/bin/env python3
"""模糊对比工具
"""
import argparse
import sys
from pathlib import Path
def main():
parser = argparse.ArgumentParser(description='文件内容模糊对比工具')
parser.add_argument('file1', help='第一个文件')
parser.add_argument('file2', help='第二个文件')
parser.add_argument('--method', choices=['exact', 'fuzzy', 'line', 'ngram', 'all'],
default='fuzzy', help='对比方法')
parser.add_argument('--threshold', type=float, default=0.6, help='相似度阈值')
args = parser.parse_args()
# 检查文件是否存在
if not Path(args.file1).exists() or not Path(args.file2).exists():
print("文件不存在！")
sys.exit(1)
# 读取文件
with open(args.file1, 'r', encoding='utf-8') as f1, \
open(args.file2, 'r', encoding='utf-8') as f2:
text1 = f1.read()
text2 = f2.read()
# 执行对比
if args.method == 'exact':
result = text1 == text2
print(f"精确匹配结果: {result}")
elif args.method == 'fuzzy':
from difflib import SequenceMatcher
similarity = SequenceMatcher(None, text1, text2).ratio()
print(f"模糊匹配相似度: {similarity*100:.2f}%")
elif args.method == 'line':
from itertools import zip_longest
lines1 = text1.splitlines()
lines2 = text2.splitlines()
match_count = 0
total = max(len(lines1), len(lines2))
for l1, l2 in zip_longest(lines1, lines2, fillvalue=''):
if l1 == l2:
match_count += 1
print(f"行匹配率: {match_count/total*100:.2f}%")
elif args.method == 'ngram':
from sklearn.feature_extraction.text import CountVectorizer
vect = CountVectorizer(analyzer='char', ngram_range=(3, 3))
try:
matrix = vect.fit_transform([text1, text2])
similarity = (matrix * matrix.T).A[0, 1]
print(f"N-gram相似度: {similarity*100:.2f}%")
except:
print("N-gram对比失败")
elif args.method == 'all':
# 综合对比
comparator = ComprehensiveFuzzyComparator()
results = comparator.compare_files(args.file1, args.file2)
for method, score in results.items():
if isinstance(score, float):
print(f"{method}: {score*100:.2f}%")
if __name__ == '__main__':
main()
使用建议
-
选择合适的算法：
- 短文本：用编辑距离或字符级N-gram
- 长文本：用TF-IDF或文档级相似度
- 代码文件：用行级别的精确匹配
-
性能优化：
# 对于大文件，使用流式处理 def compare_large_files(file1, file2, chunk_size=8192): with open(file1, 'rb') as f1, open(file2, 'rb') as f2: while True: chunk1 = f1.read(chunk_size) chunk2 = f2.read(chunk_size) if not chunk1 and not chunk2: break # 处理chunk -
安装依赖：
pip install difflib scikit-learn # 根据需要安装
这些方法可以根据你的具体需求选择使用,也可以组合使用以达到更好的对比效果。
