本文目录导读：
模糊粗糙建模的方法,从简单到复杂：
基于词频的简单建模 (Python)
import re
from collections import Counter
import random
class FuzzyFileModel:
def __init__(self, ngram_size=2):
self.ngram_size = ngram_size
self.ngrams = Counter()
self.words = []
def train(self, file_path):
"""从文件学习内容模式"""
with open(file_path, 'r', encoding='utf-8') as f:
text = f.read()
# 提取单词
self.words = re.findall(r'\w+', text.lower())
# 构建n-gram模型
for i in range(len(self.words) - self.ngram_size + 1):
ngram = tuple(self.words[i:i+self.ngram_size])
self.ngrams[ngram] += 1
def generate(self, length=50, seed=None):
"""基于概率生成模糊内容"""
if seed is None:
seed = random.choice(list(self.ngrams.keys()))
result = list(seed)
for _ in range(length - self.ngram_size):
# 找到所有以当前n-gram开头的模式
candidates = [ng for ng in self.ngrams.keys()
if ng[:-1] == tuple(result[-(self.ngram_size-1):])]
if not candidates:
break
# 选择概率最高的
next_word = max(candidates, key=lambda x: self.ngrams[x])[-1]
result.append(next_word)
return ' '.join(result)
基于TF-IDF的文档模糊聚类
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import numpy as np
def fuzzy_document_clustering(documents, n_clusters=3):
"""对文档进行模糊聚类建模"""
# 创建TF-IDF向量
vectorizer = TfidfVectorizer(
max_features=1000,
stop_words='english',
ngram_range=(1, 2)
)
tfidf_matrix = vectorizer.fit_transform(documents)
# K-means聚类
kmeans = KMeans(
n_clusters=n_clusters,
random_state=42,
n_init=10
)
clusters = kmeans.fit_predict(tfidf_matrix)
# 获取每个聚类的关键词
feature_names = vectorizer.get_feature_names_out()
cluster_keywords = {}
for i in range(n_clusters):
# 找出该聚类的中心
center = kmeans.cluster_centers_[i]
top_keywords = [feature_names[idx]
for idx in center.argsort()[-10:]]
cluster_keywords[i] = top_keywords
return clusters, cluster_keywords
基于相似度的文件内容建模
import hashlib
from typing import List, Set
class RoughContentModel:
def __init__(self, shingle_size=3):
self.shingle_size = shingle_size
self.shingles: Set[str] = set()
self.stats = {}
def build_from_file(self, file_path: str):
"""构建粗糙的shingle模型"""
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
# 生成shingles
content = content.lower()
for i in range(len(content) - self.shingle_size + 1):
shingle = content[i:i+self.shingle_size]
self.shingles.add(shingle)
# 统计基本信息
self.stats = {
'total_shingles': len(self.shingles),
'unique_chars': len(set(content)),
'line_count': len(content.split('\n')),
'word_count': len(content.split())
}
def similarity_to(self, other_model) -> float:
"""计算与其他模型的相似度"""
if not self.shingles or not other_model.shingles:
return 0.0
intersection = self.shingles & other_model.shingles
union = self.shingles | other_model.shingles
return len(intersection) / len(union) if union else 0
def get_fuzzy_hash(self) -> str:
"""生成模糊特征哈希"""
sorted_shingles = sorted(list(self.shingles))
combined = ''.join(sorted_shingles[:100]) # 只取前100个
return hashlib.md5(combined.encode()).hexdigest()
实践示例：文件分类器
import os
from pathlib import Path
class FileFuzzyClassifier:
def __init__(self):
self.models = {}
self.labels = {}
def train(self, training_dir: str):
"""训练模糊分类器"""
for label_dir in Path(training_dir).iterdir():
if label_dir.is_dir():
label = label_dir.name
self.labels[label] = []
for file_path in label_dir.glob('*'):
model = RoughContentModel(shingle_size=4)
model.build_from_file(str(file_path))
self.models[str(file_path)] = (model, label)
self.labels[label].append(model)
def classify(self, file_path: str) -> str:
"""对文件进行分类"""
query_model = RoughContentModel(shingle_size=4)
query_model.build_from_file(file_path)
# 计算与各类别的平均相似度
avg_similarities = {}
for label, models in self.labels.items():
similarities = [
query_model.similarity_to(model)
for model in models
]
avg_similarities[label] = np.mean(similarities)
return max(avg_similarities, key=avg_similarities.get)
使用示例
# 示例：创建模糊模型
model = FuzzyFileModel(ngram_size=3)
model.train('sample.txt')
generated_content = model.generate(length=100)
print("生成的内容:", generated_content)
# 文件相似度比较
model1 = RoughContentModel(shingle_size=3)
model1.build_from_file('file1.txt')
model2 = RoughContentModel(shingle_size=3)
model2.build_from_file('file2.txt')
similarity = model1.similarity_to(model2)
print(f"文件相似度: {similarity:.2%}")
适用场景
推荐的模糊匹配推荐 2. 文档聚类将相似文档分到同一组 3. 文本分类的类别判断 4. 去重检测模糊查重 5. 模板生成**：基于学习的内容生成
这些方法各有特点,选择时需要考虑：
- 数据规模
- 精确度要求
- 处理速度需求
- 语言特性
需要我详细解释某个具体方法吗？
