本文目录导读：
我来为您介绍如何用Python实现文件内容的模糊C均值（FCM）聚类算法。
基本实现方案
import numpy as np
import os
from typing import List, Tuple
import re
class FuzzyCMeans:
def __init__(self, n_clusters=3, m=2, max_iter=100, epsilon=1e-5):
"""
模糊C均值聚类
Parameters:
- n_clusters: 聚类数量
- m: 模糊指数（通常为2）
- max_iter: 最大迭代次数
- epsilon: 收敛阈值
"""
self.n_clusters = n_clusters
self.m = m
self.max_iter = max_iter
self.epsilon = epsilon
def _initialize_membership(self, n_samples):
"""初始化隶属度矩阵"""
membership = np.random.dirichlet(np.ones(self.n_clusters), size=n_samples)
return membership.T
def _compute_centers(self, data, membership):
"""计算聚类中心"""
um = membership ** self.m
centers = np.dot(um, data) / np.sum(um, axis=1, keepdims=True)
return centers
def _update_membership(self, data, centers):
"""更新隶属度矩阵"""
n_samples = data.shape[0]
membership = np.zeros((self.n_clusters, n_samples))
for i in range(self.n_clusters):
for j in range(n_samples):
# 计算样本到各中心的距离
distances = np.linalg.norm(data[j] - centers, axis=1)
# 避免除零
distances = np.maximum(distances, 1e-10)
# 计算隶属度
membership[i, j] = 1.0 / np.sum((distances[i] / distances) ** (2/(self.m-1)))
return membership
def fit(self, data):
"""执行FCM聚类"""
n_samples = data.shape[0]
# 初始化隶属度矩阵
membership = self._initialize_membership(n_samples)
# 迭代优化
for iteration in range(self.max_iter):
# 计算聚类中心
centers = self._compute_centers(data, membership)
# 更新隶属度
new_membership = self._update_membership(data, centers)
# 检查收敛
if np.max(np.abs(new_membership - membership)) < self.epsilon:
print(f"迭代 {iteration+1} 次后收敛")
break
membership = new_membership
self.membership_ = membership
self.cluster_centers_ = centers
self.labels_ = np.argmax(membership, axis=0)
return self
class FileContentCluster:
def __init__(self, fcm_params=None):
"""文件内容聚类器"""
if fcm_params is None:
fcm_params = {}
self.fcm = FuzzyCMeans(**fcm_params)
self.vectorizer = None
def _extract_features(self, content: str) -> np.ndarray:
"""从文本内容提取特征向量"""
# 简单词频统计
words = re.findall(r'\w+', content.lower())
word_freq = {}
for word in words:
word_freq[word] = word_freq.get(word, 0) + 1
# 提取一些简单特征
features = [
len(words), # 词数
len(set(words)), # 不同词数
len(content), # 字符数
len(re.findall(r'[.!?]', content)), # 句子数
np.mean([len(w) for w in words]) if words else 0, # 平均词长
]
return np.array(features)
def cluster_files(self, file_contents: List[str]) -> Tuple[np.ndarray, np.ndarray]:
"""
聚类文件内容
Parameters:
- file_contents: 文件内容列表
Returns:
- cluster_labels: 聚类标签
- membership: 隶属度矩阵
"""
# 提取特征
features = np.array([self._extract_features(content) for content in file_contents])
# 标准化特征
features = (features - features.mean(axis=0)) / (features.std(axis=0) + 1e-10)
# 执行聚类
self.fcm.fit(features)
return self.fcm.labels_, self.fcm.membership_
def cluster_files_with_paths(self, file_paths: List[str]) -> dict:
"""
从文件路径读取并聚类
Parameters:
- file_paths: 文件路径列表
Returns:
- result: 聚类结果字典
"""
file_contents = []
valid_files = []
for file_path in file_paths:
try:
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
file_contents.append(content)
valid_files.append(file_path)
except Exception as e:
print(f"读取文件 {file_path} 失败: {e}")
if not file_contents:
return {}
# 聚类
labels, membership = self.cluster_files(file_contents)
# 组织结果
result = {}
for i, (file_path, label) in enumerate(zip(valid_files, labels)):
cluster_key = f"Cluster_{label}"
if cluster_key not in result:
result[cluster_key] = {
'files': [],
'membership': []
}
result[cluster_key]['files'].append(file_path)
result[cluster_key]['membership'].append(membership[label, i])
return result
# 使用示例
if __name__ == "__main__":
# 示例文件内容
contents = [
"Python is a programming language. It is easy to learn.",
"Machine learning uses algorithms to learn from data.",
"The weather is nice today. Let's go outside.",
"Deep learning is a subset of machine learning."
]
# 创建聚类器
clusterer = FileContentCluster(n_clusters=2, m=2)
# 聚类
labels, membership = clusterer.cluster_files(contents)
# 打印结果
for i, (content, label) in enumerate(zip(contents, labels)):
print(f"文档 {i+1}: 聚类 {label}, 隶属度: {membership[label, i]:.3f}")
print(f" 内容: {content[:50]}...")
print()
# 文件路径示例
file_paths = ['file1.txt', 'file2.txt', 'file3.txt']
if all(os.path.exists(p) for p in file_paths):
result = clusterer.cluster_files_with_paths(file_paths)
for cluster_key, cluster_info in result.items():
print(f"\n{cluster_key}:")
for file_path, mem in zip(cluster_info['files'], cluster_info['membership']):
print(f" - {file_path} (隶属度: {mem:.3f})")
高级实现：使用文本向量化
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
class AdvancedFileCluster:
def __init__(self, n_clusters=3, m=2):
self.n_clusters = n_clusters
self.m = m
self.vectorizer = TfidfVectorizer(max_features=100, stop_words='english')
self.fcm = FuzzyCMeans(n_clusters=n_clusters, m=m)
def cluster_with_tfidf(self, file_contents):
"""
使用TF-IDF向量化进行聚类
"""
# TF-IDF向量化
tfidf_matrix = self.vectorizer.fit_transform(file_contents)
data = tfidf_matrix.toarray()
# 聚类
self.fcm.fit(data)
return self.fcm.labels_, self.fcm.membership_
# 使用示例
contents = [
"Python programming machine learning data science",
"JavaScript web development frontend backend",
"Deep learning neural networks AI artificial intelligence",
"CSS HTML web design responsive layout"
]
clusterer = AdvancedFileCluster(n_clusters=2)
labels, membership = clusterer.cluster_with_tfidf(contents)
for i, (content, label) in enumerate(zip(contents, labels)):
print(f"文档 {i+1} 聚类: {label}, 隶属度: {membership[label, i]:.3f}")
优化建议
- 特征工程：根据具体文件类型（代码、文档、日志等）设计更合适的特征
- 并行处理：处理大量文件时使用多线程/进程
- 增量学习：对新增文件进行增量聚类
- 结果评估：添加轮廓系数等评估指标
- 可视化：使用PCA/t-SNE降维可视化聚类结果
这个实现可以根据您的具体需求进行调整和优化。
