本文目录导读：
模糊分类的方法,从简单到复杂：
基于关键词匹配的简单分类
import os
import re
def classify_by_keywords(file_path):
"""基于关键词的简单分类"""
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read().lower()
# 定义分类规则（关键词到类别的映射）
categories = {
'编程': ['python', 'java', 'javascript', 'code', 'programming'],
'金融': ['money', 'stock', 'finance', 'investment'],
'科技': ['tech', 'AI', 'machine learning', 'technology'],
'教育': ['education', 'school', 'university', 'learn']
}
scores = {}
for category, keywords in categories.items():
score = sum(content.count(keyword) for keyword in keywords)
if score > 0:
scores[category] = score
if not scores:
return "未分类"
# 返回得分最高的类别
return max(scores, key=scores.get)
基于TF-IDF的文本分类
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import numpy as np
def tfidf_classify(texts, n_clusters=5):
"""使用TF-IDF和KMeans进行模糊分类"""
# 文本向量化
vectorizer = TfidfVectorizer(max_features=1000, stop_words='english')
tfidf_matrix = vectorizer.fit_transform(texts)
# KMeans聚类
kmeans = KMeans(n_clusters=n_clusters, random_state=42)
clusters = kmeans.fit_predict(tfidf_matrix)
return clusters
# 获取每个聚类的关键词
def get_cluster_keywords(vectorizer, kmeans, n_words=10):
feature_names = vectorizer.get_feature_names_out()
cluster_keywords = {}
for i in range(kmeans.n_clusters):
center = kmeans.cluster_centers_[i]
top_indices = center.argsort()[-n_words:][::-1]
cluster_keywords[i] = [feature_names[idx] for idx in top_indices]
return cluster_keywords
基于规则的模糊分类器
import re
from collections import Counter
class FuzzyClassifier:
def __init__(self):
self.rules = {
'technical': {
'keywords': ['algorithm', 'database', 'system', 'code'],
'patterns': [r'\b[A-Z]{2,}\b', r'\d+\.\d+'],
'weights': {'keywords': 0.6, 'patterns': 0.4}
},
'business': {
'keywords': ['market', 'strategy', 'revenue', 'profit'],
'patterns': [r'\$\d+', r'\d+%', r'quarter[ly]?\s+\d'],
'weights': {'keywords': 0.7, 'patterns': 0.3}
},
'academic': {
'keywords': ['research', 'study', 'analysis', 'methodology'],
'patterns': [r'\[\d+\]', r'Figure\s+\d+', r'Table\s+\d+'],
'weights': {'keywords': 0.5, 'patterns': 0.5}
}
}
def classify(self, text):
"""模糊分类主方法"""
text = text.lower()
scores = {}
for category, rule in self.rules.items():
keyword_score = self._score_keywords(text, rule['keywords'])
pattern_score = self._score_patterns(text, rule['patterns'])
total_score = (
keyword_score * rule['weights']['keywords'] +
pattern_score * rule['weights']['patterns']
)
scores[category] = total_score
# 归一化并返回结果
total = sum(scores.values())
if total == 0:
return {'category': 'unknown', 'confidence': 0}
best_category = max(scores, key=scores.get)
confidence = scores[best_category] / total
return {
'category': best_category,
'confidence': confidence,
'all_scores': scores
}
def _score_keywords(self, text, keywords):
"""计算关键词得分"""
return sum(1 for kw in keywords if kw in text) / len(keywords)
def _score_patterns(self, text, patterns):
"""计算模式匹配得分"""
matches = sum(1 for pattern in patterns if re.search(pattern, text))
return matches / len(patterns)
完整文件分类脚本
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import re
import argparse
from collections import defaultdict
class FileClassifier:
def __init__(self):
self.category_rules = {
'技术文档': {
'keywords': ['API', 'SDK', '代码', '程序', '开发'],
'extensions': ['.py', '.java', '.js', '.cpp', '.h'],
'min_content_words': 100
},
'项目计划': {
'keywords': ['里程碑', 'deadline', '预算', '资源', '进度'],
'patterns': [r'\d{4}-\d{2}-\d{2}', r'Sprint\d+']
},
'报告分析': {
'keywords': ['分析', '#39;, '#39;, '建议', '趋势'],
'patterns': [r'\d+\.\d+%', r'同比增长']
}
}
def classify_file(self, filepath):
"""对单个文件进行模糊分类"""
if not os.path.exists(filepath):
return None
# 获取文件信息
filename = os.path.basename(filepath)
ext = os.path.splitext(filename)[1].lower()
# 读取文件内容
try:
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read()
except:
return None
# 特征提取
features = self._extract_features(content, filename, ext)
# 分类匹配
results = {}
for category, rules in self.category_rules.items():
score = self._calculate_similarity(features, rules)
if score > 0.3: # 阈值过滤
results[category] = score
if not results:
return {'category': '其他', 'confidence': 0, 'features': features}
best_match = max(results, key=results.get)
return {
'category': best_match,
'confidence': results[best_match],
'all_matches': results,
'features': features
}
def _extract_features(self, content, filename, ext):
"""提取文件特征"""
features = {
'filename': filename,
'extension': ext,
'word_count': len(content.split()),
'char_count': len(content),
'lines': content.count('\n'),
'capital_words': len(re.findall(r'\b[A-Z]{2,}\b', content)),
'numbers': len(re.findall(r'\d+', content)),
'special_chars': len(re.findall(r'[!@#$%^&*()]', content)),
}
# 提取关键词频率
all_keywords = set()
for rules in self.category_rules.values():
all_keywords.update(rules.get('keywords', []))
keyword_freq = {}
for kw in all_keywords:
keyword_freq[kw] = content.count(kw)
features['keyword_freq'] = keyword_freq
return features
def _calculate_similarity(self, features, rules):
"""计算相似度分数"""
score = 0
# 关键词匹配
if 'keywords' in rules:
keyword_matches = sum(
1 for kw in rules['keywords']
if features['keyword_freq'].get(kw, 0) > 0
)
score += keyword_matches / len(rules['keywords']) * 0.5
# 文件扩展名匹配
if 'extensions' in rules:
if features['extension'] in rules['extensions']:
score += 0.3
# 内容长度检查
if 'min_content_words' in rules:
if features['word_count'] >= rules['min_content_words']:
score += 0.2
return score
# 使用示例
def batch_classify(directory):
"""批量分类目录中的文件"""
classifier = FileClassifier()
results = defaultdict(list)
for root, dirs, files in os.walk(directory):
for file in files:
filepath = os.path.join(root, file)
result = classifier.classify_file(filepath)
if result:
results[result['category']].append({
'file': filepath,
'confidence': result['confidence']
})
return results
# 主函数
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='文件模糊分类工具')
parser.add_argument('path', help='文件或目录路径')
parser.add_argument('--verbose', '-v', action='store_true', help='显示详细信息')
args = parser.parse_args()
if os.path.isfile(args.path):
classifier = FileClassifier()
result = classifier.classify_file(args.path)
if result:
print(f"文件: {args.path}")
print(f"分类: {result['category']}")
print(f"置信度: {result['confidence']:.2%}")
elif os.path.isdir(args.path):
results = batch_classify(args.path)
for category, files in results.items():
print(f"\n{category} (共{len(files)}个文件):")
for f in files:
print(f" - {f['file']} (置信度: {f['confidence']:.2%})")
使用建议
- 根据需求定制分类规则：调整关键词、阈值和权重
- 渐进式优化：从简单规则开始，逐步添加复杂特征
- 处理中文文本：可能需要分词工具（如jieba）
- 性能优化：对大文件进行采样分析
这个脚本可以根据你的具体需求进行调整和扩展。
