本文目录导读：
我来详细介绍如何在PHP项目中实现TF-IDF和文本相似度计算。
TF-IDF基本概念
TF-IDF = TF（词频） × IDF（逆文档频率）
- TF（Term Frequency）：词在文档中出现的频率
- IDF（Inverse Document Frequency）：包含该词的文档占总文档数的比例的对数
PHP实现TF-IDF
基础类结构
<?php
class TFIDF {
private $documents = [];
private $processedDocs = [];
private $vocabulary = [];
private $idfCache = [];
public function __construct(array $documents = []) {
$this->documents = $documents;
$this->processDocuments();
}
/**
* 分词处理（中文分词示例）
*/
private function tokenize($text) {
// 简单的分词，实际项目中建议使用 jieba-php 或 scws
$text = mb_strtolower($text, 'UTF-8');
// 去除标点符号和数字
$text = preg_replace('/[^\p{Han}\p{L}\s]/u', ' ', $text);
// 简单按空格分词
$words = preg_split('/\s+/', trim($text));
// 移除停用词
$stopWords = $this->getStopWords();
$words = array_filter($words, function($word) use ($stopWords) {
return !empty($word) && !in_array($word, $stopWords) && mb_strlen($word) > 1;
});
return array_values($words);
}
/**
* 处理文档
*/
private function processDocuments() {
$this->processedDocs = [];
$this->vocabulary = [];
foreach ($this->documents as $id => $document) {
$words = $this->tokenize($document);
$wordCount = array_count_values($words);
$this->processedDocs[$id] = [
'content' => $document,
'words' => $words,
'wordCount' => $wordCount,
'totalWords' => count($words)
];
// 构建词汇表
foreach ($wordCount as $word => $count) {
if (!isset($this->vocabulary[$word])) {
$this->vocabulary[$word] = 0;
}
$this->vocabulary[$word]++;
}
}
}
/**
* 计算TF值
*/
private function calculateTF($wordCount, $totalWords, $word) {
if (!isset($wordCount[$word]) || $totalWords == 0) {
return 0;
}
// TF = 词频 / 总词数
return $wordCount[$word] / $totalWords;
}
/**
* 计算IDF值
*/
private function calculateIDF($word) {
if (!isset($this->vocabulary[$word])) {
return 0;
}
if (isset($this->idfCache[$word])) {
return $this->idfCache[$word];
}
$totalDocs = count($this->documents);
$docsWithWord = $this->vocabulary[$word];
// IDF = log(总文档数 / (包含该词的文档数 + 1))
$idf = log(($totalDocs + 1) / ($docsWithWord + 1)) + 1;
$this->idfCache[$word] = $idf;
return $idf;
}
/**
* 计算文档的TF-IDF向量
*/
public function getDocumentVector($docId = null) {
$vector = [];
if ($docId !== null && isset($this->processedDocs[$docId])) {
$doc = $this->processedDocs[$docId];
$words = array_keys($this->vocabulary);
foreach ($words as $word) {
$tf = $this->calculateTF($doc['wordCount'], $doc['totalWords'], $word);
$idf = $this->calculateIDF($word);
$vector[$word] = $tf * $idf;
}
} else {
// 计算所有文档的向量
foreach ($this->processedDocs as $id => $doc) {
$words = array_keys($this->vocabulary);
$docVector = [];
foreach ($words as $word) {
$tf = $this->calculateTF($doc['wordCount'], $doc['totalWords'], $word);
$idf = $this->calculateIDF($word);
$docVector[$word] = $tf * $idf;
}
$vector[$id] = $docVector;
}
}
return $vector;
}
/**
* 获取文档中TF-IDF最高的关键词
*/
public function getKeywords($docId, $topN = 10) {
$vector = $this->getDocumentVector($docId);
if (empty($vector)) {
return [];
}
arsort($vector);
return array_slice($vector, 0, $topN, true);
}
/**
* 获取停用词列表
*/
private function getStopWords() {
return ['的', '了', '在', '是', '我', '有', '和', '就', '不', '人', '都', '一', '一个', '上', '也', '很', '到', '说', '要', '去', '你', '会', '着', '没有', '看', '好', '自己', '这', '他', '她', '它', '们'];
}
}
文本相似度计算
余弦相似度实现
<?php
class TextSimilarity {
/**
* 计算余弦相似度
*/
public static function cosineSimilarity(array $vectorA, array $vectorB) {
$dotProduct = 0;
$normA = 0;
$normB = 0;
// 获取所有唯一的key
$allKeys = array_unique(array_merge(array_keys($vectorA), array_keys($vectorB)));
foreach ($allKeys as $key) {
$valueA = $vectorA[$key] ?? 0;
$valueB = $vectorB[$key] ?? 0;
$dotProduct += $valueA * $valueB;
$normA += $valueA * $valueA;
$normB += $valueB * $valueB;
}
$normA = sqrt($normA);
$normB = sqrt($normB);
if ($normA == 0 || $normB == 0) {
return 0;
}
return $dotProduct / ($normA * $normB);
}
/**
* 计算Jaccard相似度
*/
public static function jaccardSimilarity(array $setA, array $setB) {
$intersection = array_intersect($setA, $setB);
$union = array_unique(array_merge($setA, $setB));
if (empty($union)) {
return 0;
}
return count($intersection) / count($union);
}
/**
* 计算欧几里得距离
*/
public static function euclideanDistance(array $vectorA, array $vectorB) {
$distance = 0;
$allKeys = array_unique(array_merge(array_keys($vectorA), array_keys($vectorB)));
foreach ($allKeys as $key) {
$valueA = $vectorA[$key] ?? 0;
$valueB = $vectorB[$key] ?? 0;
$distance += pow($valueA - $valueB, 2);
}
return sqrt($distance);
}
/**
* 计算皮尔逊相关系数
*/
public static function pearsonCorrelation(array $vectorA, array $vectorB) {
$allKeys = array_unique(array_merge(array_keys($vectorA), array_keys($vectorB)));
$n = count($allKeys);
if ($n < 2) return 0;
$sumA = 0;
$sumB = 0;
$sumAB = 0;
$sumA2 = 0;
$sumB2 = 0;
foreach ($allKeys as $key) {
$valueA = $vectorA[$key] ?? 0;
$valueB = $vectorB[$key] ?? 0;
$sumA += $valueA;
$sumB += $valueB;
$sumAB += $valueA * $valueB;
$sumA2 += $valueA * $valueA;
$sumB2 += $valueB * $valueB;
}
$numerator = $n * $sumAB - $sumA * $sumB;
$denominator = sqrt(($n * $sumA2 - $sumA * $sumA) * ($n * $sumB2 - $sumB * $sumB));
if ($denominator == 0) {
return 0;
}
return $numerator / $denominator;
}
}
完整使用示例
<?php
require_once 'TFIDF.php';
require_once 'TextSimilarity.php';
// 创建文档集合
$documents = [
1 => "PHP是一种流行的服务器端脚本语言，广泛用于Web开发",
2 => "PHP语言非常适合Web开发，特别是动态网站开发",
3 => "Python也是一种流行的编程语言，用于数据分析和人工智能",
4 => "JavaScript是前端开发的主要语言，也用于后端开发"
];
// 初始化TF-IDF
$tfidf = new TFIDF($documents);
// 获取文档向量
$vector1 = $tfidf->getDocumentVector(1);
$vector2 = $tfidf->getDocumentVector(2);
// 计算相似度
$cosineSim = TextSimilarity::cosineSimilarity($vector1, $vector2);
echo "文档1和文档2的余弦相似度: " . round($cosineSim, 4) . "\n";
// 获取关键词
$keywords1 = $tfidf->getKeywords(1, 5);
echo "\n文档1的关键词:\n";
foreach ($keywords1 as $word => $score) {
echo "$word: " . round($score, 4) . "\n";
}
// 比较所有文档
echo "\n所有文档相似度矩阵:\n";
$totalDocs = count($documents);
for ($i = 1; $i <= $totalDocs; $i++) {
for ($j = $i + 1; $j <= $totalDocs; $j++) {
$vecI = $tfidf->getDocumentVector($i);
$vecJ = $tfidf->getDocumentVector($j);
$sim = TextSimilarity::cosineSimilarity($vecI, $vecJ);
echo "文档{$i} - 文档{$j}: " . round($sim, 4) . "\n";
}
}
// 搜索相关文档
$searchQuery = "Web开发语言";
$searchResults = searchDocuments($tfidf, $documents, $searchQuery, $tfidf);
echo "\n搜索 '{$searchQuery}' 的结果:\n";
foreach ($searchResults as $docId => $score) {
echo "文档{$docId}: " . round($score, 4) . " - " . mb_substr($documents[$docId], 0, 30) . "...\n";
}
/**
* 搜索文档
*/
function searchDocuments($tfidf, $documents, $query, $existingTfidf) {
// 创建临时文档集处理查询
$allDocs = $documents;
$allDocs['query'] = $query;
$tempTfidf = new TFIDF($allDocs);
$queryVector = $tempTfidf->getDocumentVector('query');
$results = [];
foreach ($documents as $id => $content) {
$docVector = $tempTfidf->getDocumentVector($id);
$results[$id] = TextSimilarity::cosineSimilarity($queryVector, $docVector);
}
arsort($results);
return $results;
}
性能优化建议
<?php
class OptimizedTFIDF extends TFIDF {
/**
* 使用字典树优化词汇表
*/
private $trie = [];
/**
* 缓存已计算的向量
*/
private $vectorCache = [];
/**
* 批量处理文档
*/
public function batchProcess(array $documents) {
$batches = array_chunk($documents, 100); // 每批处理100个文档
foreach ($batches as $batch) {
$this->processBatch($batch);
}
}
/**
* 增量更新
*/
public function addDocument($id, $content) {
$this->documents[$id] = $content;
$words = $this->tokenize($content);
$wordCount = array_count_values($words);
$this->processedDocs[$id] = [
'content' => $content,
'words' => $words,
'wordCount' => $wordCount,
'totalWords' => count($words)
];
// 更新词汇表
foreach ($wordCount as $word => $count) {
if (!isset($this->vocabulary[$word])) {
$this->vocabulary[$word] = 0;
}
$this->vocabulary[$word]++;
}
// 清除缓存
$this->vectorCache = [];
$this->idfCache = [];
}
}
注意事项
中文分词
- 需要接入专业分词库（如 scws, jieba-php）
- 处理新词识别和歧义消除
性能考虑
- 文档量较大时考虑使用缓存
- 可以使用内存数据库（Redis）存储TF-IDF向量
- 大规模文档建议使用搜索引擎（如Elasticsearch）
准确度优化
- 选择合适的停用词表
- 考虑词性过滤
- 可以对不同词性赋予不同权重
这个实现提供了完整的TF-IDF和文本相似度计算功能，可以根据实际需求进行调整和优化。
