本文目录导读：
我来详细讲解ThinkPHP项目中大文本的存储与检索方案。
存储方案选择
数据库存储方案
MySQL TEXT类型选择
// 根据数据大小选择合适类型 - TINYTEXT: 255字节 - TEXT: 64KB - MEDIUMTEXT: 16MB - LONGTEXT: 4GB
建表示例
CREATE TABLE `article` ( `id` int(11) NOT NULL AUTO_INCREMENT, varchar(255) NOT NULL, `content` longtext, `content_hash` char(32) DEFAULT NULL, `word_count` int(11) DEFAULT '0', `created_at` datetime DEFAULT NULL, `updated_at` datetime DEFAULT NULL, PRIMARY KEY (`id`), KEY `idx_content_hash` (`content_hash`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
文件系统存储
<?php
namespace app\common\storage;
use think\facade\Filesystem;
class TextFileStorage
{
/**
* 保存大文本到文件
*/
public static function save($content, $path = 'text')
{
$filename = md5(uniqid() . time()) . '.txt';
$filePath = $path . '/' . date('Y/m/d') . '/' . $filename;
// 使用ThinkPHP文件系统
Filesystem::put($filePath, $content);
return [
'path' => $filePath,
'url' => Filesystem::url($filePath)
];
}
/**
* 读取文件内容
*/
public static function read($filePath)
{
return Filesystem::get($filePath);
}
/**
* 删除文件
*/
public static function delete($filePath)
{
return Filesystem::delete($filePath);
}
}
模型层设计
模型类实现
<?php
namespace app\common\model;
use think\Model;
use think\facade\Cache;
class Article extends Model
{
protected $name = 'article';
protected $pk = 'id';
// 自动时间戳
protected $autoWriteTimestamp = true;
protected $createTime = 'created_at';
protected $updateTime = 'updated_at';
// 字段类型转换
protected $type = [
'id' => 'integer',
'word_count' => 'integer',
'created_at' => 'datetime',
'updated_at' => 'datetime',
];
// JSON字段
protected $json = ['extra_data'];
/**
* 保存大文本内容
*/
public function saveContent($title, $content, $options = [])
{
$this->title = $title;
$this->content = $content;
$this->content_hash = md5($content);
$this->word_count = mb_strlen(strip_tags($content), 'utf-8');
if (isset($options['extra_data'])) {
$this->extra_data = $options['extra_data'];
}
return $this->save();
}
/**
* 获取文章内容（带缓存）
*/
public function getContentWithCache($id)
{
$cacheKey = 'article_content_' . $id;
return Cache::remember($cacheKey, function() use ($id) {
$article = self::find($id);
return $article ? $article->content : null;
}, 3600);
}
/**
* 清空文章缓存
*/
public function clearCache($id)
{
Cache::delete('article_content_' . $id);
Cache::delete('article_detail_' . $id);
}
}
检索方案实现
MySQL全文索引
<?php
namespace app\common\search;
use think\facade\Db;
class MysqlSearch
{
/**
* 创建全文索引
*/
public function createFulltextIndex()
{
$sql = "ALTER TABLE article ADD FULLTEXT INDEX ft_content (title, content) WITH PARSER ngram";
Db::execute($sql);
}
/**
* 全文搜索
*/
public function search($keyword, $page = 1, $limit = 10)
{
$where = "MATCH(title, content) AGAINST (:keyword IN NATURAL LANGUAGE MODE)";
$total = Db::name('article')
->whereRaw($where, ['keyword' => $keyword])
->count();
$list = Db::name('article')
->whereRaw($where, ['keyword' => $keyword])
->field('id, title,
LEFT(content, 200) as content_preview,
MATCH(title, content) AGAINST (:keyword2) as relevance')
->bind(['keyword2' => $keyword])
->order('relevance DESC')
->page($page, $limit)
->select();
return [
'total' => $total,
'list' => $list
];
}
}
使用Elasticsearch（推荐）
<?php
namespace app\common\search;
use Elasticsearch\ClientBuilder;
class ElasticsearchService
{
private $client;
public function __construct()
{
$config = config('elasticsearch');
$this->client = ClientBuilder::create()
->setHosts($config['hosts'])
->build();
}
/**
* 创建索引
*/
public function createIndex($indexName = 'articles')
{
$params = [
'index' => $indexName,
'body' => [
'settings' => [
'number_of_shards' => 3,
'number_of_replicas' => 1
],
'mappings' => [
'properties' => [
'id' => ['type' => 'integer'],
'title' => [
'type' => 'text',
'analyzer' => 'ik_max_word',
'search_analyzer' => 'ik_smart'
],
'content' => [
'type' => 'text',
'analyzer' => 'ik_max_word',
'search_analyzer' => 'ik_smart',
'index_options' => 'offsets'
],
'created_at' => ['type' => 'date']
]
]
]
];
return $this->client->indices()->create($params);
}
/**
* 索引文档
*/
public function indexDocument($article)
{
$params = [
'index' => 'articles',
'id' => $article['id'],
'body' => [
'id' => $article['id'],
'title' => $article['title'],
'content' => $article['content'],
'created_at' => $article['created_at']
]
];
return $this->client->index($params);
}
/**
* 搜索
*/
public function search($keyword, $page = 1, $size = 10)
{
$params = [
'index' => 'articles',
'body' => [
'from' => ($page - 1) * $size,
'size' => $size,
'query' => [
'multi_match' => [
'query' => $keyword,
'fields' => ['title^3', 'content'],
'operator' => 'and'
]
],
'highlight' => [
'pre_tags' => ['<em class="highlight">'],
'post_tags' => ['</em>'],
'fields' => [
'title' => ['fragment_size' => 100],
'content' => ['fragment_size' => 200, 'number_of_fragments' => 2]
]
]
]
];
$response = $this->client->search($params);
return $this->formatSearchResult($response, $page, $size);
}
/**
* 格式化搜索结果
*/
private function formatSearchResult($response, $page, $size)
{
$hits = $response['hits'];
$list = [];
foreach ($hits['hits'] as $hit) {
$item = $hit['_source'];
$item['highlight'] = $hit['highlight'] ?? [];
$list[] = $item;
}
return [
'total' => $hits['total']['value'],
'page' => $page,
'size' => $size,
'list' => $list
];
}
}
简易搜索封装
<?php
namespace app\common\search;
use think\facade\Db;
use app\common\search\ElasticsearchService;
class TextSearch
{
private $searchEngine;
public function __construct($engine = null)
{
$this->searchEngine = $engine ?: new ElasticsearchService();
}
/**
* 综合搜索
*/
public function search($keyword, $options = [])
{
// 关键词预处理
$keyword = trim($keyword);
if (mb_strlen($keyword) < 2) {
return ['total' => 0, 'list' => []];
}
// 根据关键词长度选择搜索方式
if (mb_strlen($keyword) <= 10) {
// 短关键词使用ES
return $this->searchEngine->search($keyword,
$options['page'] ?? 1,
$options['size'] ?? 10);
} else {
// 长文本使用MySQL LIKE查询
return $this->mysqlSearch($keyword, $options);
}
}
/**
* MySQL搜索
*/
private function mysqlSearch($keyword, $options)
{
$page = $options['page'] ?? 1;
$limit = $options['size'] ?? 10;
$like = '%' . $keyword . '%';
$query = Db::name('article')
->where('title', 'like', $like)
->whereOr('content', 'like', $like);
$total = $query->count();
$list = $query
->field('id, title, LEFT(content, 200) as content_preview')
->page($page, $limit)
->select();
return [
'total' => $total,
'page' => $page,
'size' => $limit,
'list' => $list
];
}
}
大文本优化策略
分块存储
<?php
namespace app\common\storage;
class TextChunk
{
private $chunkSize = 60000; // 每块大小
/**
* 分块保存
*/
public function saveChunked($content, $articleId)
{
// 计算分块数量
$chunks = mb_str_split($content, $this->chunkSize);
// 保存到数据库
$data = [];
foreach ($chunks as $index => $chunk) {
$data[] = [
'article_id' => $articleId,
'chunk_index' => $index,
'content' => $chunk
];
}
Db::name('article_chunks')->insertAll($data);
}
/**
* 读取完整内容
*/
public function getFullContent($articleId)
{
$chunks = Db::name('article_chunks')
->where('article_id', $articleId)
->order('chunk_index')
->column('content');
return implode('', $chunks);
}
}
内容压缩存储
<?php
namespace app\common\storage;
class CompressedStorage
{
/**
* 压缩后保存
*/
public function saveCompressed($content, $articleId)
{
// GZIP压缩
$compressed = gzcompress($content, 9);
// Base64编码存储
$encoded = base64_encode($compressed);
return Db::name('article_compressed')->insert([
'article_id' => $articleId,
'content' => $encoded,
'original_size' => strlen($content),
'compressed_size' => strlen($encoded),
'created_at' => date('Y-m-d H:i:s')
]);
}
/**
* 读取解压
*/
public function getContent($articleId)
{
$record = Db::name('article_compressed')
->where('article_id', $articleId)
->find();
if (!$record) {
return null;
}
// 解码并解压
$compressed = base64_decode($record['content']);
return gzuncompress($compressed);
}
}
完整使用示例
<?php
namespace app\index\controller;
use think\Controller;
use app\common\model\Article;
use app\common\search\TextSearch;
use app\common\storage\TextFileStorage;
class ArticleController extends Controller
{
/**
* 保存文章
*/
public function save()
{
$data = $this->request->post();
// 验证数据
validate(['title' => 'require|max:255', 'content' => 'require'])
->check($data);
$article = new Article();
// 保存文章
$article->saveContent($data['title'], $data['content'], [
'extra_data' => [
'source' => $data['source'] ?? '',
'author' => $data['author'] ?? ''
]
]);
// 索引到ES
try {
$es = new ElasticsearchService();
$es->indexDocument($article->toArray());
} catch (\Exception $e) {
// 记录日志，不影响主流程
trace('ES索引失败: ' . $e->getMessage(), 'error');
}
return json(['status' => 1, 'message' => '保存成功']);
}
/**
* 搜索文章
*/
public function search()
{
$keyword = $this->request->get('keyword', '');
$page = $this->request->get('page', 1);
if (empty($keyword)) {
return json(['status' => 0, 'message' => '关键词不能为空']);
}
$searchService = new TextSearch();
$result = $searchService->search($keyword, ['page' => $page]);
return json([
'status' => 1,
'data' => $result
]);
}
/**
* 获取完整文章
*/
public function detail($id)
{
$article = Article::find($id);
if (!$article) {
return json(['status' => 0, 'message' => '文章不存在']);
}
// 浏览量增加
$article->setInc('views');
// 获取内容（带缓存）
$content = $article->getContentWithCache($id);
return json([
'status' => 1,
'data' => [
'id' => $article->id,
'title' => $article->title,
'content' => $content,
'created_at' => $article->created_at,
'views' => $article->views
]
]);
}
}
性能优化建议
缓存策略
- 使用Redis缓存热门文章内容
- 缓存搜索结果
- 合理的缓存过期时间
数据库优化
- 合理使用索引
- 分表存储大数据量文本
- 使用读写分离
检索优化
- 结合ES进行全文检索
- 使用IK分词器提高中文搜索准确性
- 优化查询语句
异步处理
- 使用消息队列异步索引
- 异步生成摘要
这个方案提供了完整的大文本存储与检索解决方案,可以根据实际需求选择合适的方式组合使用。
