本文目录导读：
在PHP中处理大文件时，分块读取是关键策略，避免一次性加载整个文件到内存,以下是几种分块处理方法：
基础分块读取（fread）
function readLargeFile($filePath, $chunkSize = 8192) {
$handle = fopen($filePath, 'rb');
if (!$handle) {
throw new Exception("无法打开文件");
}
try {
while (!feof($handle)) {
$chunk = fread($handle, $chunkSize);
// 处理每一块数据
processChunk($chunk);
}
} finally {
fclose($handle);
}
}
逐行读取（fgets）
适用于文本文件：
function readLineByLine($filePath) {
$handle = fopen($filePath, 'r');
if (!$handle) {
throw new Exception("无法打开文件");
}
try {
while (($line = fgets($handle)) !== false) {
// 处理每一行
processLine($line);
}
} finally {
fclose($handle);
}
}
使用迭代器模式
class FileIterator implements Iterator {
private $handle;
private $line;
private $lineNumber = 0;
private $chunkSize;
public function __construct($filePath, $chunkSize = 8192) {
$this->handle = fopen($filePath, 'r');
$this->chunkSize = $chunkSize;
}
public function current() {
return $this->line;
}
public function key() {
return $this->lineNumber;
}
public function next() {
$this->line = fgets($this->handle, $this->chunkSize);
$this->lineNumber++;
}
public function rewind() {
rewind($this->handle);
$this->lineNumber = 0;
$this->next();
}
public function valid() {
return $this->line !== false;
}
public function __destruct() {
if ($this->handle) {
fclose($this->handle);
}
}
}
// 使用
$iterator = new FileIterator('large_file.txt');
foreach ($iterator as $lineNumber => $line) {
echo "第{$lineNumber}行: $line";
}
大文件处理类
class LargeFileProcessor {
private $filePath;
private $chunkSize;
private $callback;
public function __construct($filePath, $chunkSize = 65536) {
$this->filePath = $filePath;
$this->chunkSize = $chunkSize;
}
public function process($callback) {
$handle = fopen($this->filePath, 'rb');
if (!$handle) {
throw new RuntimeException("无法打开文件：{$this->filePath}");
}
$buffer = '';
try {
while (!feof($handle)) {
$chunk = fread($handle, $this->chunkSize);
$buffer .= $chunk;
// 按行处理缓冲区内容
while (($pos = strpos($buffer, "\n")) !== false) {
$line = substr($buffer, 0, $pos);
$buffer = substr($buffer, $pos + 1);
$result = call_user_func($callback, $line);
if ($result === false) {
break 2; // 提前终止
}
}
}
// 处理最后一行（如果没有换行符）
if ($buffer !== '') {
call_user_func($callback, $buffer);
}
} finally {
fclose($handle);
}
}
}
// 使用示例
$processor = new LargeFileProcessor('data.csv');
$processor->process(function($line) {
$data = str_getcsv($line);
// 处理CSV数据
echo "名称: {$data[0]}, 年龄: {$data[1]}\n";
});
处理二进制文件
function processBinaryFile($filePath, $chunkSize = 16384) {
$handle = fopen($filePath, 'rb');
if (!$handle) {
throw new Exception("无法打开文件");
}
$bytesRead = 0;
try {
while (!feof($handle)) {
$chunk = fread($handle, $chunkSize);
$bytesRead += strlen($chunk);
// 计算进度
$fileSize = filesize($filePath);
$progress = ($bytesRead / $fileSize) * 100;
// 处理二进制数据
processBinaryChunk($chunk, $progress);
}
} finally {
fclose($handle);
}
}
流式处理（memory_limit考虑）
function streamProcessFile($filePath, $chunkSize = 1024 * 1024) {
$stream = fopen('php://temp', 'w+');
try {
// 模拟从远程或特殊源读取
$input = fopen($filePath, 'rb');
while (!feof($input)) {
$chunk = fread($input, $chunkSize);
// 写入临时流
fwrite($stream, $chunk);
// 重置指针并处理
rewind($stream);
processStreamData($stream);
// 清空流
ftruncate($stream, 0);
rewind($stream);
}
} finally {
fclose($stream);
}
}
带进度条的完整示例
class FileProgress {
public static function processFile($filePath, $callback) {
$handle = fopen($filePath, 'rb');
if (!$handle) {
throw new Exception("无法打开文件");
}
$fileSize = filesize($filePath);
$processedSize = 0;
$chunkSize = 8192;
$lastProgress = 0;
try {
while (!feof($handle)) {
$chunk = fread($handle, $chunkSize);
$processedSize += strlen($chunk);
// 调用处理回调
call_user_func($callback, $chunk);
// 计算和显示进度
$progress = ($fileSize > 0) ? intval(($processedSize / $fileSize) * 100) : 100;
if ($progress > $lastProgress) {
echo "\r处理进度: {$progress}%";
$lastProgress = $progress;
}
// 检查是否超过内存限制
if (memory_get_usage(true) > 80 * 1024 * 1024) {
gc_collect_cycles();
}
}
echo "\n处理完成！\n";
} finally {
fclose($handle);
}
}
}
// 使用
FileProgress::processFile('big_file.txt', function($chunk) {
// 处理数据块
// hash计算、数据库写入等
});
关键优化建议
- 合理设置块大小：8KB-1MB 之间，根据文件类型和内存限制调整
- 使用缓冲：php.ini 中设置
output_buffering - 及时释放内存：处理完的数据立即 unset
- 使用生成器：PHP 5.5+ 支持 yield，提供内存高效迭代
- 避免一次性处理：不要在同一内存中保留大量数据
// 优化示例：处理CSV并写入数据库
function processCSVToDB($filePath, $dbh) {
$handle = fopen($filePath, 'r');
fgetcsv($handle); // 跳过标题行
$batch = [];
$batchSize = 1000;
while (($data = fgetcsv($handle, 4096)) !== false) {
$batch[] = $data;
if (count($batch) >= $batchSize) {
insertBatch($dbh, $batch);
$batch = []; // 释放内存
}
}
// 处理剩余数据
if (!empty($batch)) {
insertBatch($dbh, $batch);
}
fclose($handle);
}
这些方法能有效处理GB级别的大文件，同时保持内存占用在一个合理范围（< 100MB）,选择哪种方法取决于你的具体需求和文件格式。
