本文目录导读：
在 PHP 爬虫中控制抓取频率是避免被封禁的重要措施,以下是完整的频率控制方案：
基础延时控制
<?php
class Crawler {
private $minDelay = 1; // 最小延时（秒）
private $maxDelay = 3; // 最大延时（秒）
// 简单延时
private function simpleDelay() {
sleep(rand($this->minDelay, $this->maxDelay));
}
// 毫秒级延时
private function microDelay() {
usleep(rand($this->minDelay * 1000000, $this->maxDelay * 1000000));
}
public function fetch($url) {
$content = $this->getContent($url);
$this->simpleDelay(); // 抓取后延时
return $content;
}
}
?>
动态自适应频率控制
<?php
class AdaptiveCrawler {
private $requestInterval; // 当前请求间隔
private $minInterval = 0.5; // 最小间隔（秒）
private $maxInterval = 10; // 最大间隔（秒）
private $successCount = 0; // 连续成功次数
private $failCount = 0; // 连续失败次数
private $lastRequestTime = 0;
private $config;
public function __construct($config = []) {
$this->config = $config;
$this->requestInterval = $this->minInterval;
}
public function fetch($url) {
$this->waitForNextRequest();
$result = $this->performRequest($url);
$this->adjustInterval($result['success']);
$this->lastRequestTime = microtime(true);
return $result['data'];
}
private function waitForNextRequest() {
$elapsed = microtime(true) - $this->lastRequestTime;
$waitTime = $this->requestInterval - $elapsed;
if ($waitTime > 0) {
usleep($waitTime * 1000000);
}
}
private function performRequest($url) {
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_TIMEOUT, 10);
$data = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$totalTime = curl_getinfo($ch, CURLINFO_TOTAL_TIME);
curl_close($ch);
$success = ($httpCode >= 200 && $httpCode < 400);
return [
'success' => $success,
'data' => $data,
'time' => $totalTime
];
}
private function adjustInterval($success) {
if ($success) {
$this->successCount++;
$this->failCount = 0;
// 连续成功20次，适当加快速度
if ($this->successCount % 20 == 0 && $this->requestInterval > $this->minInterval) {
$this->requestInterval = max(
$this->minInterval,
$this->requestInterval * 0.8
);
}
} else {
$this->failCount++;
$this->successCount = 0;
// 失败时增加间隔
if ($this->failCount >= 1) {
$this->requestInterval = min(
$this->maxInterval,
$this->requestInterval * 2
);
}
}
}
}
?>
队列 + 令牌桶算法
<?php
class TokenBucket {
private $tokens; // 当前令牌数
private $maxTokens; // 最大令牌数
private $rate; // 令牌生成速率（个/秒）
private $lastRefillTime; // 上次补充时间
public function __construct($rate, $maxTokens = null) {
$this->rate = $rate;
$this->maxTokens = $maxTokens ?: $rate;
$this->tokens = $this->maxTokens;
$this->lastRefillTime = microtime(true);
}
public function consume() {
$this->refill();
if ($this->tokens >= 1) {
$this->tokens--;
return true;
}
return false;
}
private function refill() {
$now = microtime(true);
$delta = $now - $this->lastRefillTime;
$tokensToAdd = $delta * $this->rate;
$this->tokens = min($this->maxTokens, $this->tokens + $tokensToAdd);
$this->lastRefillTime = $now;
}
public function waitAndConsume() {
while (!$this->consume()) {
usleep(500000); // 等待500ms
}
}
}
class QueueCrawler {
private $urlQueue; // URL队列
private $tokenBucket; // 令牌桶
private $isProcessing = false;
public function __construct() {
$this->urlQueue = [];
$this->tokenBucket = new TokenBucket(2); // 每秒2个请求
}
public function addUrl($url) {
$this->urlQueue[] = $url;
$this->processQueue();
}
private function processQueue() {
if ($this->isProcessing || empty($this->urlQueue)) {
return;
}
$this->isProcessing = true;
while (!empty($this->urlQueue)) {
$url = array_shift($this->urlQueue);
// 等待可用的令牌
$this->tokenBucket->waitAndConsume();
// 执行抓取
$this->fetchUrl($url);
// 可选的随机间隔
usleep(rand(100000, 500000)); // 0.1-0.5秒
}
$this->isProcessing = false;
}
private function fetchUrl($url) {
// 抓取逻辑
echo "抓取: {$url} - " . date('H:i:s') . PHP_EOL;
}
}
?>
记忆化频率控制（基于文件缓存）
<?php
class FileBasedRateLimiter {
private $cacheDir;
private $cacheTime = 3600; // 缓存时间（秒）
public function __construct($cacheDir = '/tmp/crawler_cache') {
$this->cacheDir = $cacheDir;
if (!is_dir($cacheDir)) {
mkdir($cacheDir, 0755, true);
}
}
public function checkLimit($url, $maxRequests = 10, $timeWindow = 60) {
$urlHash = md5($url);
$cacheFile = $this->cacheDir . '/' . $urlHash . '.json';
$data = $this->readCache($cacheFile);
$now = time();
// 清理过期记录
$data = array_filter($data, function($time) use ($now, $timeWindow) {
return ($now - $time) < $timeWindow;
});
if (count($data) >= $maxRequests) {
return false; // 已达到限制
}
$data[] = $now;
$this->writeCache($cacheFile, $data);
return true;
}
private function readCache($file) {
if (file_exists($file)) {
$content = file_get_contents($file);
return json_decode($content, true) ?: [];
}
return [];
}
private function writeCache($file, $data) {
file_put_contents($file, json_encode($data), LOCK_EX);
}
}
?>
完整示例：带频率控制的爬虫类
<?php
class SmartCrawler {
private $userAgent;
private $delay;
private $jitter;
private $proxies = [];
private $currentProxy = 0;
private $rateLimiter;
private $semaphore; // 信号量（用于多线程）
public function __construct($config = []) {
$this->userAgent = $config['user_agent'] ?? 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)';
$this->delay = $config['delay'] ?? 2;
$this->jitter = $config['jitter'] ?? 0.5;
$this->rateLimiter = new TokenBucket($config['rate'] ?? 2);
// 可选：信号量控制并发
if (isset($config['semaphore_key'])) {
$this->semaphore = sem_get($config['semaphore_key'], 1);
}
}
public function fetch($url, $retry = 3) {
if ($this->semaphore) {
sem_acquire($this->semaphore);
}
// 速率控制
$this->rateLimiter->waitAndConsume();
// 添加随机延迟
$this->randomDelay();
$result = $this->makeRequest($url);
// 重试逻辑
if (!$result['success'] && $retry > 0) {
usleep(1000000); // 1秒后重试
$this->fetch($url, $retry - 1);
}
if ($this->semaphore) {
sem_release($this->semaphore);
}
return $result['data'];
}
private function randomDelay() {
$baseDelay = $this->delay;
$jitterAmount = rand(0, $this->jitter * 1000000) / 1000000;
$totalDelay = $baseDelay + $jitterAmount;
sleep($totalDelay);
}
private function makeRequest($url) {
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_USERAGENT, $this->userAgent);
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
// 代理设置
if (!empty($this->proxies)) {
$proxy = $this->getNextProxy();
curl_setopt($ch, CURLOPT_PROXY, $proxy['host']);
curl_setopt($ch, CURLOPT_PROXYPORT, $proxy['port']);
if (isset($proxy['auth'])) {
curl_setopt($ch, CURLOPT_PROXYUSERPWD, $proxy['auth']);
}
}
$data = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
$error = curl_error($ch);
curl_close($ch);
// 处理可能的封禁状态码
if (in_array($httpCode, [429, 403, 503])) {
$this->increaseDelay();
return ['success' => false, 'data' => null];
}
return [
'success' => ($httpCode == 200),
'data' => $data,
'http_code' => $httpCode,
'error' => $error
];
}
private function increaseDelay() {
$this->delay = min($this->delay * 2, 30); // 最大30秒
}
private function getNextProxy() {
if (empty($this->proxies)) return null;
$proxy = $this->proxies[$this->currentProxy];
$this->currentProxy = ($this->currentProxy + 1) % count($this->proxies);
return $proxy;
}
public function setProxies($proxies) {
$this->proxies = $proxies;
}
public function resetDelay() {
$this->delay = 2;
}
}
// 使用示例
$crawler = new SmartCrawler([
'delay' => 2, // 基础延迟2秒
'jitter' => 1, // 随机延迟0-1秒
'rate' => 1, // 每秒1个请求
]);
$urls = [
'https://example.com/page1',
'https://example.com/page2',
// ... more URLs
];
foreach ($urls as $url) {
$content = $crawler->fetch($url);
// 处理内容...
}
?>
其他优化建议
// 1. 设置请求头模拟浏览器
$headers = [
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8',
'Connection: keep-alive',
'DNT: 1'
];
// 2. 缓存已抓取的内容避免重复请求
function getWithCache($url, $cacheTime = 3600) {
$cacheFile = 'cache/' . md5($url);
if (file_exists($cacheFile) && (time() - filemtime($cacheFile)) < $cacheTime) {
return file_get_contents($cacheFile);
}
$content = fetchContent($url);
file_put_contents($cacheFile, $content);
return $content;
}
// 3. 随机化请求时间
function getRandomDelay($url) {
// 基于URL生成确定性随机数
$seed = crc32($url);
mt_srand($seed);
return mt_rand(2, 5); // 2-5秒
}
最佳实践建议
- 遵守robots.txt：先检查目标网站的robots.txt
- 从低速开始：初始频率设低，逐步提高
- 设置请求超时：避免僵死连接消耗资源
- 监控响应状态：检测到429或403时自动降频
- 使用日志记录：记录每次请求的时间和状态
- 分布式爬取：使用队列系统（Redis）实现分布式控制
选择合适的频率控制策略取决于目标网站的性质和你的需求，建议从保守的设置开始,根据实际成功率进行调整。
