本文目录导读：
我来详细介绍如何使用PHP构建爬虫管理系统。
基础爬虫框架
简单的爬虫类
<?php
class SimpleCrawler {
private $ch;
private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
private $cookies = [];
private $timeout = 30;
private $maxRetries = 3;
private $proxy = null;
public function __construct($config = []) {
if (isset($config['userAgent'])) $this->userAgent = $config['userAgent'];
if (isset($config['timeout'])) $this->timeout = $config['timeout'];
if (isset($config['proxy'])) $this->proxy = $config['proxy'];
}
/**
* 发送GET请求
*/
public function get($url, $headers = []) {
return $this->request('GET', $url, null, $headers);
}
/**
* 发送POST请求
*/
public function post($url, $data = [], $headers = []) {
return $this->request('POST', $url, $data, $headers);
}
/**
* 核心请求方法
*/
private function request($method, $url, $data = null, $headers = []) {
$attempts = 0;
while ($attempts < $this->maxRetries) {
try {
$this->ch = curl_init();
// 基础配置
curl_setopt_array($this->ch, [
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 5,
CURLOPT_TIMEOUT => $this->timeout,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_SSL_VERIFYHOST => false,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_HTTPHEADER => $this->buildHeaders($headers),
]);
// 请求类型处理
if ($method === 'POST') {
curl_setopt($this->ch, CURLOPT_POST, true);
if ($data) {
curl_setopt($this->ch, CURLOPT_POSTFIELDS, is_array($data) ? http_build_query($data) : $data);
}
}
// Cookie处理
if (!empty($this->cookies)) {
curl_setopt($this->ch, CURLOPT_COOKIE, $this->buildCookieString());
}
// 代理设置（可选）
if ($this->proxy) {
curl_setopt($this->ch, CURLOPT_PROXY, $this->proxy);
}
// 执行请求
$response = curl_exec($this->ch);
$httpCode = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
// 处理Cookie
preg_match_all('/^Set-Cookie:\s*([^;]*)/mi', $response, $matches);
foreach ($matches[1] as $cookie) {
$parts = explode('=', $cookie, 2);
if (count($parts) == 2) {
$this->cookies[$parts[0]] = $parts[1];
}
}
curl_close($this->ch);
if ($httpCode === 200) {
return [
'success' => true,
'http_code' => $httpCode,
'content' => $response,
'cookies' => $this->cookies
];
}
// 处理限流
if ($httpCode === 429 || $httpCode === 503) {
sleep(5); // 等待5秒
$attempts++;
continue;
}
return [
'success' => false,
'http_code' => $httpCode,
'content' => $response
];
} catch (Exception $e) {
$attempts++;
if ($attempts >= $this->maxRetries) {
return [
'success' => false,
'error' => $e->getMessage()
];
}
sleep(1);
}
}
return ['success' => false, 'error' => 'Max retries exceeded'];
}
/**
* 构建请求头
*/
private function buildHeaders($customHeaders) {
$defaultHeaders = [
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language: en-US,en;q=0.5',
'Connection: keep-alive',
'Upgrade-Insecure-Requests: 1',
];
return array_merge($defaultHeaders, $customHeaders);
}
/**
* 构建Cookie字符串
*/
private function buildCookieString() {
$cookieStr = '';
foreach ($this->cookies as $key => $value) {
$cookieStr .= $key . '=' . $value . '; ';
}
return rtrim($cookieStr, '; ');
}
/**
* 添加Cookie
*/
public function addCookie($name, $value) {
$this->cookies[$name] = $value;
}
/**
* 清除Cookie
*/
public function clearCookies() {
$this->cookies = [];
}
/**
* 获取页面链接
*/
public function extractLinks($html, $baseUrl) {
$links = [];
// 提取HTML中的链接
preg_match_all('/<a[^>]*href=["\']([^"\']+)["\']/i', $html, $matches);
foreach ($matches[1] as $link) {
// 处理相对路径和绝对路径
$absoluteLink = $this->resolveUrl($baseUrl, $link);
if ($absoluteLink && !in_array($absoluteLink, $links)) {
$links[] = $absoluteLink;
}
}
return $links;
}
/**
* URL解析
*/
private function resolveUrl($baseUrl, $url) {
// 如果是绝对URL，直接返回
if (filter_var($url, FILTER_VALIDATE_URL)) {
return $url;
}
// 处理相对URL
$parsed = parse_url($baseUrl);
// 处理协议相对URL
if (strpos($url, '//') === 0) {
return $parsed['scheme'] . ':' . $url;
}
// 处理锚点和JavaScript链接
if (strpos($url, '#') === 0 || strpos($url, 'javascript:') === 0) {
return null;
}
// 处理相对路径
$base = $parsed['scheme'] . '://' . $parsed['host'];
if (isset($parsed['port'])) {
$base .= ':' . $parsed['port'];
}
// 处理路径
$path = isset($parsed['path']) ? $parsed['path'] : '/';
$path = str_replace('\\', '/', $path);
if (strpos($url, '/') === 0) {
// 绝对路径
return $base . $url;
} else {
// 相对路径
$dir = dirname($path);
return $base . $dir . '/' . $url;
}
}
}
爬虫管理系统
数据库设计
-- 任务表
CREATE TABLE `crawler_tasks` (
`id` INT(11) NOT NULL AUTO_INCREMENT,
`name` VARCHAR(100) NOT NULL COMMENT '任务名称',
`target_url` VARCHAR(500) NOT NULL COMMENT '目标URL',
`max_pages` INT(11) DEFAULT 100 COMMENT '最大爬取页数',
`interval_time` INT(11) DEFAULT 2 COMMENT '请求间隔(秒)',
`status` ENUM('pending','running','paused','completed','failed') DEFAULT 'pending',
`last_run_at` TIMESTAMP NULL DEFAULT NULL,
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
`updated_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
INDEX `idx_status` (`status`)
) ENGINE=InnoDB;
-- URL队列表
CREATE TABLE `crawler_urls` (
`id` INT(11) NOT NULL AUTO_INCREMENT,
`task_id` INT(11) NOT NULL,
`url` VARCHAR(500) NOT NULL,
`status` ENUM('pending','processing','completed','failed') DEFAULT 'pending',
`retry_count` INT(11) DEFAULT 0,
`error_message` TEXT,
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `unique_url` (`task_id`, `url`),
INDEX `idx_task_status` (`task_id`, `status`)
) ENGINE=InnoDB;
-- 爬取数据表
CREATE TABLE `crawler_data` (
`id` INT(11) NOT NULL AUTO_INCREMENT,
`task_id` INT(11) NOT NULL,
`url` VARCHAR(500) NOT NULL, VARCHAR(500),
`content` LONGTEXT,
`meta` TEXT COMMENT '元数据',
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
INDEX `idx_task_id` (`task_id`)
) ENGINE=InnoDB;
-- 代理池表
CREATE TABLE `proxies` (
`id` INT(11) NOT NULL AUTO_INCREMENT,
`ip` VARCHAR(50) NOT NULL,
`port` INT(11) NOT NULL,
`type` ENUM('http','https') DEFAULT 'http',
`status` ENUM('active','inactive') DEFAULT 'active',
`speed` INT(11) DEFAULT 0 COMMENT '响应速度ms',
`last_checked_at` TIMESTAMP NULL DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB;
任务管理器类
<?php
class CrawlerManager {
private $db;
private $logger;
public function __construct($db, $logger = null) {
$this->db = $db;
$this->logger = $logger;
}
/**
* 创建爬虫任务
*/
public function createTask($data) {
$sql = "INSERT INTO crawler_tasks (name, target_url, max_pages, interval_time)
VALUES (?, ?, ?, ?)";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('ssii', $data['name'], $data['target_url'], $data['max_pages'], $data['interval_time']);
$stmt->execute();
return $this->db->insert_id;
}
/**
* 启动任务
*/
public function startTask($taskId) {
// 更新任务状态
$this->updateTaskStatus($taskId, 'running');
// 获取任务信息
$task = $this->getTask($taskId);
if (!$task) return false;
// 初始化URL队列
$this->initUrlQueue($taskId, $task);
// 启动爬虫进程
return $this->startCrawlerProcess($taskId);
}
/**
* 暂停任务
*/
public function pauseTask($taskId) {
$this->updateTaskStatus($taskId, 'paused');
// 停止爬虫进程
$this->stopCrawlerProcess($taskId);
// 保存进度
$this->saveTaskProgress($taskId);
}
/**
* 初始化URL队列
*/
private function initUrlQueue($taskId, $task) {
// 检查是否已有队列
$count = $this->getQueueCount($taskId);
if ($count == 0) {
$sql = "INSERT INTO crawler_urls (task_id, url) VALUES (?, ?)";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('is', $taskId, $task['target_url']);
$stmt->execute();
}
}
/**
* 开始爬虫进程
*/
private function startCrawlerProcess($taskId) {
// 使用pcntl_fork创建子进程（如果可用）
if (extension_loaded('pcntl')) {
$pid = pcntl_fork();
if ($pid == -1) {
// 无法创建子进程
return false;
} elseif ($pid) {
// 父进程
return $pid;
} else {
// 子进程执行爬虫
$this->runCrawlerInSubprocess($taskId);
exit();
}
} else {
// 使用curl_multi并发爬取
$this->runCrawlerInMultithread($taskId);
}
return true;
}
/**
* 子进程爬虫执行
*/
private function runCrawlerInSubprocess($taskId) {
$crawler = new SimpleCrawler();
$task = $this->getTask($taskId);
// 获取待处理的URL
$urls = $this->getPendingUrls($taskId, 10);
while (count($urls) > 0 && $this->getTaskStatus($taskId) == 'running') {
foreach ($urls as $url) {
// 标记为处理中
$this->updateUrlStatus($url['id'], 'processing');
try {
// 抓取页面
$result = $crawler->get($url['url']);
if ($result['success']) {
// 保存数据
$this->saveCrawledData($taskId, $url['url'], $result['content']);
// 提取新链接
$links = $crawler->extractLinks($result['content'], $url['url']);
$this->addUrlsToQueue($taskId, $links);
// 更新状态
$this->updateUrlStatus($url['id'], 'completed');
} else {
// 处理失败
$this->handleUrlFailure($url['id'], $result['error']);
}
} catch (Exception $e) {
$this->handleUrlFailure($url['id'], $e->getMessage());
}
// 间隔等待
sleep($task['interval_time']);
}
// 获取下一批URL
$urls = $this->getPendingUrls($taskId, 10);
}
// 完成或暂停任务
$this->updateTaskStatus($taskId, array_key_exists('urls', $urls) ? 'completed' : 'paused');
}
/**
* 使用curl_multi并发爬取
*/
private function runCrawlerInMultithread($taskId) {
$task = $this->getTask($taskId);
// 获取待处理的URL
$urls = $this->getPendingUrls($taskId, 10);
while (count($urls) > 0 && $this->getTaskStatus($taskId) == 'running') {
// 创建多个curl句柄
$handles = [];
$multiHandle = curl_multi_init();
foreach ($urls as $url) {
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url['url']);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
curl_setopt($ch, CURLOPT_TIMEOUT, 30);
curl_multi_add_handle($multiHandle, $ch);
$handles[$url['id']] = $ch;
}
// 执行并发请求
$running = null;
do {
curl_multi_exec($multiHandle, $running);
usleep(100000); // 0.1秒
} while ($running > 0);
// 处理响应
foreach ($handles as $urlId => $ch) {
$content = curl_multi_getcontent($ch);
$info = curl_getinfo($ch);
if ($info['http_code'] == 200) {
// 保存数据
$this->saveCrawledData($taskId, $info['url'], $content);
$this->updateUrlStatus($urlId, 'completed');
} else {
$this->handleUrlFailure($urlId, "HTTP Code: " . $info['http_code']);
}
curl_multi_remove_handle($multiHandle, $ch);
curl_close($ch);
}
curl_multi_close($multiHandle);
// 间隔等待
sleep($task['interval_time']);
// 获取下一批URL
$urls = $this->getPendingUrls($taskId, 10);
}
$remaining = $this->getPendingCount($taskId);
if ($remaining == 0) {
$this->updateTaskStatus($taskId, 'completed');
} else {
$this->updateTaskStatus($taskId, 'paused');
}
}
/**
* 保存爬取数据
*/
private function saveCrawledData($taskId, $url, $content) {
// 解析HTML内容
$title = $this->extractTitle($content);
$text = $this->extractText($content);
$sql = "INSERT INTO crawler_data (task_id, url, title, content) VALUES (?, ?, ?, ?)";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('isss', $taskId, $url, $title, $text);
$stmt->execute();
}
/**
* 提取标题
*/
private function extractTitle($html) {
preg_match('/<title[^>]*>(.*?)<\/title>/i', $html, $matches);
return isset($matches[1]) ? trim($matches[1]) : '';
}
/**
* 提取文本内容
*/
private function extractText($html) {
// 移除script和style
$html = preg_replace('/<script[^>]*>.*?<\/script>/is', '', $html);
$html = preg_replace('/<style[^>]*>.*?<\/style>/is', '', $html);
// 转换为纯文本
$text = strip_tags($html);
return trim($text);
}
/**
* 获取待处理的URL
*/
private function getPendingUrls($taskId, $limit) {
$sql = "SELECT * FROM crawler_urls
WHERE task_id = ? AND status = 'pending'
LIMIT ?";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('ii', $taskId, $limit);
$stmt->execute();
return $stmt->get_result()->fetch_all(MYSQLI_ASSOC);
}
/**
* 更新URL状态
*/
private function updateUrlStatus($urlId, $status) {
$sql = "UPDATE crawler_urls SET status = ? WHERE id = ?";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('si', $status, $urlId);
$stmt->execute();
}
/**
* 处理失败URL
*/
private function handleUrlFailure($urlId, $error) {
$sql = "UPDATE crawler_urls
SET status = 'failed', error_message = ?,
retry_count = retry_count + 1
WHERE id = ? AND retry_count < 3";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('si', $error, $urlId);
$stmt->execute();
}
/**
* 添加URL到队列
*/
private function addUrlsToQueue($taskId, $urls) {
$sql = "INSERT IGNORE INTO crawler_urls (task_id, url) VALUES (?, ?)";
$stmt = $this->db->prepare($sql);
foreach ($urls as $url) {
$stmt->bind_param('is', $taskId, $url);
$stmt->execute();
}
}
/**
* 获取任务信息
*/
public function getTask($taskId) {
$sql = "SELECT * FROM crawler_tasks WHERE id = ?";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('i', $taskId);
$stmt->execute();
return $stmt->get_result()->fetch_assoc();
}
/**
* 更新任务状态
*/
private function updateTaskStatus($taskId, $status) {
$sql = "UPDATE crawler_tasks SET status = ?, last_run_at = NOW() WHERE id = ?";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('si', $status, $taskId);
$stmt->execute();
}
/**
* 统计队列数量
*/
private function getQueueCount($taskId) {
$sql = "SELECT COUNT(*) as count FROM crawler_urls WHERE task_id = ?";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('i', $taskId);
$stmt->execute();
$result = $stmt->get_result()->fetch_assoc();
return $result['count'];
}
/**
* 统计待处理任务数
*/
private function getPendingCount($taskId) {
$sql = "SELECT COUNT(*) as count FROM crawler_urls
WHERE task_id = ? AND status = 'pending'";
$stmt = $this->db->prepare($sql);
$stmt->bind_param('i', $taskId);
$stmt->execute();
$result = $stmt->get_result()->fetch_assoc();
return $result['count'];
}
/**
* 保存任务进度
*/
private function saveTaskProgress($taskId) {
// 记录当前进度
$pendingCount = $this->getPendingCount($taskId);
$totalCount = $this->getQueueCount($taskId);
$this->logger->info("Task {$taskId}: Saved progress. {$pendingCount}/{$totalCount} pending");
}
}
使用示例
基础使用
<?php
// 数据库连接
$db = new mysqli('localhost', 'username', 'password', 'database');
// 创建爬虫管理器
$manager = new CrawlerManager($db);
// 创建任务
$taskId = $manager->createTask([
'name' => 'Example Site Crawler',
'target_url' => 'https://example.com',
'max_pages' => 100,
'interval_time' => 2
]);
// 启动任务
$manager->startTask($taskId);
// 暂停任务
// $manager->pauseTask($taskId);
// 获取任务状态
$task = $manager->getTask($taskId);
echo $task['status'];
简单爬虫示例
<?php
require_once 'SimpleCrawler.php';
$crawler = new SimpleCrawler([
'userAgent' => 'MyCustomBot/1.0',
'timeout' => 10,
'proxy' => 'http://127.0.0.1:8080' // 可选
]);
// 登录后爬取
$loginData = [
'username' => 'user',
'password' => 'password'
];
$result = $crawler->post('https://example.com/login', $loginData);
if ($result['success']) {
// 登录成功后再爬取
$page = $crawler->get('https://example.com/protected-page');
if ($page['success']) {
echo $page['content'];
}
}
命令行管理工具
<?php
// crawler.php
class CrawlerCLI {
private $manager;
public function __construct($db) {
$this->manager = new CrawlerManager($db);
}
public function handle($command, $args) {
switch ($command) {
case 'start':
return $this->startCrawler($args[0]);
case 'stop':
return $this->stopCrawler($args[0]);
case 'status':
return $this->getStatus($args[0]);
case 'stats':
return $this->getStats();
case 'list':
return $this->listTasks();
default:
return $this->help();
}
}
private function startCrawler($taskId) {
$result = $this->manager->startTask($taskId);
return $result ? "Task started" : "Failed to start task";
}
private function stopCrawler($taskId) {
$result = $this->manager->pauseTask($taskId);
return $result ? "Task stopped" : "Failed to stop task";
}
private function getStatus($taskId) {
$task = $this->manager->getTask($taskId);
return json_encode($task);
}
private function getStats() {
// 获取统计数据
$stats = [
'total_tasks' => $this->countTasks(),
'running_tasks' => $this->countRunningTasks(),
'total_pages' => $this->countPages()
];
return json_encode($stats);
}
private function listTasks() {
$tasks = $this->getAllTasks();
return json_encode($tasks);
}
}
最佳实践建议
使用队列系统
// 使用Redis队列
$redis = new Redis();
$redis->connect('127.0.0.1', 6379);
// 添加URL
$redis->lpush('crawler_queue', $url);
// 获取URL
$url = $redis->rpop('crawler_queue');
使用框架或库
- Goutte - 基于Guzzle的Web爬虫框架
- Symfony DomCrawler - 灵活的DOM操作
- PHP Simple HTML DOM Parser - 简单的HTML解析
异常处理
try {
// 爬虫逻辑
} catch (ConnectException $e) {
// 连接错误
} catch (RequestException $e) {
// 请求错误
} catch (\Exception $e) {
// 其他错误
}
遵守robots.txt
// 检查robots.txt
$robots = file_get_contents('https://example.com/robots.txt');
if (strpos($robots, 'Disallow: /crawl') !== false) {
// 不爬取该路径
}
这个完整的爬虫管理系统包含了：
- 基础爬虫类
- 任务管理（创建、启动、暂停、恢复）
- 并发控制
- 代理支持
- Cookie管理
- 错误处理
- 数据库存储
- 命令行工具
你可以根据实际需求进行调整和扩展。
