本文目录导读：
我来帮你搭建一个PHP简易爬虫框架,我们需要设计一个灵活且可扩展的架构。
项目结构
crawler/
├── src/
│ ├── Core/
│ │ ├── Crawler.php
│ │ ├── Request.php
│ │ └── Response.php
│ ├── Parser/
│ │ ├── ParserInterface.php
│ │ └── HTMLParser.php
│ ├── Storage/
│ │ ├── StorageInterface.php
│ │ └── DatabaseStorage.php
│ └── Utils/
│ └── Helper.php
├── config/
│ └── config.php
├── tests/
└── composer.json核心类实现
Request.php
<?php
namespace Crawler\Core;
class Request {
private $url;
private $method;
private $headers = [];
private $body = [];
private $options = [];
private $timeout = 30;
private $maxRetries = 3;
private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
private $proxy = null;
public function __construct($url, $method = 'GET', $headers = [], $body = []) {
$this->url = $url;
$this->method = $method;
$this->headers = $headers;
$this->body = $body;
}
public function getUrl() { return $this->url; }
public function setUrl($url) { $this->url = $url; return $this; }
public function getMethod() { return $this->method; }
public function setMethod($method) { $this->method = $method; return $this; }
public function addHeader($key, $value) {
$this->headers[$key] = $value;
return $this;
}
public function getHeaders() { return $this->headers; }
public function setBody($data) {
$this->body = $data;
return $this;
}
public function getBody() { return $this->body; }
public function setTimeout($timeout) { $this->timeout = $timeout; return $this; }
public function getTimeout() { return $this->timeout; }
public function setUserAgent($ua) { $this->userAgent = $ua; return $this; }
public function getUserAgent() { return $this->userAgent; }
public function setProxy($proxy) { $this->proxy = $proxy; return $this; }
public function getProxy() { return $this->proxy; }
public function toCurlOptions() {
$curlOptions = [
CURLOPT_URL => $this->url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_TIMEOUT => $this->timeout,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_HEADER => false
];
// 设置请求头
if (!empty($this->headers)) {
$curlOptions[CURLOPT_HTTPHEADER] = $this->headers;
}
// 设置请求方法
switch (strtoupper($this->method)) {
case 'POST':
$curlOptions[CURLOPT_POST] = true;
$curlOptions[CURLOPT_POSTFIELDS] = $this->body;
break;
case 'PUT':
$curlOptions[CURLOPT_CUSTOMREQUEST] = 'PUT';
$curlOptions[CURLOPT_POSTFIELDS] = $this->body;
break;
case 'DELETE':
$curlOptions[CURLOPT_CUSTOMREQUEST] = 'DELETE';
break;
case 'HEAD':
$curlOptions[CURLOPT_NOBODY] = true;
break;
}
// 设置代理
if ($this->proxy) {
$curlOptions[CURLOPT_PROXY] = $this->proxy;
}
return $curlOptions;
}
}
Response.php
<?php
namespace Crawler\Core;
class Response {
private $statusCode;
private $headers = [];
private $body;
private $error;
private $executionTime;
private $finalUrl;
public function __construct($statusCode, $headers, $body, $error = null, $executionTime = 0, $finalUrl = null) {
$this->statusCode = $statusCode;
$this->headers = $headers;
$this->body = $body;
$this->error = $error;
$this->executionTime = $executionTime;
$this->finalUrl = $finalUrl;
}
public function getStatusCode() { return $this->statusCode; }
public function getHeaders() { return $this->headers; }
public function getBody() { return $this->body; }
public function getError() { return $this->error; }
public function getExecutionTime() { return $this->executionTime; }
public function getFinalUrl() { return $this->finalUrl; }
public function isSuccess() {
return $this->statusCode >= 200 && $this->statusCode < 300;
}
public function toArray() {
return [
'status_code' => $this->statusCode,
'headers' => $this->headers,
'body' => $this->body,
'error' => $this->error,
'execution_time' => $this->executionTime,
'final_url' => $this->finalUrl
];
}
}
Crawler.php
<?php
namespace Crawler\Core;
use Crawler\Parser\ParserInterface;
use Crawler\Storage\StorageInterface;
class Crawler {
protected $request;
protected $response;
protected $parser;
protected $storage;
protected $config = [];
// 爬取统计
protected $stats = [
'total_requests' => 0,
'total_pages' => 0,
'total_bytes' => 0,
'errors' => 0,
'start_time' => null,
'end_time' => null
];
// 回调函数
protected $callbacks = [
'onSuccess' => null,
'onError' => null,
'onParse' => null
];
public function __construct(array $config = []) {
$this->config = array_merge([
'max_retries' => 3,
'timeout' => 30,
'concurrency' => 1,
'use_random_user_agent' => false,
'respect_robots_txt' => true,
'user_agents' => $this->getDefaultUserAgents()
], $config);
$this->init();
}
protected function init() {
// 可以在这里初始化数据库连接等
}
public function setRequest(Request $request) {
$this->request = $request;
return $this;
}
public function setParser(ParserInterface $parser) {
$this->parser = $parser;
return $this;
}
public function setStorage(StorageInterface $storage) {
$this->storage = $storage;
return $this;
}
public function setCallback($event, $callback) {
$this->callbacks[$event] = $callback;
return $this;
}
public function fetch($url) {
$this->stats['start_time'] = microtime(true);
try {
$this->makeRequest($url);
if ($this->response->isSuccess()) {
$this->stats['total_pages']++;
$this->handleSuccess();
} else {
$this->stats['errors']++;
$this->handleError();
}
return $this->response;
} finally {
$this->stats['end_time'] = microtime(true);
}
}
protected function makeRequest($url) {
$startTime = microtime(true);
$retries = 0;
do {
$ch = curl_init();
$request = new Request($url);
if (!empty($this->request->getHeaders())) {
foreach ($this->request->getHeaders() as $key => $value) {
$request->addHeader($key, $value);
}
}
// 随机User-Agent
if ($this->config['use_random_user_agent']) {
$randomUA = $this->config['user_agents'][array_rand($this->config['user_agents'])];
$request->setUserAgent($randomUA);
} else {
$request->setUserAgent($this->request->getUserAgent());
}
$options = $request->toCurlOptions();
curl_setopt_array($ch, $options);
// 发送请求
$body = curl_exec($ch);
$error = curl_error($ch);
$info = curl_getinfo($ch);
curl_close($ch);
if ($error) {
$this->showMessage("Request error: " . $error, 'error');
$retries++;
if ($retries >= $this->config['max_retries']) {
break;
}
sleep(1); // 等待1秒后重试
continue;
}
break;
} while (true);
$executionTime = microtime(true) - $startTime;
$this->response = new Response(
isset($info['http_code']) ? $info['http_code'] : 0,
$this->parseHeaders($info),
$body,
$error,
$executionTime,
isset($info['url']) ? $info['url'] : $url
);
$this->stats['total_requests']++;
$this->stats['total_bytes'] += strlen($body);
}
protected function handleSuccess() {
// 输出响应信息
$this->showMessage("获取成功: " . $this->response->getFinalUrl(), 'success');
// 执行解析
if ($this->parser !== null) {
$data = $this->parser->parse($this->response->getBody());
if ($this->callbacks['onParse']) {
$data = call_user_func($this->callbacks['onParse'], $data, $this->response);
}
// 存储数据
if ($this->storage !== null && $data) {
$this->storage->save($data);
}
}
// 执行成功回调
if ($this->callbacks['onSuccess']) {
call_user_func($this->callbacks['onSuccess'], $this->response);
}
}
protected function handleError() {
$this->showMessage("请求失败，状态码: " . $this->response->getStatusCode(), 'error');
if ($this->callbacks['onError']) {
call_user_func($this->callbacks['onError'], $this->response);
}
}
protected function parseHeaders($info) {
// 这个方法需要从curl响应中提取headers
// 简化实现，实际可以从$info获取
return [];
}
public function getStats() {
$this->stats['duration'] = $this->stats['end_time'] - $this->stats['start_time'];
return $this->stats;
}
protected function showMessage($message, $type = 'info') {
if (PHP_SAPI === 'cli') {
$colors = [
'success' => "\033[32m",
'error' => "\033[31m",
'info' => "\033[33m",
'reset' => "\033[0m"
];
echo $colors[$type] . $message . $colors['reset'] . PHP_EOL;
} else {
echo $message . "<br>";
}
}
protected function getDefaultUserAgents() {
return [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15'
];
}
}
解析器接口和实现
ParserInterface.php
<?php
namespace Crawler\Parser;
interface ParserInterface {
public function parse($html);
}
HTMLParser.php
<?php
namespace Crawler\Parser;
use DOMDocument;
use DOMXPath;
class HTMLParser implements ParserInterface {
protected $xpath = null;
protected $config = [];
public function __construct($config = []) {
$this->config = array_merge([
'selectors' => [],
'remove_tags' => [],
'encoding' => 'UTF-8'
], $config);
}
public function parse($html) {
$document = new DOMDocument();
libxml_use_internal_errors(true);
@$document->loadHTML('<?xml encoding="' . $this->config['encoding'] . '">' . $html);
libxml_clear_errors();
$this->xpath = new DOMXPath($document);
$result = [];
foreach ($this->config['selectors'] as $name => $selector) {
$result[$name] = $this->extractBySelector($selector);
}
return $result;
}
protected function extractBySelector($selector) {
// 支持简单的CSS选择器转XPath
$xpath = $this->cssToXPath($selector['selector']);
if (isset($selector['attribute'])) {
return $this->getAttribute($xpath, $selector['attribute']);
} else {
return $this->getText($xpath);
}
}
protected function cssToXPath($css) {
// 简单实现，可根据需要扩展
// 这里只处理最基本的后代选择器
$parts = explode(' ', trim($css));
$xpath = '';
foreach ($parts as $part) {
$xpath .= '/' . $part;
}
return $xpath;
}
protected function getText($xpath) {
$nodes = $this->xpath->query($xpath);
$texts = [];
foreach ($nodes as $node) {
if ($node->nodeType === XML_TEXT_NODE) {
$texts[] = trim($node->nodeValue);
} else {
$texts[] = trim($node->textContent);
}
}
return $texts;
}
protected function getAttribute($xpath, $attribute) {
$nodes = $this->xpath->query($xpath);
$values = [];
foreach ($nodes as $node) {
if ($node->nodeType === XML_ELEMENT_NODE) {
$values[] = $node->getAttribute($attribute);
}
}
return $values;
}
}
存储接口和实现
StorageInterface.php
<?php
namespace Crawler\Storage;
interface StorageInterface {
public function save(array $data);
public function get();
public function clear();
public function count();
}
DatabaseStorage.php
<?php
namespace Crawler\Storage;
class DatabaseStorage implements StorageInterface {
protected $pdo;
protected $table;
public function __construct($host, $dbname, $user, $password, $table) {
try {
$this->pdo = new \PDO(
"mysql:host=$host;dbname=$dbname;charset=utf8mb4",
$user,
$password,
[\PDO::ATTR_ERRMODE => \PDO::ERRMODE_EXCEPTION]
);
$this->table = $table;
} catch (\PDOException $e) {
throw new \Exception("数据库连接失败: " . $e->getMessage());
}
}
public function save(array $data) {
$columns = array_keys($data);
$placeholders = array_fill(0, count($columns), '?');
$sql = "INSERT INTO " . $this->table . " (" . implode(', ', $columns) . ")
VALUES (" . implode(', ', $placeholders) . ")";
try {
$stmt = $this->pdo->prepare($sql);
$stmt->execute(array_values($data));
return $this->pdo->lastInsertId();
} catch (\PDOException $e) {
throw new \Exception("数据存储失败: " . $e->getMessage());
}
}
public function get() {
$stmt = $this->pdo->query("SELECT * FROM " . $this->table);
return $stmt->fetchAll(\PDO::FETCH_ASSOC);
}
public function clear() {
$this->pdo->query("DELETE FROM " . $this->table);
return true;
}
public function count() {
$stmt = $this->pdo->query("SELECT COUNT(*) FROM " . $this->table);
return $stmt->fetchColumn();
}
}
配置文件
config.php
<?php
return [
'settings' => [
'max_retries' => 3,
'timeout' => 30,
'concurrency' => 1,
'use_random_user_agent' => true,
'respect_robots_txt' => true
],
'database' => [
'host' => 'localhost',
'dbname' => 'crawler',
'user' => 'root',
'password' => '',
'table' => 'crawled_data'
],
'output' => [
'format' => 'json', // json, csv, database
'path' => __DIR__ . '/data/'
],
'selectors' => [
'title' => [
'selector' => 'h1',
'attribute' => null // null表示获取文本
],
'links' => [
'selector' => 'a',
'attribute' => 'href'
],
'content' => [
'selector' => '.content',
'attribute' => null
]
]
];
使用示例
基本使用 (examples/basic.php)
<?php
require_once __DIR__ . '/vendor/autoload.php';
use Crawler\Core\Crawler;
use Crawler\Core\Request;
use Crawler\Parser\HTMLParser;
use Crawler\Storage\DatabaseStorage;
// 加载配置
$config = require __DIR__ . '/../config/config.php';
// 创建爬虫实例
$crawler = new Crawler($config['settings']);
// 设置请求
$request = new Request('https://example.com');
$request->setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64)');
$crawler->setRequest($request);
// 设置解析器
$parser = new HTMLParser($config['selectors']);
$crawler->setParser($parser);
// 设置存储
$storage = new DatabaseStorage(
$config['database']['host'],
$config['database']['dbname'],
$config['database']['user'],
$config['database']['password'],
$config['database']['table']
);
$crawler->setStorage($storage);
// 设置回调
$crawler->setCallback('onSuccess', function($response) {
echo "成功获取页面: " . $response->getUrl() . " (耗时: " . $response->getExecutionTime() . "s)\n";
});
$crawler->setCallback('onError', function($response) {
echo "页面获取失败: " . $response->getUrl() . " (错误: " . $response->getError() . ")\n";
});
// 执行爬取
$response = $crawler->fetch('https://example.com');
// 获取统计信息
$stats = $crawler->getStats();
print_r($stats);
Composer 配置文件
composer.json
{
"name": "mycrawler/mycrawler",
"description": "PHP微爬虫框架",
"type": "library",
"require": {
"php": ">=7.4",
"ext-curl": "*",
"ext-dom": "*",
"ext-pdo": "*"
},
"autoload": {
"psr-4": {
"Crawler\\": "src/"
}
},
"config": {
"optimize-autoloader": true
}
}
高级功能
多线程爬取 (ConcurrentCrawler)
<?php
namespace Crawler\Core;
use parallel\Runtime;
class ConcurrentCrawler {
protected $config;
protected $crawlers = [];
public function __construct($config = []) {
$this->config = array_merge([
'max_crawlers' => 5,
'max_requests_per_crawler' => 100
], $config);
}
public function crawl($urls) {
$chunks = array_chunk($urls, ceil(count($urls) / $this->config['max_crawlers']));
foreach ($chunks as $chunk) {
$runtime = new Runtime();
$this->crawlers[] = $runtime->run(function($urls, $config) {
// 每个爬虫实例的代码
require_once 'path/to/vendor/autoload.php';
$crawler = new Crawler($config);
foreach ($urls as $url) {
$crawler->fetch($url);
}
}, [$chunk, $this->config]);
}
// 等待所有爬虫完成
foreach ($this->crawlers as $crawler) {
$crawler->join();
}
}
}
测试文件
tests/UnitTest.php
<?php
use PHPUnit\Framework\TestCase;
use Crawler\Core\Crawler;
use Crawler\Core\Request;
class CrawlerTest extends TestCase {
public function testFetchSuccess() {
$crawler = new Crawler(['max_retries' => 3]);
$request = new Request('https://example.com');
$crawler->setRequest($request);
$response = $crawler->fetch('https://example.com');
$this->assertTrue($response->isSuccess());
$this->assertNotEmpty($response->getBody());
}
public function testFetchFailure() {
$crawler = new Crawler(['max_retries' => 1]);
$response = $crawler->fetch('https://invalid.example.com');
$this->assertNotTrue($response->isSuccess());
}
}
运行命令
# 生成自动加载文件 composer dump-autoload # 运行测试 vendor/bin/phpunit tests/ # 运行基本示例 php examples/basic.php
扩展建议
- 添加更多解析器：支持CSS选择器、XPath、JSONPath等
- 缓存功能：使用Redis/Memcached缓存已爬取的数据
- 断点续爬：记录爬取状态，支持中断后继续
- 数据清洗：添加数据过滤、去重、清洗功能
- 反爬处理：支持IP池、代理池、验证码识别等
- 队列系统：集成Redis队列实现任务分发
- 监控日志：添加详细的日志和监控功能
这个框架设计灵活，你可以根据实际需求进行扩展和调整，核心的Crawler类已经实现了基本功能,你可以在此基础上添加更多高级特性。
