本文目录导读：
我来为您提供一个完整的PHP抓取网页内容并存入数据库的解决方案：
数据库准备
CREATE TABLE IF NOT EXISTS `scraped_data` ( `id` INT AUTO_INCREMENT PRIMARY KEY, `url` VARCHAR(500) NOT NULL, VARCHAR(500), `content` TEXT, `images` TEXT, `links` TEXT, `scraped_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP, INDEX `idx_url` (`url`(191)) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
基础抓取类
<?php
class WebScraper {
private $pdo;
private $timeout = 30;
private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
public function __construct($pdo) {
$this->pdo = $pdo;
}
/**
* 抓取网页内容
* @param string $url 目标URL
* @return array 抓取的数据
*/
public function scrape($url) {
try {
// 获取网页内容
$html = $this->fetchContent($url);
if (!$html) {
throw new Exception('无法获取网页内容');
}
// 解析网页内容
$data = $this->parseContent($html, $url);
// 保存到数据库
$this->saveToDatabase($data);
return $data;
} catch (Exception $e) {
error_log('抓取失败: ' . $e->getMessage());
return false;
}
}
/**
* 获取网页内容
*/
private function fetchContent($url) {
$ch = curl_init();
$options = [
CURLOPT_URL => $url,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 5,
CURLOPT_TIMEOUT => $this->timeout,
CURLOPT_CONNECTTIMEOUT => 10,
CURLOPT_USERAGENT => $this->userAgent,
CURLOPT_SSL_VERIFYPEER => false,
CURLOPT_ENCODING => '', // 支持压缩
CURLOPT_HTTPHEADER => [
'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8',
],
];
curl_setopt_array($ch, $options);
$content = curl_exec($ch);
$httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
if (curl_errno($ch)) {
error_log('CURL错误: ' . curl_error($ch));
return false;
}
curl_close($ch);
if ($httpCode !== 200) {
error_log("HTTP状态码: $httpCode");
return false;
}
return $content;
}
/**
* 解析网页内容
*/
private function parseContent($html, $url) {
$dom = new DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML('<?xml encoding="UTF-8">' . $html);
libxml_clear_errors();
$xpath = new DOMXPath($dom);
// 提取标题
$titleNode = $xpath->query('//title');
$title = $titleNode->length > 0 ? trim($titleNode->item(0)->textContent) : '';
// 提取主体内容（可根据需要调整选择器）
$contentNode = $xpath->query('//article | //main | //div[@class="content"] | //div[@id="content"]');
$content = '';
if ($contentNode->length > 0) {
$content = $this->cleanContent($contentNode->item(0)->textContent);
} else {
// 回退到body内容
$bodyNode = $dom->getElementsByTagName('body');
if ($bodyNode->length > 0) {
$content = $this->cleanContent($bodyNode->item(0)->textContent);
}
}
// 提取图片
$images = [];
$imgNodes = $dom->getElementsByTagName('img');
foreach ($imgNodes as $img) {
$src = $img->getAttribute('src');
if ($src && filter_var($src, FILTER_VALIDATE_URL) === false) {
$src = $this->resolveUrl($url, $src);
}
if ($src) {
$images[] = $src;
}
}
// 提取链接
$links = [];
$linkNodes = $dom->getElementsByTagName('a');
foreach ($linkNodes as $link) {
$href = $link->getAttribute('href');
if ($href && !preg_match('/^(javascript|mailto|tel):/i', $href)) {
if (filter_var($href, FILTER_VALIDATE_URL) === false) {
$href = $this->resolveUrl($url, $href);
}
$links[] = [
'text' => trim($link->textContent),
'href' => $href
];
}
}
return [
'url' => $url,
'title' => $title,
'content' => $content,
'images' => json_encode($images),
'links' => json_encode($links),
'scraped_at' => date('Y-m-d H:i:s')
];
}
/**
* 清理内容
*/
private function cleanContent($content) {
// 去除多余空白
$content = preg_replace('/\s+/', ' ', $content);
// 去除特殊字符
$content = trim($content);
return $content;
}
/**
* 解析相对URL为绝对URL
*/
private function resolveUrl($baseUrl, $relativeUrl) {
// 解析基础URL
$parsedBase = parse_url($baseUrl);
// 如果是完整的URL则直接返回
if (filter_var($relativeUrl, FILTER_VALIDATE_URL)) {
return $relativeUrl;
}
// 处理相对URL
if (strpos($relativeUrl, '//') === 0) {
return $parsedBase['scheme'] . ':' . $relativeUrl;
}
if (strpos($relativeUrl, '/') === 0) {
return $parsedBase['scheme'] . '://' . $parsedBase['host'] . $relativeUrl;
}
$path = isset($parsedBase['path']) ? $parsedBase['path'] : '/';
$path = preg_replace('/[^\/]*$/', '', $path);
return $parsedBase['scheme'] . '://' . $parsedBase['host'] . $path . $relativeUrl;
}
/**
* 保存到数据库
*/
private function saveToDatabase($data) {
try {
$stmt = $this->pdo->prepare(
"INSERT INTO scraped_data (url, title, content, images, links, scraped_at)
VALUES (:url, :title, :content, :images, :links, :scraped_at)"
);
return $stmt->execute($data);
} catch (PDOException $e) {
error_log('数据库保存失败: ' . $e->getMessage());
return false;
}
}
/**
* 检查URL是否已存在
*/
public function isUrlExists($url) {
$stmt = $this->pdo->prepare("SELECT COUNT(*) FROM scraped_data WHERE url = :url");
$stmt->execute([':url' => $url]);
return $stmt->fetchColumn() > 0;
}
}
?>
使用示例
<?php
// 加载数据库配置
require_once 'config.php';
// 创建PDO连接
try {
$pdo = new PDO(
"mysql:host=$db_host;dbname=$db_name;charset=utf8mb4",
$db_user,
$db_pass,
[
PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
]
);
} catch (PDOException $e) {
die("数据库连接失败: " . $e->getMessage());
}
$scraper = new WebScraper($pdo);
// 要抓取的URL列表
$urls = [
'https://example.com/page1',
'https://example.com/page2',
'https://example.com/page3',
];
foreach ($urls as $url) {
// 检查是否已存在
if ($scraper->isUrlExists($url)) {
echo "URL已存在，跳过: $url\n";
continue;
}
echo "开始抓取: $url\n";
$result = $scraper->scrape($url);
if ($result) {
echo "成功抓取并保存: {$result['title']}\n";
} else {
echo "抓取失败: $url\n";
}
// 避免请求过快
usleep(rand(500000, 1000000)); // 0.5-1秒延迟
}
?>
配置文件示例 (config.php)
<?php
$db_host = 'localhost';
$db_name = 'web_scraper';
$db_user = 'root';
$db_pass = 'your_password';
// 可选：代理配置
$proxy_config = [
'proxy_type' => CURLPROXY_HTTP,
'proxy_host' => '127.0.0.1',
'proxy_port' => 8080,
'proxy_user' => '',
'proxy_pass' => '',
];
// 可选：多线程配置（需要pcntl扩展）
$threads = 5;
?>
增强功能（可选）
支持Cookie和会话
private function fetchContentWithCookies($url, $cookies) {
$cookieFile = tempnam('/tmp', 'cookie_');
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_COOKIE, $cookies);
curl_setopt($ch, CURLOPT_COOKIEJAR, $cookieFile);
curl_setopt($ch, CURLOPT_COOKIEFILE, $cookieFile);
// ... 其他设置
}
支持JavaScript渲染
// 使用无头浏览器（需要安装Puppeteer）
public function scrapeWithHeadlessBrowser($url) {
$html = shell_exec("node render.js \"$url\"");
// 然后解析HTML
}
注意事项
- 遵守robots.txt和网站服务条款
- 设置适当的请求延迟，避免给目标网站造成压力
- 处理编码问题，确保正确解析UTF-8、GBK等编码
- 错误处理和日志记录很重要
- 数据库连接使用PDO以增强安全性
- 定期清理不需要的数据
这个解决方案已经包含常用的网页抓取功能,您可以根据具体需求进行调整和扩展。
