PHP 抓取结果存入数据库

wen PHP项目 2026-08-10 138

本文目录导读：

PHP 抓取结果存入数据库

数据库准备
基础抓取类
使用示例
配置文件示例 (config.php)
增强功能（可选）
注意事项

我来为您提供一个完整的PHP抓取网页内容并存入数据库的解决方案：

数据库准备

CREATE TABLE IF NOT EXISTS `scraped_data` (
  `id` INT AUTO_INCREMENT PRIMARY KEY,
  `url` VARCHAR(500) NOT NULL, VARCHAR(500),
  `content` TEXT,
  `images` TEXT,
  `links` TEXT,
  `scraped_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
  INDEX `idx_url` (`url`(191))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;

基础抓取类

<?php
class WebScraper {
    private $pdo;
    private $timeout = 30;
    private $userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36';
    public function __construct($pdo) {
        $this->pdo = $pdo;
    }
    /**
     * 抓取网页内容
     * @param string $url 目标URL
     * @return array 抓取的数据
     */
    public function scrape($url) {
        try {
            // 获取网页内容
            $html = $this->fetchContent($url);
            if (!$html) {
                throw new Exception('无法获取网页内容');
            }
            // 解析网页内容
            $data = $this->parseContent($html, $url);
            // 保存到数据库
            $this->saveToDatabase($data);
            return $data;
        } catch (Exception $e) {
            error_log('抓取失败: ' . $e->getMessage());
            return false;
        }
    }
    /**
     * 获取网页内容
     */
    private function fetchContent($url) {
        $ch = curl_init();
        $options = [
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_MAXREDIRS => 5,
            CURLOPT_TIMEOUT => $this->timeout,
            CURLOPT_CONNECTTIMEOUT => 10,
            CURLOPT_USERAGENT => $this->userAgent,
            CURLOPT_SSL_VERIFYPEER => false,
            CURLOPT_ENCODING => '', // 支持压缩
            CURLOPT_HTTPHEADER => [
                'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8',
            ],
        ];
        curl_setopt_array($ch, $options);
        $content = curl_exec($ch);
        $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        if (curl_errno($ch)) {
            error_log('CURL错误: ' . curl_error($ch));
            return false;
        }
        curl_close($ch);
        if ($httpCode !== 200) {
            error_log("HTTP状态码: $httpCode");
            return false;
        }
        return $content;
    }
    /**
     * 解析网页内容
     */
    private function parseContent($html, $url) {
        $dom = new DOMDocument();
        libxml_use_internal_errors(true);
        $dom->loadHTML('<?xml encoding="UTF-8">' . $html);
        libxml_clear_errors();
        $xpath = new DOMXPath($dom);
        // 提取标题
        $titleNode = $xpath->query('//title');
        $title = $titleNode->length > 0 ? trim($titleNode->item(0)->textContent) : '';
        // 提取主体内容（可根据需要调整选择器）
        $contentNode = $xpath->query('//article | //main | //div[@class="content"] | //div[@id="content"]');
        $content = '';
        if ($contentNode->length > 0) {
            $content = $this->cleanContent($contentNode->item(0)->textContent);
        } else {
            // 回退到body内容
            $bodyNode = $dom->getElementsByTagName('body');
            if ($bodyNode->length > 0) {
                $content = $this->cleanContent($bodyNode->item(0)->textContent);
            }
        }
        // 提取图片
        $images = [];
        $imgNodes = $dom->getElementsByTagName('img');
        foreach ($imgNodes as $img) {
            $src = $img->getAttribute('src');
            if ($src && filter_var($src, FILTER_VALIDATE_URL) === false) {
                $src = $this->resolveUrl($url, $src);
            }
            if ($src) {
                $images[] = $src;
            }
        }
        // 提取链接
        $links = [];
        $linkNodes = $dom->getElementsByTagName('a');
        foreach ($linkNodes as $link) {
            $href = $link->getAttribute('href');
            if ($href && !preg_match('/^(javascript|mailto|tel):/i', $href)) {
                if (filter_var($href, FILTER_VALIDATE_URL) === false) {
                    $href = $this->resolveUrl($url, $href);
                }
                $links[] = [
                    'text' => trim($link->textContent),
                    'href' => $href
                ];
            }
        }
        return [
            'url' => $url,
            'title' => $title,
            'content' => $content,
            'images' => json_encode($images),
            'links' => json_encode($links),
            'scraped_at' => date('Y-m-d H:i:s')
        ];
    }
    /**
     * 清理内容
     */
    private function cleanContent($content) {
        // 去除多余空白
        $content = preg_replace('/\s+/', ' ', $content);
        // 去除特殊字符
        $content = trim($content);
        return $content;
    }
    /**
     * 解析相对URL为绝对URL
     */
    private function resolveUrl($baseUrl, $relativeUrl) {
        // 解析基础URL
        $parsedBase = parse_url($baseUrl);
        // 如果是完整的URL则直接返回
        if (filter_var($relativeUrl, FILTER_VALIDATE_URL)) {
            return $relativeUrl;
        }
        // 处理相对URL
        if (strpos($relativeUrl, '//') === 0) {
            return $parsedBase['scheme'] . ':' . $relativeUrl;
        }
        if (strpos($relativeUrl, '/') === 0) {
            return $parsedBase['scheme'] . '://' . $parsedBase['host'] . $relativeUrl;
        }
        $path = isset($parsedBase['path']) ? $parsedBase['path'] : '/';
        $path = preg_replace('/[^\/]*$/', '', $path);
        return $parsedBase['scheme'] . '://' . $parsedBase['host'] . $path . $relativeUrl;
    }
    /**
     * 保存到数据库
     */
    private function saveToDatabase($data) {
        try {
            $stmt = $this->pdo->prepare(
                "INSERT INTO scraped_data (url, title, content, images, links, scraped_at) 
                 VALUES (:url, :title, :content, :images, :links, :scraped_at)"
            );
            return $stmt->execute($data);
        } catch (PDOException $e) {
            error_log('数据库保存失败: ' . $e->getMessage());
            return false;
        }
    }
    /**
     * 检查URL是否已存在
     */
    public function isUrlExists($url) {
        $stmt = $this->pdo->prepare("SELECT COUNT(*) FROM scraped_data WHERE url = :url");
        $stmt->execute([':url' => $url]);
        return $stmt->fetchColumn() > 0;
    }
}
?>

使用示例

<?php
// 加载数据库配置
require_once 'config.php';
// 创建PDO连接
try {
    $pdo = new PDO(
        "mysql:host=$db_host;dbname=$db_name;charset=utf8mb4",
        $db_user,
        $db_pass,
        [
            PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
            PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
        ]
    );
} catch (PDOException $e) {
    die("数据库连接失败: " . $e->getMessage());
}
$scraper = new WebScraper($pdo);
// 要抓取的URL列表
$urls = [
    'https://example.com/page1',
    'https://example.com/page2',
    'https://example.com/page3',
];
foreach ($urls as $url) {
    // 检查是否已存在
    if ($scraper->isUrlExists($url)) {
        echo "URL已存在，跳过: $url\n";
        continue;
    }
    echo "开始抓取: $url\n";
    $result = $scraper->scrape($url);
    if ($result) {
        echo "成功抓取并保存: {$result['title']}\n";
    } else {
        echo "抓取失败: $url\n";
    }
    // 避免请求过快
    usleep(rand(500000, 1000000)); // 0.5-1秒延迟
}
?>

配置文件示例 (config.php)

<?php
$db_host = 'localhost';
$db_name = 'web_scraper';
$db_user = 'root';
$db_pass = 'your_password';
// 可选：代理配置
$proxy_config = [
    'proxy_type' => CURLPROXY_HTTP,
    'proxy_host' => '127.0.0.1',
    'proxy_port' => 8080,
    'proxy_user' => '',
    'proxy_pass' => '',
];
// 可选：多线程配置（需要pcntl扩展）
$threads = 5;
?>

增强功能（可选）

支持Cookie和会话

private function fetchContentWithCookies($url, $cookies) {
    $cookieFile = tempnam('/tmp', 'cookie_');
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_COOKIE, $cookies);
    curl_setopt($ch, CURLOPT_COOKIEJAR, $cookieFile);
    curl_setopt($ch, CURLOPT_COOKIEFILE, $cookieFile);
    // ... 其他设置
}

支持JavaScript渲染

// 使用无头浏览器（需要安装Puppeteer）
public function scrapeWithHeadlessBrowser($url) {
    $html = shell_exec("node render.js \"$url\"");
    // 然后解析HTML
}

注意事项

遵守robots.txt和网站服务条款
设置适当的请求延迟，避免给目标网站造成压力
处理编码问题，确保正确解析UTF-8、GBK等编码
错误处理和日志记录很重要
数据库连接使用PDO以增强安全性
定期清理不需要的数据

这个解决方案已经包含常用的网页抓取功能,您可以根据具体需求进行调整和扩展。

本文地址： https://bmjup.com/post/42958.html