IT技术社区_专业开发者平台_最新编程教程与解决方案 -沐辰社区

PHP项目日志聚合与ELK

wen PHP项目 1

PHP项目日志聚合与ELK实践指南

PHP应用 → 日志文件/标准输出 → Filebeat → Logstash → Elasticsearch → Kibana
                                   ↓
                               Redis/Kafka (缓冲层，可选)

PHP端日志配置

基础日志配置 (Monolog)

// composer.json
{
    "require": {
        "monolog/monolog": "^3.0"
    }
}
// logger.php
use Monolog\Logger;
use Monolog\Handler\StreamHandler;
use Monolog\Formatter\JsonFormatter;
$logger = new Logger('app');
$handler = new StreamHandler('/var/log/php-app/app.log', Logger::INFO);
$handler->setFormatter(new JsonFormatter());
$logger->pushHandler($handler);

结构化日志格式

<?php
// 使用结构化日志
class StructuredLogger
{
    private $logger;
    public function info($message, array $context = [])
    {
        $context['@timestamp'] = date('c');
        $context['app_name'] = 'my_app';
        $context['environment'] = $_ENV['APP_ENV'] ?? 'production';
        $context['request_id'] = $this->getRequestId();
        $this->logger->info($message, $context);
    }
    private function getRequestId()
    {
        // 从请求头获取或生成
        return $_SERVER['HTTP_X_REQUEST_ID'] ?? uniqid('req_', true);
    }
}

Filebeat配置

基本配置 (filebeat.yml)

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/php-app/*.log
  fields:
    app_type: php
    environment: production
  fields_under_root: true
  # 多行日志处理
  multiline.pattern: '^\d{4}-\d{2}-\d{2}'
  multiline.negate: true
  multiline.match: after
output.logstash:
  hosts: ["logstash:5044"]
# 输出到Elasticsearch (可选)
#output.elasticsearch:
#  hosts: ["elasticsearch:9200"]
#  index: "php-app-%{+yyyy.MM.dd}"

高级配置

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/php-app/*.log
  # 文件轮转处理
  close_inactive: 5m
  clean_inactive: 72h
  ignore_older: 48h
  # 包含/排除规则
  include_lines: ['^(INFO|ERROR|WARNING|DEBUG)']
  exclude_lines: ['^DEBUG.*health_check']
  # 字段处理
  processors:
    - add_cloud_metadata: ~
    - add_host_metadata: ~
    - add_docker_metadata: ~
    - decode_json_fields:
        fields: ["message"]
        target: "json"
        overwrite_keys: true

Logstash配置

Pipeline配置

# logstash/pipeline/php-app.conf
input {
  beats {
    port => 5044
    client_inactivity_timeout => 60
    ssl => false
  }
  # 备用：直接读取日志文件
  file {
    path => "/var/log/php-app/*.log"
    start_position => "beginning"
    sincedb_path => "/dev/null"
  }
}
filter {
  # 解析JSON日志
  json {
    source => "message"
    target => "parsed_json"
    skip_on_invalid_json => true
  }
  # 如果JSON解析成功，提取字段
  if [parsed_json] {
    mutate {
      replace => { "[message]" => "%{[parsed_json][message]}" }
      add_field => {
        "log_level" => "%{[parsed_json][level]}"
        "php_channel" => "%{[parsed_json][channel]}"
        "php_extra" => "%{[parsed_json][context]}"
      }
      remove_field => ["parsed_json"]
    }
  }
  # 日期解析
  date {
    match => ["timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss"]
    target => "@timestamp"
  }
  # 解析PHP错误格式
  grok {
    match => { "message" => [
      # PHP Fatal error
      '\[%{TIMESTAMP_ISO8601:timestamp}\] PHP %{WORD:php_error_type}: %{GREEDYDATA:php_error_msg} in %{PATH:php_error_file} on line %{NUMBER:php_error_line}',
      # Stack trace
      '\#%{NUMBER:stack_level} %{PATH:stack_file}\(%{NUMBER:stack_line}\): %{GREEDYDATA:stack_call}',
      # General PHP error
      '\[%{TIMESTAMP_ISO8601:timestamp}\] %{LOGLEVEL:log_level} %{GREEDYDATA:error_message}'
    ]}
  }
  # 地理IP解析
  geoip {
    source => "[client_ip]"
    target => "geoip"
  }
  # 用户代理解析
  useragent {
    source => "[user_agent]"
    target => "user_agent"
  }
}
output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "php-app-logs-%{+YYYY.MM.dd}"
    document_type => "log"
    template => "/etc/logstash/templates/php-app.json"
    template_name => "php-app"
    template_overwrite => true
  }
  # 调试输出
  stdout {
    codec => rubydebug
  }
}

Elasticsearch配置

索引模板

// elasticsearch/templates/php-app.json
{
  "index_patterns": ["php-app-logs-*"],
  "settings": {
    "number_of_shards": 2,
    "number_of_replicas": 1,
    "refresh_interval": "5s",
    "analysis": {
      "analyzer": {
        "php_error_analyzer": {
          "type": "custom",
          "tokenizer": "standard",
          "filter": ["lowercase", "stop"]
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "@timestamp": { "type": "date" },
      "message": { 
        "type": "text",
        "analyzer": "php_error_analyzer",
        "fields": {
          "keyword": { "type": "keyword" }
        }
      },
      "log_level": { "type": "keyword" },
      "php_error_type": { "type": "keyword" },
      "php_error_msg": { "type": "text" },
      "php_error_file": { 
        "type": "text",
        "fields": {
          "keyword": { "type": "keyword" }
        }
      },
      "php_error_line": { "type": "integer" },
      "request_id": { "type": "keyword" },
      "client_ip": { "type": "ip" },
      "geoip": {
        "properties": {
          "location": { "type": "geo_point" }
        }
      },
      "response_time": { "type": "float" },
      "memory_usage": { "type": "long" }
    }
  }
}

Docker部署方案

docker-compose.yml

version: '3.7'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.0
    environment:
      - discovery.type=single-node
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - xpack.security.enabled=false
      - xpack.monitoring.enabled=false
    ports:
      - "9200:9200"
    volumes:
      - es_data:/usr/share/elasticsearch/data
      - ./elasticsearch/templates:/etc/elasticsearch/templates
  logstash:
    image: docker.elastic.co/logstash/logstash:7.17.0
    ports:
      - "5044:5044"
    volumes:
      - ./logstash/pipeline:/usr/share/logstash/pipeline
      - ./logstash/templates:/etc/logstash/templates
    depends_on:
      - elasticsearch
  kibana:
    image: docker.elastic.co/kibana/kibana:7.17.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    depends_on:
      - elasticsearch
  filebeat:
    image: docker.elastic.co/beats/filebeat:7.17.0
    volumes:
      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
      - /var/log/php-app/:/var/log/php-app/:ro
    depends_on:
      - logstash
      - elasticsearch
  php-app:
    build: ./php-app
    volumes:
      - ./php-app/logs:/var/log/php-app
    depends_on:
      - filebeat
volumes:
  es_data:

PHP应用集成示例

Symfony框架集成

// config/packages/monolog.yaml
monolog:
    handlers:
        main:
            type: stream
            path: "%kernel.logs_dir%/%kernel.environment%.log"
            level: debug
            formatter: 'monolog.formatter.json'
        error:
            type: stream
            path: "%kernel.logs_dir%/error.log"
            level: error
            formatter: 'monolog.formatter.json'
        # ELK专用处理器
        elk:
            type: stream
            path: "%kernel.logs_dir%/elk.log"
            level: info
            formatter: 'App\Logging\ElkFormatter'
            channels: ["!event"]
// src/Logging/ElkFormatter.php
namespace App\Logging;
use Monolog\Formatter\JsonFormatter;
class ElkFormatter extends JsonFormatter
{
    public function format(array $record): string
    {
        $extra = [
            'app_name' => $_ENV['APP_NAME'] ?? 'my_app',
            'environment' => $_ENV['APP_ENV'] ?? 'dev',
            'request_id' => $_SERVER['HTTP_X_REQUEST_ID'] ?? uniqid(),
            'request_uri' => $_SERVER['REQUEST_URI'] ?? '',
            'request_method' => $_SERVER['REQUEST_METHOD'] ?? '',
            'response_time' => $this->getExecutionTime(),
            'memory_usage' => memory_get_usage(true),
        ];
        $record['extra'] = array_merge($record['extra'] ?? [], $extra);
        return parent::format($record);
    }
    private function getExecutionTime(): float
    {
        if (defined('LARAVEL_START')) {
            return microtime(true) - LARAVEL_START;
        }
        return 0;
    }
}

Laravel框架集成

// config/logging.php
'channels' => [
    'stack' => [
        'driver' => 'stack',
        'channels' => ['daily', 'elk'],
    ],
    'elk' => [
        'driver' => 'monolog',
        'handler' => Monolog\Handler\StreamHandler::class,
        'handler_with' => [
            'stream' => storage_path('logs/elk.log'),
        ],
        'formatter' => Monolog\Formatter\JsonFormatter::class,
        'level' => 'debug',
    ],
    // 自定义ELK处理器
    'elk_json' => [
        'driver' => 'monolog',
        'handler' => Monolog\Handler\StreamHandler::class,
        'handler_with' => [
            'stream' => storage_path('logs/elk-json.log'),
        ],
        'formatter' => App\Logging\ElkFormatter::class,
        'level' => 'debug',
    ],
],

Kibana仪表板配置

索引模式创建

# 通过API创建索引模式
curl -X POST "localhost:5601/api/saved_objects/index-pattern" \
  -H "kbn-xsrf: true" \
  -H "Content-Type: application/json" \
  -d '{
    "attributes": {
      "title": "php-app-logs-*",
      "timeFieldName": "@timestamp"
    }
  }'

可视化配置

// 错误类型饼图
{
  "attributes": {: "PHP Error Types Distribution",
    "visState": "{\"type\":\"pie\",\"params\":{\"addTooltip\":true,\"legendPosition\":\"right\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"php_error_type.keyword\",\"size\":5,\"order\":\"desc\"}}]}"
  }
}
// 日错误趋势图
{
  "attributes": {: "PHP Errors Over Time",
    "visState": "{\"type\":\"line\",\"params\":{},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"log_level.keyword\",\"size\":5}}]}"
  }
}

性能优化策略

日志缓冲

// 使用Redis作为缓冲层
class BufferedLogger
{
    private $redis;
    private $buffer = [];
    private $bufferSize = 100;
    public function log($level, $message, array $context = [])
    {
        $this->buffer[] = [
            'level' => $level,
            'message' => $message,
            'context' => $context,
            'timestamp' => microtime(true)
        ];
        if (count($this->buffer) >= $this->bufferSize) {
            $this->flush();
        }
    }
    public function flush()
    {
        if (empty($this->buffer)) return;
        $pipeline = $this->redis->pipeline();
        foreach ($this->buffer as $log) {
            $pipeline->rPush('log_queue', json_encode($log));
        }
        $pipeline->execute();
        $this->buffer = [];
    }
}

Filebeat优化配置

# filebeat.yml
filebeat.inputs:
- type: log
  harvester_buffer_size: 16384  # 读取缓冲区
  max_bytes: 10485760  # 最大单行长度 10MB
  backoff: 1s  # 重试间隔
  max_backoff: 10s  # 最大重试间隔
  backoff_factor: 2  # 重试因子
# 使用多输出负载均衡
output.logstash:
  hosts: ["logstash1:5044", "logstash2:5044", "logstash3:5044"]
  loadbalance: true
  bulk_max_size: 2048   # 批量发送大小
  worker: 4             # 工作者线程数

监控与告警

Watcher 配置示例

// 错误率告警
{
  "trigger": {
    "schedule": { "interval": "1m" }
  },
  "input": {
    "search": {
      "request": {
        "indices": ["php-app-logs-*"],
        "body": {
          "query": {
            "bool": {
              "filter": [
                { "range": { "@timestamp": { "gte": "now-5m" } } },
                { "term": { "log_level.keyword": "ERROR" } }
              ]
            }
          }
        }
      }
    }
  },
  "condition": {
    "compare": {
      "ctx.payload.hits.total": { "gte": 100 }
    }
  },
  "actions": {
    "email_admin": {
      "email": {
        "to": "ops@example.com",
        "subject": "High PHP Error Rate Alert",
        "body": "Detected {{ctx.payload.hits.total}} errors in last 5 minutes"
      }
    },
    "webhook": {
      "webhook": {
        "url": "http://alert-manager/webhook",
        "method": "POST"
      }
    }
  }
}

十一、常见问题排查

日志不显示排查流程

# 1. 检查Filebeat状态
systemctl status filebeat
journalctl -u filebeat -f
# 2. 测试日志路径
filebeat test output
filebeat test config
# 3. 检查Elasticsearch索引
curl -X GET "localhost:9200/_cat/indices?v"
# 4. 测试Logstash
bin/logstash -f test.conf --debug
# 5. 验证PHP JSON格式
echo '{"message":"test","level":"INFO"}' | filebeat -c filebeat.yml

性能问题诊断

# 检查Elasticsearch性能
curl -X GET "localhost:9200/_nodes/stats?pretty"
# 查看Logstash pipeline状态
curl -X GET "localhost:9600/_node/stats?pretty"
# 监控Filebeat指标
curl -X GET "localhost:5066/stats"

十二、最佳实践总结

日志规范

  • 始终使用结构化日志（JSON格式）
  • 包含唯一请求ID 便于追踪
  • 标准化日志级别 (DEBUG/INFO/WARNING/ERROR/CRITICAL)
  • 限制单条日志大小 (建议<10KB)

性能考量

  • PHP端使用异步日志写入
  • Filebeat使用压缩传输 (gzip)
  • Logstash启用批量处理
  • 设置合理的索引生命周期 (ILM)

安全建议

  • 日志中不要记录敏感信息 (密码、Token等)
  • 启用HTTPS传输
  • 配置访问控制 (RBAC)
  • 定期审计日志内容

监控指标

  • 日志产生速率 (logs/s)
  • Elasticsearch索引速率
  • 各组件CPU/内存使用率
  • 数据流延迟 (端到端<10s)

通过以上配置,您可以构建一个完整的PHP日志聚合系统，实现日志的集中管理、实时分析、故障排查和告警通知。

PHP项目日志聚合与ELK

上一篇PHP项目指标收集与Prometheus

下一篇PHP项目链路追踪与Jaeger

相关文章

抱歉，评论功能暂时关闭!