本文目录导读：
在Shell脚本中设置告警恢复通知，通常需要跟踪告警状态并在状态从“异常”转为“正常”时触发通知,以下是几种常见的实现方式：
基于状态文件的实现
#!/bin/bash
# 配置
ALERT_STATUS_FILE="/tmp/alert_status.txt"
CHECK_COMMAND="ping -c 1 example.com" # 替换为实际的检查命令
NOTIFY_COMMAND='echo "告警: $1" | mail -s "告警通知" admin@example.com'
# 获取当前状态
$CHECK_COMMAND > /dev/null 2>&1
CURRENT_STATUS=$?
# 读取上次状态
if [ -f "$ALERT_STATUS_FILE" ]; then
LAST_STATUS=$(cat "$ALERT_STATUS_FILE")
else
LAST_STATUS=0 # 默认为正常状态
fi
# 判断并处理
if [ $CURRENT_STATUS -ne 0 ]; then
# 当前异常
if [ "$LAST_STATUS" -eq 0 ]; then
# 从正常变为异常，触发告警
eval $NOTIFY_COMMAND "服务异常"
echo 1 > "$ALERT_STATUS_FILE"
fi
else
# 当前正常
if [ "$LAST_STATUS" -ne 0 ]; then
# 从异常变为正常，触发恢复通知
eval $NOTIFY_COMMAND "服务已恢复"
echo 0 > "$ALERT_STATUS_FILE"
fi
fi
使用计数器防止抖动
#!/bin/bash
# 配置
CHECK_INTERVAL=60 # 检查间隔（秒）
ALERT_THRESHOLD=3 # 连续异常次数触发告警
RECOVERY_THRESHOLD=2 # 连续正常次数触发恢复
COUNTER_FILE="/tmp/alert_counter.txt"
STATUS_FILE="/tmp/alert_status.txt"
check_service() {
# 替换为实际检查逻辑
curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health
}
init_files() {
[ ! -f "$COUNTER_FILE" ] && echo "0" > "$COUNTER_FILE"
[ ! -f "$STATUS_FILE" ] && echo "0" > "$STATUS_FILE"
}
send_notification() {
local message="$1"
echo "$message" | mail -s "服务状态通知" admin@example.com
# 或使用企业微信、钉钉等
# curl -X POST -H "Content-Type: application/json" -d "{\"msgtype\":\"text\",\"text\":{\"content\":\"$message\"}}" "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_KEY"
}
main() {
init_files
local http_code=$(check_service)
local is_healthy=false
# 判断健康状态
if [ "$http_code" -eq 200 ] || [ "$http_code" -eq 302 ]; then
is_healthy=true
fi
local counter=$(cat "$COUNTER_FILE")
local status=$(cat "$STATUS_FILE")
if $is_healthy; then
# 服务正常
if [ "$status" -eq 1 ]; then
# 之前是告警状态
counter=$((counter + 1))
if [ "$counter" -ge "$RECOVERY_THRESHOLD" ]; then
send_notification "服务已恢复正常"
echo "0" > "$STATUS_FILE"
echo "0" > "$COUNTER_FILE"
else
echo "$counter" > "$COUNTER_FILE"
fi
fi
else
# 服务异常
if [ "$status" -eq 0 ]; then
# 之前是正常状态
counter=$((counter + 1))
if [ "$counter" -ge "$ALERT_THRESHOLD" ]; then
send_notification "服务异常，HTTP状态码: $http_code"
echo "1" > "$STATUS_FILE"
echo "0" > "$COUNTER_FILE"
else
echo "$counter" > "$COUNTER_FILE"
fi
fi
fi
}
# 主循环
while true; do
main
sleep "$CHECK_INTERVAL"
done
支持多指标的监控脚本
#!/bin/bash
# 配置
ALERT_DIR="/tmp/alerts"
NOTIFY_SCRIPT="/usr/local/bin/send_notify.sh"
# 初始化告警目录
mkdir -p "$ALERT_DIR"
check_disk() {
local usage=$(df / | awk 'NR==2 {print $5}' | sed 's/%//')
echo "$usage"
}
check_memory() {
local usage=$(free | awk '/Mem/ {printf "%.0f", $3/$2 * 100}')
echo "$usage"
}
check_cpu() {
local usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}')
echo "${usage%.*}"
}
send_alert() {
local metric="$1"
local value="$2"
local threshold="$3"
local type="$4" # alert 或 recovery
local alert_file="$ALERT_DIR/${metric}_alert"
if [ "$type" = "alert" ]; then
if [ ! -f "$alert_file" ]; then
# 发送告警
$NOTIFY_SCRIPT "告警: $metric 当前值: $value, 阈值: $threshold"
touch "$alert_file"
fi
elif [ "$type" = "recovery" ]; then
if [ -f "$alert_file" ]; then
# 发送恢复通知
$NOTIFY_SCRIPT "恢复: $metric 已恢复正常, 当前值: $value"
rm "$alert_file"
fi
fi
}
# 主检查逻辑
DISK_THRESHOLD=80
MEMORY_THRESHOLD=90
CPU_THRESHOLD=80
# 磁盘检查
disk_usage=$(check_disk)
if [ "$disk_usage" -gt "$DISK_THRESHOLD" ]; then
send_alert "disk" "$disk_usage" "$DISK_THRESHOLD" "alert"
else
send_alert "disk" "$disk_usage" "$DISK_THRESHOLD" "recovery"
fi
# 内存检查
mem_usage=$(check_memory)
if [ "$mem_usage" -gt "$MEMORY_THRESHOLD" ]; then
send_alert "memory" "$mem_usage" "$MEMORY_THRESHOLD" "alert"
else
send_alert "memory" "$mem_usage" "$MEMORY_THRESHOLD" "recovery"
fi
# CPU检查
cpu_usage=$(check_cpu)
if [ "$cpu_usage" -gt "$CPU_THRESHOLD" ]; then
send_alert "cpu" "$cpu_usage" "$CPU_THRESHOLD" "alert"
else
send_alert "cpu" "$cpu_usage" "$CPU_THRESHOLD" "recovery"
fi
通知发送脚本示例
#!/bin/bash
# send_notify.sh
send_email() {
local subject="$1"
local body="$2"
echo "$body" | mail -s "$subject" admin@example.com
}
send_wechat() {
local webhook_url="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=YOUR_KEY"
local content="$1"
curl -s -X POST "$webhook_url" \
-H "Content-Type: application/json" \
-d "{\"msgtype\":\"text\",\"text\":{\"content\":\"$content\"}}" > /dev/null
}
send_dingtalk() {
local webhook_url="https://oapi.dingtalk.com/robot/send?access_token=YOUR_TOKEN"
local content="$1"
curl -s -X POST "$webhook_url" \
-H "Content-Type: application/json" \
-d "{\"msgtype\":\"text\",\"text\":{\"content\":\"$content\"}}" > /dev/null
}
# 主通知函数
message="$1"
send_email "系统监控通知" "$message"
send_wechat "$message"
send_dingtalk "$message"
使用crontab定时执行
# 编辑crontab crontab -e # 每分钟检查一次 * * * * * /path/to/monitor_script.sh # 或每5分钟检查一次 */5 * * * * /path/to/monitor_script.sh
关键点说明
- 状态持久化：使用文件记录上次告警状态
- 防抖动：使用计数器避免短时间内的状态波动
- 多指标支持：为每个监控指标维护独立的状态
- 通知渠道：支持邮件、企业微信、钉钉等多种通知方式
- 告警升级：可以实现告警级别，长时间未恢复发送更紧急的通知
选择适合你场景的实现方式,并根据实际需求调整阈值和检查间隔。
