618 大促凌晨 2 点，AI 告警把值班群炸了——误报率 97%

[INFO] 订单服务 QPS 基线计算
       - 训练数据: 2026-06-10 ~ 2026-06-17 (7天)
       - 均值: 1850 QPS     ← 全是日常流量
       - 标准差: 550
       - 动态阈值上限: 1850 + 3×550 = 3500 QPS
       - ⚠️ 大促实际峰值 8200 QPS

defget_training_data(metric: str, exclude_events: list[str] = None):
"""获取训练数据，排除异常事件影响"""
    raw_data = fetch_metric(metric, days=30)
if exclude_events:
for event in exclude_events:
            raw_data = raw_data[raw_data['ds'] notin event_dates]
return calc_dynamic_threshold(raw_data)

PROMOTION_CONFIG = {
"618": {"date": "2026-06-18", "multiplier": 3.0},   # 阈值放宽 3 倍
"双11": {"date": "2026-11-11", "multiplier": 5.0},
"黑五": {"date": "2026-11-27", "multiplier": 4.0},
}

defis_promotion_today() -> bool:
    today = datetime.now().strftime("%Y-%m-%d")
for name, config in PROMOTION_CONFIG.items():
if today == config["date"]:
return name
returnNone

promotion = is_promotion_today()
threshold_multiplier = PROMOTION_CONFIG.get(promotion, {}).get("multiplier", 1.0)
upper_threshold = mean + sigma * std * threshold_multiplier

修复前: 告警 60+ 条，误报 58 条，误报率 97%
修复后: 告警 5 条，误报 1 条，误报率 2%
真实故障捕获: 2/2（100% 全覆盖）

过去 7 天同时段数据 → 计算均值 + 标准差 → 动态阈值 = 均值 + 3σ
                          ↓
                  实时指标与动态阈值对比
                     ↓          ↓
                  超阈值→告警   正常→跳过

import requests
import numpy as np
from datetime import datetime, timedelta

PROMETHEUS_URL = "http://localhost:9090"

defget_history(metric: str, hours: int = 168) -> list:
"""获取过去 N 小时的指标数据"""
    end = datetime.now()
    start = end - timedelta(hours=hours)
    response = requests.get(
f"{PROMETHEUS_URL}/api/v1/query_range",
        params={
"query": metric,
"start": start.timestamp(),
"end": end.timestamp(),
"step": "300s"
        }
    )
    values = [float(v[1]) for v in response.json()["data"]["result"][0]["values"]]
return values

defcalc_dynamic_threshold(values: list, sigma: float = 3.0) -> dict:
"""计算动态阈值（3-Sigma）"""
    arr = np.array(values)
    mean = np.mean(arr)
    std = np.std(arr)
    upper = mean + sigma * std
    lower = mean - sigma * std
return {
"mean": round(mean, 2),
"std": round(std, 2),
"upper_threshold": round(upper, 2),
"current": round(values[-1], 2),
"is_anomaly": values[-1] > upper or values[-1] < lower
    }

# 示例
result = calc_dynamic_threshold(get_history('node_cpu_seconds_total{mode="idle"}'))
print(f"动态阈值上界: {result['upper_threshold']}%")
print(f"当前值: {result['current']}%")
print(f"是否异常: {'⚠️ 是'if result['is_anomaly'] else'✅ 否'}")

"""告警聚类降噪：将关联告警合并为一条事件"""
from collections import defaultdict

CORRELATION_RULES = [
    {
"name": "数据库慢查询链",
"triggers": ["MySQLSlowQueries", "HTTPP99High"],
"group": "db_performance"
    },
    {
"name": "内存泄漏链",
"triggers": ["MemoryHigh", "ProcessRestarted", "OOMKilled"],
"group": "memory_issue"
    },
]

defcluster_alerts(active_alerts: list) -> list:
"""将活跃告警按关联规则聚类"""
    groups = defaultdict(list)
    ungrouped = []
for alert in active_alerts:
        matched = False
for rule in CORRELATION_RULES:
if alert["name"] in rule["triggers"]:
                groups[rule["group"]].append(alert)
                matched = True
break
ifnot matched:
            ungrouped.append(alert)
# 合并每组为一条事件
    events = []
for group_name, alerts in groups.items():
        events.append({
"event_type": group_name,
"alert_count": len(alerts),
"alerts": [a["name"] for a in alerts],
"severity": max(a.get("severity", "warning") for a in alerts)
        })
for alert in ungrouped:
        events.append({
"event_type": alert["name"],
"alert_count": 1,
"alerts": [alert["name"]],
"severity": alert.get("severity", "info")
        })
return events

hermes chat "为以下服务生成 Prometheus 告警规则：
- 服务名：order-service
- 类型：Spring Boot REST API
- SLA：99.9% 可用性，P99 < 500ms
- 依赖：MySQL、Redis、Kafka

要求：
1. 覆盖可用性、延迟、错误率、资源四大类
2. 每条规则标注严重级别（P0-P3）
3. 输出 Prometheus rule YAML 格式"

groups:
-name:order-service-availability
rules:
-alert:OrderServiceDown
expr:up{job="order-service"}==0
for:1m
labels:
severity:P0
annotations:
summary:"订单服务不可用"

-alert:OrderServiceHighErrorRate
expr:|
          (
            sum(rate(http_requests_total{job="order-service",code=~"5.."}[5m]))
            /
            sum(rate(http_requests_total{job="order-service"}[5m]))
          ) > 0.01
for:3m
labels:
severity:P1
annotations:
summary:"订单服务错误率超过 1%"

#!/bin/bash
# =====================================================
# dynamic_threshold_cron.sh - AI 动态阈值更新
# =====================================================

PROMETHEUS="http://localhost:9090"
ALERT_DIR="/etc/prometheus/rules/dynamic"
BACKUP_DIR="/var/backup/prometheus_rules"
DATE=$(date +%Y%m%d_%H%M%S)

# 备份当前规则
mkdir -p $BACKUP_DIR
cp -r $ALERT_DIR$BACKUP_DIR/rules_$DATE
find $BACKUP_DIR -name "rules_*" -mtime +30 -execrm -rf {} \;

# 计算动态阈值
for metric in"cpu_usage""memory_usage""qps""latency_p99"; do
    python3 /opt/scripts/calc_threshold.py \
        --prometheus $PROMETHEUS \
        --metric $metric \
        --output $ALERT_DIR/${metric}_dynamic.yml
done

# 验证规则语法
promtool check rules $ALERT_DIR/*.yml
if [ $? -ne 0 ]; then
echo"规则验证失败，回滚！"
cp -r $BACKUP_DIR/rules_$(ls -t $BACKUP_DIR | head -2 | tail -1)/* $ALERT_DIR/
exit 1
fi

# 重新加载 Prometheus 规则
curl -X POST http://localhost:9090/-/reload
echo"✅ 动态阈值更新完成: $DATE"

# Crontab 配置 - 每 5 分钟更新动态阈值
*/5 * * * * /opt/scripts/dynamic_threshold_cron.sh
0 0 * * 1   /opt/scripts/prometheus_backup.sh