AIOps 落地实战：用 AI 重构服务器运维全流程，告别熬夜排障

import psutil
import time
import datetime
import requests

# 配置监控阈值与全局参数
# CPU、内存、磁盘告警阈值（百分比）
CPU_THRESHOLD = 85
MEM_THRESHOLD = 85
DISK_THRESHOLD = 90
# 连续异常次数判定，避免瞬时波动误报
ERROR_COUNT = 3
# 日志记录路径
LOG_PATH = "/var/log/aiops_monitor.log"
# 告警推送接口（企业微信/钉钉机器人）
ALERT_URL = "你的告警机器人接口地址"

# 全局异常计数器
cpu_error = 0
mem_error = 0
disk_error = 0

# 日志记录函数
def write_log(content):
    """写入运维日志，记录每一次监控与处置行为"""
    now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    log_content = f"[{now}] {content}\n"
    with open(LOG_PATH, "a", encoding="utf-8") as f:
        f.write(log_content)
    print(log_content.strip())

# 告警推送函数
def send_alert(msg):
    """异常告警推送，通知运维人员"""
    data = {
        "msgtype": "text",
        "text": {"content": f"AIOps运维告警：{msg}"}
    }
    try:
        requests.post(ALERT_URL, json=data, timeout=5)
        write_log(f"告警推送成功：{msg}")
    except Exception as e:
        write_log(f"告警推送失败：{str(e)}")

# AI趋势简易分析：判断资源占用是否持续攀升
def trend_judge(now_val, last_val):
    """
    简易时序趋势判断
    返回True：资源持续上涨，存在潜在故障风险
    """
    if now_val > last_val:
        return True
    return False

# 服务器异常自动修复函数
def auto_repair(repair_type):
    """
    分类执行自动修复操作
    1. 内存异常：清理系统缓存
    2. CPU异常：终止高占用无用进程
    3. 磁盘异常：清理日志垃圾文件
    """
    if repair_type == "memory":
        # 清理Linux系统缓存
        with open("/proc/sys/vm/drop_caches", "w") as f:
            f.write("3")
        write_log("执行内存异常修复：系统缓存清理完成")
    
    elif repair_type == "disk":
        # 清理30天前过期日志文件
        import os
        os.system("find /var/log -name \"*.log\" -mtime +30 -delete")
        write_log("执行磁盘异常修复：过期日志清理完成")
    
    elif repair_type == "cpu":
        # 查杀占用CPU过高的临时进程
        for proc in psutil.process_iter(["pid", "name", "cpu_percent"]):
            if proc.info["cpu_percent"] > 90 and proc.info["name"] not in ["systemd", "python"]:
                psutil.Process(proc.info["pid"]).terminate()
                write_log(f"终止高CPU占用进程：{proc.info['name']}（PID：{proc.info['pid']}）")

# 核心监控主函数
def monitor_server():
    global cpu_error, mem_error, disk_error
    last_cpu = 0
    last_mem = 0

    write_log("===== AIOps智能运维监控服务启动成功 =====")
    while True:
        # 采集服务器核心指标
        cpu_percent = psutil.cpu_percent(interval=1)
        mem_percent = psutil.virtual_memory().percent
        disk_percent = psutil.disk_usage("/").percent

        # AI趋势预判 + 异常计数统计
        if cpu_percent > CPU_THRESHOLD:
            cpu_error += 1
            # 判定资源持续上涨，提前预警
            if trend_judge(cpu_percent, last_cpu) and cpu_error < ERROR_COUNT:
                write_log(f"CPU资源持续攀升，当前使用率：{cpu_percent}%，潜在风险预警")
        else:
            cpu_error = 0

        if mem_percent > MEM_THRESHOLD:
            mem_error += 1
            if trend_judge(mem_percent, last_mem) and mem_error < ERROR_COUNT:
                write_log(f"内存资源持续攀升，当前使用率：{mem_percent}%，潜在风险预警")
        else:
            mem_error = 0

        if disk_percent > DISK_THRESHOLD:
            disk_error += 1
        else:
            disk_error = 0

        # 达到异常阈值，执行自动修复+告警
        if cpu_error >= ERROR_COUNT:
            auto_repair("cpu")
            send_alert(f"服务器CPU持续高占用，使用率：{cpu_percent}%，已自动执行修复")
            cpu_error = 0

        if mem_error >= ERROR_COUNT:
            auto_repair("memory")
            send_alert(f"服务器内存持续高占用，使用率：{mem_percent}%，已自动执行修复")
            mem_error = 0

        if disk_error >= ERROR_COUNT:
            auto_repair("disk")
            send_alert(f"服务器磁盘空间不足，使用率：{disk_percent}%，已自动清理垃圾文件")
            disk_error = 0

        # 更新历史数据，用于下一轮趋势分析
        last_cpu = cpu_percent
        last_mem = mem_percent
        time.sleep(10)

if __name__ == "__main__":
    monitor_server()