Prometheus企业级监控架构设计与生产实践——从单体到高可用集群的完整路径

global:
  scrape_interval: 15s      # 采集间隔，生产环境默认15秒
  evaluation_interval: 15s  # 告警规则评估间隔
  external_labels:
    cluster: 'prod-cluster'
    region: 'cn-guangzhou'

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

rule_files:
  - "/etc/prometheus/rules/*.yml"

scrape_configs:
  # 监控Prometheus自身
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Node Exporter采集服务器基础指标
  - job_name: 'node'
    static_configs:
      - targets: ['node-exporter:9100']
    relabel_configs:
      - source_labels: [__address__]
        target_label: instance
        regex: '([^:]+):.*'
        replacement: '${1}'

scrape_configs:
  # 发现Kubernetes Pods
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      # 只采集带有特定注解的Pod
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      # 从注解中读取采集路径和端口
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
      # 添加Kubernetes元数据标签
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: example-app
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: example-app
  endpoints:
    - port: web
      path: /metrics
      interval: 30s
  namespaceSelector:
    any: true

# Prometheus启动参数
prometheus:
  args:
    - '--config.file=/etc/prometheus/prometheus.yml'
    - '--storage.tsdb.path=/prometheus'
    - '--storage.tsdb.retention.time=15d'
    - '--storage.tsdb.retention.size=50GB'
    - '--web.enable-lifecycle'

# Thanos Sidecar容器
thanos-sidecar:
  image: quay.io/thanos/thanos:v0.42.2
  args:
    - 'sidecar'
    - '--prometheus.url=http://localhost:9090'
    - '--objstore.config-file=/etc/thanos/objstore.yml'

type: S3
config:
  bucket: prometheus-data
  endpoint: s3.amazonaws.com
  access_key: YOUR_ACCESS_KEY
  secret_key: YOUR_SECRET_KEY
  region: us-east-1

# Thanos Querier配置，对接多个集群的Thanos Store
query:
  stores:
    - thanos-store-cluster-a:10901
    - thanos-store-cluster-b:10901
    - thanos-sidecar-cluster-a:10901
    - thanos-sidecar-cluster-b:10901

./prometheus \
  --config.file=/etc/prometheus/prometheus.yml \
  --storage.tsdb.path=/var/lib/prometheus \
  --storage.tsdb.retention.time=30d \
  --storage.tsdb.retention.size=100GB \
  --query.max-concurrency=20 \
  --query.timeout=2m \
  --web.enable-lifecycle

# ✅ 高效：精确匹配，利用倒排索引
http_requests_total{status="500"}

# ❌ 低效：正则匹配需要扫描更多倒排列表
http_requests_total{status=~"500"}

# ❌ 危险：匹配所有可能的值，极易导致性能问题
http_requests_total{job=~".*"}

# ✅ 推荐：明确指定保留的标签
sum by (service, code) (rate(http_requests_total[5m]))

# ❌ 不推荐：隐式聚合，新标签可能破坏逻辑
sum(rate(http_requests_total[5m]))

# ✅ 告警规则中使用rate，平滑瞬时峰值
rate(http_requests_total[5m]) > 10

# ✅ 排查瞬时抖动时使用irate，灵敏捕捉瞬间峰值
irate(http_requests_total[1m])

groups:
  - name: api_server_aggregation
    interval: 60s
    rules:
      # 预计算API请求速率
      - record: job:api_server_requests:rate5m
        expr: sum by (job, instance) (rate(api_server_requests_total[5m]))
      
      # 预计算P99延迟
      - record: job:api_server_latency:p99
        expr: histogram_quantile(0.99, sum by (job, le) (rate(api_server_request_duration_seconds_bucket[5m])))

# 集成配置参数说明
参数:
  名称: 集成名称，需符合正则 '^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$'
  地域: 云产品所在地域，支持广州、ap-guangzhou等格式
  云产品选择: 勾选想要采集的云产品
  数据拉取配置: 单位为秒，控制数据采集延迟
  实例刷新间隔: 最小10分钟
  实例ID过滤: 键值对形式，控制采集的实例范围
  云标签过滤: 支持按标签过滤实例

global:
  smtp_smarthost: 'smtp.example.com:587'
  smtp_from: 'alertmanager@example.com'
  smtp_auth_username: 'alertmanager'
  smtp_auth_password: 'password'

route:
  group_by: ['alertname', 'cluster']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default'
  routes:
    - match:
        severity: critical
      receiver: 'critical'
      group_wait: 10s
      repeat_interval: 1h
    - match:
        severity: warning
      receiver: 'warning'
      group_wait: 30s
      repeat_interval: 4h

receivers:
  - name: 'default'
    email_configs:
      - to: 'ops-team@example.com'
  - name: 'critical'
    email_configs:
      - to: 'oncall@example.com'
    webhook_configs:
      - url: 'https://your-webhook-url'