吃透云原生可观测：Metrics、Logging、Tracing 架构底层逻辑与实战全指南

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>3.2.5</version>
        <relativePath/>
    </parent>
    <groupId>com.example</groupId>
    <artifactId>observability-demo</artifactId>
    <version>0.0.1-SNAPSHOT</version>
    <name>observability-demo</name>
    <dependencies>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-actuator</artifactId>
        </dependency>
        <dependency>
            <groupId>io.micrometer</groupId>
            <artifactId>micrometer-registry-prometheus</artifactId>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>
</project>

spring:
  application:
    name:order-service
server:
port:8080
management:
endpoints:
    web:
      exposure:
        include:prometheus,health
metrics:
    tags:
      application:${spring.application.name}
    distribution:
      percentiles-histogram:
        http.server.requests:true

package com.example.observabilitydemo.controller;

import io.micrometer.core.annotation.Timed;
import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.DistributionSummary;
import io.micrometer.core.instrument.Gauge;
import io.micrometer.core.instrument.MeterRegistry;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.concurrent.atomic.AtomicInteger;

@RestController
@RequestMapping("/api/order")
publicclass OrderController {

    privatefinal Counter orderCreateCounter;
    privatefinal Counter orderFailCounter;
    privatefinal AtomicInteger activeOrderCount;
    privatefinal DistributionSummary orderAmountSummary;

    public OrderController(MeterRegistry meterRegistry) {
        this.orderCreateCounter = Counter.builder("order.create.total")
                .description("订单创建总次数")
                .register(meterRegistry);
        this.orderFailCounter = Counter.builder("order.create.fail.total")
                .description("订单创建失败总次数")
                .register(meterRegistry);
        this.activeOrderCount = new AtomicInteger(0);
        Gauge.builder("order.active.count", activeOrderCount, AtomicInteger::get)
                .description("当前活跃订单数")
                .register(meterRegistry);
        this.orderAmountSummary = DistributionSummary.builder("order.amount.summary")
                .description("订单金额分布统计")
                .publishPercentiles(0.5, 0.9, 0.99)
                .register(meterRegistry);
    }

    @GetMapping("/create/{amount}")
    @Timed(value = "order.create.time", description = "订单创建接口耗时")
    public String createOrder(@PathVariable Long amount) {
        orderCreateCounter.increment();
        activeOrderCount.incrementAndGet();
        orderAmountSummary.record(amount);
        try {
            if (amount <= 0) {
                thrownew IllegalArgumentException("订单金额不能小于等于0");
            }
            return"订单创建成功";
        } catch (Exception e) {
            orderFailCounter.increment();
            return"订单创建失败";
        } finally {
            activeOrderCount.decrementAndGet();
        }
    }
}

global:
  scrape_interval:15s
evaluation_interval:15s
scrape_configs:
-job_name:"spring-boot-application"
    metrics_path:"/actuator/prometheus"
    static_configs:
      -targets:["host.docker.internal:8080"]

<dependency>
    <groupId>net.logstash.logback</groupId>
    <artifactId>logstash-logback-encoder</artifactId>
    <version>8.0</version>
</dependency>

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <include resource="org/springframework/boot/logging/logback/defaults.xml"/>
    <include resource="org/springframework/boot/logging/logback/console-appender.xml"/>
    <springProperty scope="context" name="applicationName" source="spring.application.name"/>
    <appender name="JSON_CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
        <encoder class="net.logstash.logback.encoder.LogstashEncoder">
            <includeMdcKeyName>trace_id</includeMdcKeyName>
            <includeMdcKeyName>span_id</includeMdcKeyName>
            <includeMdcKeyName>user_id</includeMdcKeyName>
            <includeMdcKeyName>order_id</includeMdcKeyName>
            <customFields>{"service":"${applicationName}"}</customFields>
            <timestampPattern>yyyy-MM-dd'T'HH:mm:ss.SSSXXX</timestampPattern>
            <writeDurationAsNumber>true</writeDurationAsNumber>
        </encoder>
    </appender>
    <root level="INFO">
        <appender-ref ref="JSON_CONSOLE"/>
    </root>
</configuration>

package com.example.observabilitydemo.service;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.stereotype.Service;

@Service
publicclass OrderService {

    privatestaticfinal Logger log = LoggerFactory.getLogger(OrderService.class);

    public void createOrder(Long orderId, Long userId, Long amount) {
        MDC.put("order_id", orderId.toString());
        MDC.put("user_id", userId.toString());
        try {
            log.info("开始创建订单");
            if (amount <= 0) {
                log.error("订单创建失败，金额非法: {}", amount);
                thrownew IllegalArgumentException("订单金额非法");
            }
            log.info("订单创建成功");
        } finally {
            MDC.remove("order_id");
            MDC.remove("user_id");
        }
    }
}

[SERVICE]
    Flush        1
    Log_Level    info
    Daemon       off
    Parsers_File parsers.conf

[INPUT]
    Name         tail
    Tag          kube.*
    Path         /var/log/containers/*.log
    Parser       docker
    Refresh_Interval 10
    Mem_Buf_Limit 5MB
    Skip_Long_Lines On

[FILTER]
    Name         kubernetes
    Match        kube.*
    Kube_URL     https://kubernetes.default.svc:443
    Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
    Merge_Log    On
    K8S-Logging.Parser On
    K8S-Logging.Exclude Off

[OUTPUT]
    Name         opensearch
    Match        *
    Host         opensearch-cluster.default.svc
    Port         9200
    Index        k8s-logs-%Y.%m.%d
    Suppress_Type_Name On
    Logstash_Format Off
    Auto_Index    On

java -javaagent:opentelemetry-javaagent-1.39.0.jar \
-Dotel.service.name=order-service \
-Dotel.traces.exporter=jaeger \
-Dotel.exporter.jaeger.endpoint=http://jaeger:14250 \
-Dotel.logs.exporter=none \
-Dotel.metrics.exporter=none \
-jar observability-demo-0.0.1-SNAPSHOT.jar

package com.example.observabilitydemo.service;

import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.context.Scope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.slf4j.MDC;
import org.springframework.stereotype.Service;

@Service
publicclass PaymentService {

    privatestaticfinal Logger log = LoggerFactory.getLogger(PaymentService.class);
    privatefinal Tracer tracer;

    public PaymentService(OpenTelemetry openTelemetry) {
        this.tracer = openTelemetry.getTracer("com.example.observabilitydemo.service.PaymentService");
    }

    public void processPayment(Long orderId, Long userId, Long amount) {
        Span paymentSpan = tracer.spanBuilder("processPayment")
                .setAttribute("order_id", orderId)
                .setAttribute("user_id", userId)
                .setAttribute("amount", amount)
                .startSpan();
        try (Scope scope = paymentSpan.makeCurrent()) {
            MDC.put("trace_id", Span.current().getSpanContext().getTraceId());
            MDC.put("span_id", Span.current().getSpanContext().getSpanId());
            log.info("开始处理支付");
            paymentSpan.addEvent("支付校验开始");
            if (amount > 10000) {
                thrownew RuntimeException("支付金额超过限额");
            }
            paymentSpan.addEvent("支付校验完成");
            log.info("支付处理完成");
        } catch (Exception e) {
            paymentSpan.recordException(e);
            log.error("支付处理失败", e);
            throw e;
        } finally {
            paymentSpan.end();
            MDC.remove("trace_id");
            MDC.remove("span_id");
        }
    }
}

version: '3.8'
services:
jaeger:
    image:jaegertracing/all-in-one:1.58.0
    container_name:jaeger
    ports:
      -"16686:16686"
      -"14250:14250"
    environment:
      -COLLECTOR_OTLP_ENABLED=true
      -LOG_LEVEL=info