Practical Examples on Advanced Beginner

Environment Setup

d8lzz1gpw@mozmail.com (kimbenji) — Mon, 12 Jan 2026 00:00:00 +0000

Time Required: 15 minutes Prerequisites: Docker, Docker Compose basics What You’ll Learn: Build a complete observability stack locally

Overall Architecture#

graph TB
 APP["Spring Boot App"] --> |"metrics"| PROM["Prometheus"]
 APP --> |"logs"| LOKI["Loki"]
 APP --> |"traces"| TEMPO["Tempo"]

 PROM --> GF["Grafana"]
 LOKI --> GF
 TEMPO --> GF

 PROM --> AM["Alertmanager"]

Step 1: Create Directory Structure#

mkdir -p observability-stack/{prometheus/rules,grafana/{provisioning/datasources,provisioning/dashboards,dashboards},loki,promtail}
cd observability-stack

Step 2: Write Docker Compose#

# docker-compose.yml
services:
 # Metrics collection
 prometheus:
 image: prom/prometheus:v2.50.0
 container_name: prometheus
 ports:
 - "9090:9090"
 volumes:
 - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
 - ./prometheus/rules:/etc/prometheus/rules
 - prometheus-data:/prometheus
 command:
 - '--config.file=/etc/prometheus/prometheus.yml'
 - '--storage.tsdb.path=/prometheus'
 - '--web.enable-lifecycle'
 - '--web.enable-remote-write-receiver'

 # Alert management
 alertmanager:
 image: prom/alertmanager:v0.26.0
 container_name: alertmanager
 ports:
 - "9093:9093"
 volumes:
 - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml

 # Visualization
 grafana:
 image: grafana/grafana:10.3.0
 container_name: grafana
 ports:
 - "3000:3000"
 environment:
 - GF_SECURITY_ADMIN_PASSWORD=admin
 - GF_AUTH_ANONYMOUS_ENABLED=true
 - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer
 volumes:
 - ./grafana/provisioning:/etc/grafana/provisioning
 - ./grafana/dashboards:/var/lib/grafana/dashboards
 - grafana-data:/var/lib/grafana

 # Log collection
 loki:
 image: grafana/loki:2.9.0
 container_name: loki
 ports:
 - "3100:3100"
 volumes:
 - ./loki/loki.yml:/etc/loki/local-config.yaml
 - loki-data:/loki
 command: -config.file=/etc/loki/local-config.yaml

 # Log agent
 promtail:
 image: grafana/promtail:2.9.0
 container_name: promtail
 volumes:
 - ./promtail/promtail.yml:/etc/promtail/config.yml
 - /var/log:/var/log:ro
 - /var/lib/docker/containers:/var/lib/docker/containers:ro
 command: -config.file=/etc/promtail/config.yml

 # Distributed tracing
 tempo:
 image: grafana/tempo:2.3.0
 container_name: tempo
 ports:
 - "4317:4317" # OTLP gRPC
 - "4318:4318" # OTLP HTTP
 - "3200:3200" # Tempo API
 volumes:
 - ./tempo/tempo.yml:/etc/tempo/config.yaml
 - tempo-data:/var/tempo
 command: -config.file=/etc/tempo/config.yaml

volumes:
 prometheus-data:
 grafana-data:
 loki-data:
 tempo-data:

Step 3: Prometheus Configuration#

# prometheus/prometheus.yml
global:
 scrape_interval: 15s
 evaluation_interval: 15s

alerting:
 alertmanagers:
 - static_configs:
 - targets:
 - alertmanager:9093

rule_files:
 - /etc/prometheus/rules/*.yml

scrape_configs:
 - job_name: 'prometheus'
 static_configs:
 - targets: ['localhost:9090']

 - job_name: 'spring-app'
 metrics_path: '/actuator/prometheus'
 static_configs:
 - targets: ['host.docker.internal:8080']

Step 4: Loki Configuration#

# loki/loki.yml
auth_enabled: false

server:
 http_listen_port: 3100

common:
 path_prefix: /loki
 storage:
 filesystem:
 chunks_directory: /loki/chunks
 rules_directory: /loki/rules
 replication_factor: 1
 ring:
 instance_addr: 127.0.0.1
 kvstore:
 store: inmemory

schema_config:
 configs:
 - from: 2020-10-24
 store: boltdb-shipper
 object_store: filesystem
 schema: v11
 index:
 prefix: index_
 period: 24h

limits_config:
 retention_period: 168h

Step 5: Promtail Configuration#

# promtail/promtail.yml
server:
 http_listen_port: 9080

positions:
 filename: /tmp/positions.yaml

clients:
 - url: http://loki:3100/loki/api/v1/push

scrape_configs:
 - job_name: containers
 static_configs:
 - targets:
 - localhost
 labels:
 job: containerlogs
 __path__: /var/lib/docker/containers/*/*log

 pipeline_stages:
 - json:
 expressions:
 output: log
 stream: stream
 attrs:
 - json:
 expressions:
 tag:
 source: attrs
 - regex:
 expression: (?P<container_name>(?:[^|]*[^|]))
 source: tag
 - labels:
 container_name:
 stream:

Step 6: Tempo Configuration#

# tempo/tempo.yml
server:
 http_listen_port: 3200

distributor:
 receivers:
 otlp:
 protocols:
 grpc:
 endpoint: 0.0.0.0:4317
 http:
 endpoint: 0.0.0.0:4318

storage:
 trace:
 backend: local
 local:
 path: /var/tempo/traces
 wal:
 path: /var/tempo/wal

metrics_generator:
 registry:
 external_labels:
 source: tempo
 storage:
 path: /var/tempo/generator/wal
 remote_write:
 - url: http://prometheus:9090/api/v1/write
 send_exemplars: true

Step 7: Grafana Datasource Configuration#

# grafana/provisioning/datasources/datasources.yml
apiVersion: 1
datasources:
 - name: Prometheus
 type: prometheus
 access: proxy
 url: http://prometheus:9090
 isDefault: true

 - name: Loki
 type: loki
 access: proxy
 url: http://loki:3100

 - name: Tempo
 type: tempo
 access: proxy
 url: http://tempo:3200
 jsonData:
 tracesToLogsV2:
 datasourceUid: loki
 filterByTraceID: true
 tracesToMetrics:
 datasourceUid: prometheus
 serviceMap:
 datasourceUid: prometheus

Step 8: Alertmanager Configuration#

mkdir -p alertmanager

# alertmanager/alertmanager.yml
global:
 resolve_timeout: 5m

route:
 receiver: 'default'
 group_by: ['alertname', 'job']
 group_wait: 30s
 group_interval: 5m
 repeat_interval: 4h

receivers:
 - name: 'default'
 webhook_configs:
 - url: 'http://localhost:5001/'
 send_resolved: true

Step 9: Start the Stack#

docker compose up -d

Expected Output:

Spring Boot Metrics

d8lzz1gpw@mozmail.com (kimbenji) — Mon, 12 Jan 2026 00:00:00 +0000

Time Required: 20 minutes Prerequisites: Spring Boot, Environment Setup What You’ll Learn: Apply observability to Spring Boot applications

Step 1/8: Add Dependencies (2 min)#

// build.gradle.kts
dependencies {
 // Actuator + Prometheus
 implementation("org.springframework.boot:spring-boot-starter-actuator")
 implementation("io.micrometer:micrometer-registry-prometheus")

 // Tracing
 implementation("io.micrometer:micrometer-tracing-bridge-otel")
 implementation("io.opentelemetry:opentelemetry-exporter-otlp")

 // Logging (Loki)
 implementation("com.github.loki4j:loki-logback-appender:1.4.2")
}

Step 2/8: application.yml Configuration (3 min)#

spring:
 application:
 name: order-service

management:
 endpoints:
 web:
 exposure:
 include: health,info,prometheus,metrics
 endpoint:
 health:
 show-details: always
 metrics:
 tags:
 application: ${spring.application.name}
 distribution:
 percentiles-histogram:
 http.server.requests: true
 slo:
 http.server.requests: 50ms, 100ms, 200ms, 500ms, 1s
 tracing:
 sampling:
 probability: 1.0
 otlp:
 tracing:
 endpoint: http://localhost:4318/v1/traces

logging:
 pattern:
 level: "%5p [${spring.application.name:},%X{traceId:-},%X{spanId:-}]"

Step 3/8: Loki Log Configuration (3 min)#

<!-- src/main/resources/logback-spring.xml -->
<?xml version="1.0" encoding="UTF-8"?>
<configuration>
 <include resource="org/springframework/boot/logging/logback/defaults.xml"/>

 <springProperty scope="context" name="appName" source="spring.application.name"/>

 <!-- Console Appender -->
 <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
 <encoder>
 <pattern>%d{ISO8601} %5p [${appName},%X{traceId:-},%X{spanId:-}] --- [%t] %-40.40logger{39} : %m%n</pattern>
 </encoder>
 </appender>

 <!-- Loki Appender -->
 <appender name="LOKI" class="com.github.loki4j.logback.Loki4jAppender">
 <http>
 <url>http://localhost:3100/loki/api/v1/push</url>
 </http>
 <format>
 <label>
 <pattern>app=${appName},host=${HOSTNAME},level=%level</pattern>
 </label>
 <message>
 <pattern>{"timestamp":"%d{ISO8601}","level":"%level","logger":"%logger","message":"%msg","traceId":"%X{traceId:-}","spanId":"%X{spanId:-}"}</pattern>
 </message>
 </format>
 </appender>

 <root level="INFO">
 <appender-ref ref="CONSOLE"/>
 <appender-ref ref="LOKI"/>
 </root>
</configuration>

Intermediate Verification (Steps 1~3 Complete)#

Dependencies added to build.gradle.kts
Management configuration complete in application.yml
logback-spring.xml file created
After ./gradlew bootRun, metrics displayed at http://localhost:8080/actuator/prometheus

Step 4/8: Add Custom Metrics (5 min)#

@RestController
@RequiredArgsConstructor
@RequestMapping("/orders")
public class OrderController {
 private final MeterRegistry meterRegistry;
 private final OrderService orderService;

 // Counter: Order count
 private Counter orderCounter(String status) {
 return Counter.builder("orders_total")
 .tag("status", status)
 .description("Total number of orders")
 .register(meterRegistry);
 }

 // Gauge: Orders in progress
 private final AtomicInteger ordersInProgress = new AtomicInteger(0);

 @PostConstruct
 void registerGauge() {
 Gauge.builder("orders_in_progress", ordersInProgress, AtomicInteger::get)
 .description("Orders currently being processed")
 .register(meterRegistry);
 }

 @PostMapping
 public ResponseEntity<Order> createOrder(@RequestBody OrderRequest request) {
 ordersInProgress.incrementAndGet();
 try {
 // Timer to measure processing time
 return Timer.builder("order_processing_duration_seconds")
 .tag("type", request.getType())
 .publishPercentileHistogram()
 .register(meterRegistry)
 .record(() -> {
 Order order = orderService.create(request);
 orderCounter("success").increment();
 return ResponseEntity.ok(order);
 });
 } catch (Exception e) {
 orderCounter("failed").increment();
 throw e;
 } finally {
 ordersInProgress.decrementAndGet();
 }
 }
}

Step 5/8: Manual Tracing (3 min)#

@Service
@RequiredArgsConstructor
@Slf4j
public class OrderService {
 private final Tracer tracer;
 private final PaymentClient paymentClient;
 private final InventoryClient inventoryClient;

 public Order create(OrderRequest request) {
 // Add tags to current Span
 Span currentSpan = tracer.currentSpan();
 if (currentSpan != null) {
 currentSpan.tag("order.type", request.getType());
 currentSpan.tag("order.items", String.valueOf(request.getItems().size()));
 }

 // Create new Span
 Span inventorySpan = tracer.nextSpan().name("checkInventory").start();
 try (Tracer.SpanInScope ws = tracer.withSpan(inventorySpan)) {
 inventoryClient.check(request.getItems());
 inventorySpan.event("Inventory checked");
 } finally {
 inventorySpan.end();
 }

 // Process payment
 Span paymentSpan = tracer.nextSpan().name("processPayment").start();
 try (Tracer.SpanInScope ws = tracer.withSpan(paymentSpan)) {
 paymentClient.process(request.getPayment());
 paymentSpan.event("Payment processed");
 } finally {
 paymentSpan.end();
 }

 log.info("Order created: {}", request.getId());
 return new Order(request);
 }
}

Intermediate Verification (Steps 4~5 Complete)#

Counter, Gauge, Timer metrics added to OrderController
Manual Span creation code added to OrderService
Build successful without compilation errors: ./gradlew build

Step 6/8: Verify Metrics (2 min)#

Prometheus Endpoint#

curl http://localhost:8080/actuator/prometheus

Expected Output:

Kafka Monitoring

d8lzz1gpw@mozmail.com (kimbenji) — Mon, 12 Jan 2026 00:00:00 +0000

Time Required: 20 minutes Prerequisites: Kafka basics, Environment Setup What You’ll Learn: Monitor Kafka clusters and Consumer Lag

Core Kafka Monitoring Metrics#

Category	Metric	Meaning
Consumer	Lag	Processing delay (most important)
Broker	Under-replicated Partitions	Replication issues
Producer	Record Error Rate	Send failure rate
Broker	Disk Usage	Storage space

Step 1: Kafka + JMX Exporter Configuration#

# Add to docker-compose.yml
services:
 kafka:
 image: confluentinc/cp-kafka:7.5.0
 container_name: kafka
 ports:
 - "9092:9092"
 - "9101:9101" # JMX
 environment:
 KAFKA_NODE_ID: 1
 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
 KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
 KAFKA_PROCESS_ROLES: broker,controller
 KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:29093
 KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093
 KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
 CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
 KAFKA_JMX_PORT: 9101
 KAFKA_JMX_HOSTNAME: localhost
 KAFKA_OPTS: -javaagent:/opt/jmx-exporter/jmx_prometheus_javaagent.jar=7071:/opt/jmx-exporter/kafka.yml
 volumes:
 - ./jmx-exporter:/opt/jmx-exporter

 kafka-exporter:
 image: danielqsj/kafka-exporter:latest
 container_name: kafka-exporter
 ports:
 - "9308:9308"
 command:
 - --kafka.server=kafka:9092
 - --topic.filter=.*
 - --group.filter=.*

Step 2: JMX Exporter Configuration#

mkdir -p jmx-exporter
curl -L https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.19.0/jmx_prometheus_javaagent-0.19.0.jar -o jmx-exporter/jmx_prometheus_javaagent.jar

# jmx-exporter/kafka.yml
lowercaseOutputName: true
rules:
 # Broker metrics
 - pattern: kafka.server<type=(.+), name=(.+), clientId=(.+), topic=(.+), partition=(.*)><>Value
 name: kafka_server_$1_$2
 type: GAUGE
 labels:
 clientId: "$3"
 topic: "$4"
 partition: "$5"

 - pattern: kafka.server<type=(.+), name=(.+)><>Value
 name: kafka_server_$1_$2
 type: GAUGE

 # Under-replicated Partitions
 - pattern: kafka.server<type=ReplicaManager, name=UnderReplicatedPartitions><>Value
 name: kafka_server_replicamanager_underreplicatedpartitions
 type: GAUGE

 # Messages In Per Sec
 - pattern: kafka.server<type=BrokerTopicMetrics, name=MessagesInPerSec, topic=(.+)><>OneMinuteRate
 name: kafka_server_brokertopicmetrics_messagesin_total
 type: GAUGE
 labels:
 topic: "$1"

 # Bytes In Per Sec
 - pattern: kafka.server<type=BrokerTopicMetrics, name=BytesInPerSec, topic=(.+)><>OneMinuteRate
 name: kafka_server_brokertopicmetrics_bytesin_total
 type: GAUGE
 labels:
 topic: "$1"

Step 3: Add Prometheus Configuration#

# Add to prometheus/prometheus.yml
scrape_configs:
 - job_name: 'kafka'
 static_configs:
 - targets: ['kafka:7071']

 - job_name: 'kafka-exporter'
 static_configs:
 - targets: ['kafka-exporter:9308']

Step 4: Core PromQL Queries#

Consumer Lag#

# Lag by consumer group
sum by (consumergroup, topic) (kafka_consumergroup_lag)

# Groups with lag over 10000
sum by (consumergroup, topic) (kafka_consumergroup_lag) > 10000

# Lag trends
sum(kafka_consumergroup_lag) by (consumergroup)

Broker Status#

# Under-replicated Partitions
kafka_server_replicamanager_underreplicatedpartitions

# ISR (In-Sync Replicas) shrinkage
kafka_server_replicamanager_isrshrinks_total

Traffic#

# Messages per second by topic
sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total[5m]))

# Bytes per second by topic
sum by (topic) (rate(kafka_server_brokertopicmetrics_bytesin_total[5m]))

Step 5: Alert Rules#

# prometheus/rules/kafka-alerts.yml
groups:
 - name: kafka
 rules:
 - alert: KafkaConsumerLagHigh
 expr: sum by (consumergroup, topic) (kafka_consumergroup_lag) > 10000
 for: 10m
 labels:
 severity: warning
 annotations:
 summary: "High consumer lag: {{ $labels.consumergroup }}"
 description: "Lag is {{ $value }} on topic {{ $labels.topic }}"

 - alert: KafkaUnderReplicatedPartitions
 expr: kafka_server_replicamanager_underreplicatedpartitions > 0
 for: 5m
 labels:
 severity: critical
 annotations:
 summary: "Kafka under-replicated partitions"
 description: "{{ $value }} partitions are under-replicated"

 - alert: KafkaBrokerDown
 expr: count(kafka_server_kafkaserver_brokerstate) < 3
 for: 5m
 labels:
 severity: critical
 annotations:
 summary: "Kafka broker down"

Step 6: Recording Rules#

# prometheus/rules/kafka-recording.yml
groups:
 - name: kafka_recording
 rules:
 - record: topic:kafka_messages:rate5m
 expr: sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total[5m]))

 - record: consumergroup:kafka_lag:sum
 expr: sum by (consumergroup) (kafka_consumergroup_lag)

 - record: :kafka_underreplicated:sum
 expr: sum(kafka_server_replicamanager_underreplicatedpartitions)

Step 7: Grafana Dashboard#

Row 1: Overview#

Panel	Query	Type
Total Lag	`sum(kafka_consumergroup_lag)`	Stat
Messages/sec	`sum(rate(kafka_server_brokertopicmetrics_messagesin_total[5m]))`	Stat
Under-replicated	`sum(kafka_server_replicamanager_underreplicatedpartitions)`	Stat

Row 2: Consumer Lag#

# Time Series: Lag trends
sum by (consumergroup) (kafka_consumergroup_lag)

# Table: Details
sum by (consumergroup, topic, partition) (kafka_consumergroup_lag)

Row 3: Traffic#

# Messages per second by topic
sum by (topic) (rate(kafka_server_brokertopicmetrics_messagesin_total[5m]))

Spring Kafka Metrics#

# application.yml
management:
 metrics:
 enable:
 kafka: true

# Producer metrics
kafka_producer_record_send_total
kafka_producer_record_error_total

# Consumer metrics
kafka_consumer_records_consumed_total
kafka_consumer_fetch_manager_records_lag

Verification Checklist#

kafka-exporter metrics exposed
Consumer Lag query works
Grafana dashboard configured
Alert rules tested

Next Steps#

Recommended Order	Document	What You’ll Learn
1	Full-Stack Example	Integrated example
2	By Service Type	Kafka golden signals

Full-Stack Observability

d8lzz1gpw@mozmail.com (kimbenji) — Mon, 12 Jan 2026 00:00:00 +0000

Time Required: 30 minutes Prerequisites: Spring Boot Metrics What You’ll Learn: Track issues by connecting the three pillars (Metrics, Logs, Traces)

Goal#

“Error rate spike” → “Find trace_id in logs” → “Identify root cause with traces”

graph LR
 A["1. Metrics<br>Detect 5% error rate"]
 B["2. Logs<br>Find trace_id"]
 C["3. Traces<br>Discover bottleneck"]
 D["4. Resolution<br>Optimize DB query"]

 A --> B --> C --> D

Scenario: Order Service Failure Analysis#

System Architecture#

graph LR
 USER["User"] --> GW["API Gateway"]
 GW --> ORDER["Order Service"]
 ORDER --> PAYMENT["Payment Service"]
 ORDER --> INVENTORY["Inventory Service"]
 PAYMENT --> DB["Payment DB"]

Step 1: Failure Detection (Metrics)#

Grafana Alert Triggered#

Alert: HighErrorRate
Service: order-service
Error Rate: 5.2%
Threshold: 1%

Check Dashboard#

# Confirm error rate spike
sum(rate(http_server_requests_seconds_count{application="order-service",status=~"5.."}[5m]))
/ sum(rate(http_server_requests_seconds_count{application="order-service"}[5m]))

# Which endpoint?
sum by (uri) (rate(http_server_requests_seconds_count{application="order-service",status=~"5.."}[5m]))

Result: Errors occurring on /orders POST endpoint