Monitoring and Observability in Distributed Systems

┌───────────────────────────────────────────────────┐
│                                                   │
│                  Observability                    │
│                                                   │
├───────────────┬───────────────┬───────────────────┤
│               │               │                   │
│    Metrics    │     Logs      │      Traces       │
│               │               │                   │
└───────────────┴───────────────┴───────────────────┘

package main

import (
    "net/http"
    "time"

    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    // Counter for total requests
    requestsTotal = promauto.NewCounterVec(
        prometheus.CounterOpts{
            Name: "http_requests_total",
            Help: "Total number of HTTP requests",
        },
        []string{"method", "endpoint", "status"},
    )

    // Histogram for request duration
    requestDuration = promauto.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "http_request_duration_seconds",
            Help:    "HTTP request duration in seconds",
            Buckets: prometheus.DefBuckets,
        },
        []string{"method", "endpoint"},
    )
)

func instrumentHandler(next http.HandlerFunc) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        // Track request duration
        start := time.Now()
        
        // Call the actual handler
        next(w, r)
        
        // Record metrics after the handler completes
        duration := time.Since(start).Seconds()
        requestDuration.WithLabelValues(r.Method, r.URL.Path).Observe(duration)
        requestsTotal.WithLabelValues(r.Method, r.URL.Path, "200").Inc()
    }
}

const winston = require('winston');
const { format } = winston;

// Define the logger
const logger = winston.createLogger({
  level: 'info',
  format: format.combine(
    format.timestamp(),
    format.errors({ stack: true }),
    format.metadata(),
    format.json()
  ),
  defaultMeta: { service: 'user-service' },
  transports: [
    new winston.transports.Console(),
    new winston.transports.File({ filename: 'error.log', level: 'error' }),
    new winston.transports.File({ filename: 'combined.log' })
  ]
});

// Example of logging in a service
function createUser(userData) {
  logger.debug('Creating user', { userData });
  
  try {
    // User creation logic
    const user = userRepository.create(userData);
    
    logger.info('User created', { 
      userId: user.id,
      email: user.email
    });
    
    return user;
  } catch (error) {
    logger.error('Failed to create user', {
      error: error.message,
      userData
    });
    
    throw error;
  }
}

from flask import Flask
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import SERVICE_NAME, Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.instrumentation.flask import FlaskInstrumentor
import requests

# Configure the tracer
resource = Resource(attributes={
    SERVICE_NAME: "order-service"
})

trace.set_tracer_provider(TracerProvider(resource=resource))
tracer = trace.get_tracer(__name__)

# Configure the exporter
otlp_exporter = OTLPSpanExporter(endpoint="jaeger:4317", insecure=True)
span_processor = BatchSpanProcessor(otlp_exporter)
trace.get_tracer_provider().add_span_processor(span_processor)

# Create and instrument the Flask app
app = Flask(__name__)
FlaskInstrumentor().instrument_app(app)

@app.route('/orders/<order_id>', methods=['GET'])
def get_order(order_id):
    with tracer.start_as_current_span("get_order_details") as span:
        span.set_attribute("order_id", order_id)
        
        # Get order from database
        order = get_order_from_db(order_id)
        
        # Get payment details
        payment = get_payment_details(order_id)
        
        # Combine and return
        result = {
            "order": order,
            "payment": payment
        }
        
        return result

// Spring Boot application with Micrometer
@SpringBootApplication
public class OrderServiceApplication {
    public static void main(String[] args) {
        SpringApplication.run(OrderServiceApplication.class, args);
    }
    
    @Bean
    public MeterRegistryCustomizer<MeterRegistry> metricsCommonTags() {
        return registry -> registry.config().commonTags(
            "application", "order-service",
            "environment", "${spring.profiles.active:default}"
        );
    }
}

// Custom metrics in a service
@Service
public class OrderService {
    private final Counter orderCreatedCounter;
    private final Timer orderProcessingTimer;
    
    public OrderService(MeterRegistry registry) {
        this.orderCreatedCounter = registry.counter("orders.created");
        this.orderProcessingTimer = registry.timer("orders.processing.time");
    }
    
    public Order createOrder(OrderRequest request) {
        return orderProcessingTimer.record(() -> {
            // Order creation logic
            Order order = new Order();
            // ... set order properties
            
            // Save order
            Order savedOrder = orderRepository.save(order);
            
            // Record metrics
            orderCreatedCounter.increment();
            
            return savedOrder;
        });
    }
}

// Request filter to extract or generate correlation IDs
@Component
public class CorrelationFilter extends OncePerRequestFilter {
    private static final String CORRELATION_ID_HEADER = "X-Correlation-ID";
    
    @Override
    protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, 
                                   FilterChain filterChain) throws ServletException, IOException {
        // Extract or generate correlation ID
        String correlationId = request.getHeader(CORRELATION_ID_HEADER);
        if (correlationId == null || correlationId.isEmpty()) {
            correlationId = UUID.randomUUID().toString();
        }
        
        // Store in MDC for logging
        MDC.put("correlationId", correlationId);
        
        // Add to response headers
        response.addHeader(CORRELATION_ID_HEADER, correlationId);
        
        try {
            filterChain.doFilter(request, response);
        } finally {
            MDC.remove("correlationId");
        }
    }
}

# prometheus-alerts.yml
groups:
- name: service-alerts
  rules:
  - alert: HighErrorRate
    expr: sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.05
    for: 5m
    labels:
      severity: critical
      team: backend
    annotations:
      summary: "High error rate detected"
      description: "Error rate is above 5% for the last 5 minutes (current value: {{ $value | humanizePercentage }})"
      
  - alert: SlowResponses
    expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)) > 1
    for: 5m
    labels:
      severity: warning
      team: backend
    annotations:
      summary: "Slow response times detected"
      description: "95th percentile of response time is above 1 second for service {{ $labels.service }}"

# prometheus-rules.yml
groups:
- name: slo-rules
  rules:
  - record: slo:request_availability:ratio_5m
    expr: sum(rate(http_requests_total{status!~"5.."}[5m])) / sum(rate(http_requests_total[5m]))
    
  - record: slo:request_latency:p95_5m
    expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))