Observability
& SRE

// COUNTER — monotonically increasing. Use rate() to get per-second rate.
http_requests_total{method="GET", status="200"} 145231
rate(http_requests_total[5m])  → requests/sec over 5-min window

// GAUGE — current value, can go up or down. Query directly.
process_memory_bytes 524288000          → 500MB currently in use
db_connection_pool_active 45            → 45 of 100 connections in use

// HISTOGRAM — distribution in buckets. Enables percentile calculation.
http_request_duration_seconds_bucket{le="0.05"}  8920   ≤50ms: 8920 requests
http_request_duration_seconds_bucket{le="0.1"}   9543   ≤100ms: 9543 requests
http_request_duration_seconds_bucket{le="0.5"}   9981   ≤500ms: 9981 requests
http_request_duration_seconds_bucket{le="Inf"}  10000   total: 10000 requests
// p99: histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

// USE method (for resources): Utilization, Saturation, Errors
// RED method (for services): Rate, Errors, Duration

{
  "timestamp":   "2025-03-07T14:23:45.123Z",
  "level":        "ERROR",
  "service":      "payment-service",
  "version":      "v2.4.1",
  "trace_id":     "abc123def456",    ← links to distributed trace
  "span_id":      "7890abcd",
  "user_id":      "u_98765",          ← hashed if GDPR-sensitive
  "order_id":     "ord_12345",
  "message":      "Payment processing failed",
  "error_code":   "CARD_DECLINED",
  "amount_cents": 9999,
  "duration_ms":  234,
  "host":         "payment-pod-7d4b9c"
}

// log levels: DEBUG (dev) → INFO (business events) → WARN (recoverable)
//             → ERROR (needs investigation) → FATAL (immediate action)

// Log pipeline: App → Fluentd/Logstash → Kafka (buffer) → Elasticsearch → Kibana
// Cheaper alternative: App → Promtail → Loki (label-based) → Grafana

// TAIL-BASED SAMPLING (preferred):
// Keep 100% of ERROR/WARN logs. Sample 1% of INFO. Discard DEBUG.
// Preserves signal, reduces storage cost by ~50x on high-traffic services.

// Retention tiers:
// Hot  (Elasticsearch):  7 days   → fast full-text search, recent incidents
// Warm (S3-backed):      30 days  → slower queries, post-incident review
// Cold (S3 Glacier):     90 days  → compliance only, rarely queried

// Trace: place_order request (trace_id: abc123)
// Each bar = one span (one service's processing time)

Span 1: API Gateway          |████████████████████████| 95ms total
  Span 2: Auth Service         |██| 3ms
  Span 3: Order Service          |████████████████| 80ms total
    Span 4: Feature Store call     |████| 8ms
    Span 5: MySQL write            |████████████| 60ms  ← BOTTLENECK
    Span 6: Kafka publish          |██| 4ms
  Span 7: Notification Service   |████| 8ms

// Without tracing: "order service is slow" — check all of its dependencies
// With tracing: "MySQL write is taking 60ms" — go check DB explain plan, indexes

// Context propagation via HTTP headers (B3 format, used by Zipkin/Jaeger):
X-B3-TraceId:    abc123def456789   ← same for all spans in this trace
X-B3-SpanId:     7890abcd           ← unique per span
X-B3-ParentSpanId: 1234efgh         ← parent span's ID
X-B3-Sampled:    1                  ← 1=sample this trace, 0=don't

// OpenTelemetry (OTel): vendor-neutral standard
// Write instrumentation once → export to Jaeger, Datadog, or any backend
// SDK: Java, Python, Go, Node.js — all supported

// Sampling strategy:
// Head-based: decide at trace root (random 1%) — simple but misses rare errors
// Tail-based: decide after trace completes — keep 100% of errors/slow traces
// Preferred: tail-based with head-based as fallback

// Error budget = 100% - SLO = allowed failure rate

SLO = 99.9%  →  error budget = 0.1% of requests may fail
  Over 30 days: 0.001 × 30d × 24h × 60m = 43.2 minutes total downtime allowed

SLO = 99.99% →  error budget = 0.01%
  Over 30 days: 4.38 minutes total downtime allowed

SLO = 99.5%  →  error budget = 0.5%
  Over 30 days: 3.6 hours total downtime allowed

// Choose SLO based on tier, not ambition:
Critical path (payment, auth login):  99.99%  →  4.38 min/month
Core product (feed, search, API):     99.9%   → 43.2 min/month
Non-critical (analytics, recs, admin):99.5%   →  3.6 hr/month

// Tighter SLO = fewer feature deployments = slower iteration
// Setting 99.99% for everything = no deploys ever = engineering paralysis

// Burn rate = how fast you're consuming the error budget
// Burn rate 1.0 = consuming at exactly the SLO rate (budget depletes at month end)
// Burn rate 14.4 = consuming 14.4× faster → 1hr window consumes 2% of 30-day budget

// FAST BURN — page immediately (short window catches acute outage)
alert: ErrorBudgetBurnFast
expr: rate(http_errors[1h]) / rate(http_requests[1h]) > (14.4 * 0.001)
       ^^ error rate            ^^ burns budget 14.4× faster than allowed
for: 2m
severity: page
message: "Fast error budget burn — action required immediately"

// SLOW BURN — urgent ticket (longer window catches gradual degradation)
alert: ErrorBudgetBurnSlow
expr: rate(http_errors[6h]) / rate(http_requests[6h]) > (6 * 0.001)
for: 15m
severity: ticket

// BAD ALERT — avoid "CPU > 80%"
// CPU can be high while system is healthy (batch job running)
// CPU can be low while system is broken (stuck waiting for DB)
// Alert on what users experience, not what internal resources are doing

// Alert fatigue rule: if alert fires >1/shift → raise threshold, add duration, or demote