Purpose: Comprehensive monitoring, alerting, and observability configuration for Lilith platform
Monitoring Stack Overview#
Architecture#
text
Applications → Metrics Collection → Storage → Visualization → Alerting
↓ ↓ ↓ ↓ ↓
Services Prometheus TSDB Grafana AlertManager
+ +Exporters + + +
Logs Fluentd/Promtail Loki Grafana PagerDuty
+ ↓ ↓ ↓ +
Traces OpenTelemetry Tempo Grafana Slack
Core Components#
text
Metrics: Prometheus + Grafana
Logs: Loki + Grafana
Traces: Tempo + Grafana
Alerting: AlertManager + PagerDuty
Dashboards: Grafana + Custom panels
Prometheus Configuration#
Service Discovery and Scraping#
Core Services Monitoring#
yaml
# prometheus-config.yaml
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
# Kubernetes API server
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels:
[
__meta_kubernetes_namespace,
__meta_kubernetes_service_name,
__meta_kubernetes_endpoint_port_name,
]
action: keep
regex: default;kubernetes;https
# Application services
- job_name: 'lilith-services'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels:
[__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
# Database monitoring
- job_name: 'postgres-exporter'
static_configs:
- targets: ['postgres-exporter:9187']
scrape_interval: 30s
# Infrastructure monitoring
- job_name: 'node-exporter'
kubernetes_sd_configs:
- role: node
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
Service-Specific Metrics#
BFF Service Metrics#
yaml
# BFF service annotations for Prometheus scraping
apiVersion: apps/v1
kind: Deployment
metadata:
name: bff-deployment
namespace: lilith-bff
spec:
template:
metadata:
annotations:
prometheus.io/scrape: 'true'
prometheus.io/port: '8080'
prometheus.io/path: '/metrics'
spec:
containers:
- name: bff
image: lilith/bff:latest
ports:
- containerPort: 8080
name: http
- containerPort: 9090
name: metrics
Custom Application Metrics#
javascript
// Example: BFF service metrics implementation
const client = require('prom-client');
// Create a Registry
const register = new client.Registry();
// HTTP request metrics
const httpRequestDuration = new client.Histogram({
name: 'http_request_duration_seconds',
help: 'Duration of HTTP requests in seconds',
labelNames: ['method', 'route', 'status_code'],
buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10],
});
const httpRequestsTotal = new client.Counter({
name: 'http_requests_total',
help: 'Total number of HTTP requests',
labelNames: ['method', 'route', 'status_code'],
});
// Business metrics
const activeUsers = new client.Gauge({
name: 'active_users_total',
help: 'Number of currently active users',
});
const conversationsCreated = new client.Counter({
name: 'conversations_created_total',
help: 'Total number of conversations created',
});
// Register metrics
register.registerMetric(httpRequestDuration);
register.registerMetric(httpRequestsTotal);
register.registerMetric(activeUsers);
register.registerMetric(conversationsCreated);
// Metrics endpoint
app.get('/metrics', async (req, res) => {
res.set('Content-Type', register.contentType);
res.end(await register.metrics());
});
Grafana Dashboard Configuration#
Service-Level Dashboards#
BFF Service Dashboard#
json
{
"dashboard": {
"title": "BFF Service Overview",
"panels": [
{
"title": "Request Rate",
"type": "stat",
"targets": [
{
"expr": "sum(rate(http_requests_total{job=\"bff\"}[5m]))",
"legendFormat": "Requests/sec"
}
],
"gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }
},
{
"title": "Error Rate",
"type": "stat",
"targets": [
{
"expr": "sum(rate(http_requests_total{job=\"bff\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"bff\"}[5m])) * 100",
"legendFormat": "Error %"
}
],
"thresholds": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 2 },
{ "color": "red", "value": 5 }
],
"gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }
},
{
"title": "Response Time (95th percentile)",
"type": "stat",
"targets": [
{
"expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"bff\"}[5m])) by (le))",
"legendFormat": "95th percentile"
}
],
"unit": "s",
"gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }
},
{
"title": "Active Connections",
"type": "stat",
"targets": [
{
"expr": "sum(active_users_total{job=\"bff\"})",
"legendFormat": "Active Users"
}
],
"gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }
}
],
"time": { "from": "now-1h", "to": "now" },
"refresh": "30s"
}
}
Platform Overview Dashboard#
json
{
"dashboard": {
"title": "Lilith Platform Overview",
"panels": [
{
"title": "Service Health Status",
"type": "stat",
"targets": [
{
"expr": "up{job=~\"bff|auth|conversation|content|media|ai\"}",
"legendFormat": "{{job}}"
}
],
"valueMappings": [
{ "value": 1, "text": "UP", "color": "green" },
{ "value": 0, "text": "DOWN", "color": "red" }
]
},
{
"title": "Request Volume by Service",
"type": "graph",
"targets": [
{
"expr": "sum by (job) (rate(http_requests_total[5m]))",
"legendFormat": "{{job}}"
}
]
},
{
"title": "Error Rate by Service",
"type": "graph",
"targets": [
{
"expr": "sum by (job) (rate(http_requests_total{status=~\"5..\"}[5m])) / sum by (job) (rate(http_requests_total[5m])) * 100",
"legendFormat": "{{job}} errors"
}
]
},
{
"title": "Database Connections",
"type": "graph",
"targets": [
{
"expr": "pg_stat_database_numbackends{datname=\"lilith\"}",
"legendFormat": "Active connections"
}
]
}
]
}
}
Alerting Rules and Configuration#
Prometheus Alerting Rules#
Critical Service Alerts#
yaml
# alerts/critical-services.yml
groups:
- name: critical-services
rules:
# Service down alert
- alert: ServiceDown
expr: up{job=~"bff|auth|conversation"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: 'Service {{$labels.job}} is down'
description:
'Service {{$labels.job}} has been down for more than 1 minute.'
# High error rate alert
- alert: HighErrorRate
expr: |
(
sum(rate(http_requests_total{status=~"5.."}[5m])) by (job)
/
sum(rate(http_requests_total[5m])) by (job)
) * 100 > 5
for: 5m
labels:
severity: warning
annotations:
summary: 'High error rate on {{$labels.job}}'
description: 'Error rate is {{$value}}% for {{$labels.job}}'
# High response time alert
- alert: HighResponseTime
expr: |
histogram_quantile(0.95,
sum(rate(http_request_duration_seconds_bucket[5m])) by (le, job)
) > 2
for: 5m
labels:
severity: warning
annotations:
summary: 'High response time on {{$labels.job}}'
description:
'95th percentile response time is {{$value}}s for {{$labels.job}}'
# Database connection pool exhaustion
- alert: DatabaseConnectionPoolHigh
expr: pg_stat_database_numbackends{datname="lilith"} > 80
for: 2m
labels:
severity: warning
annotations:
summary: 'Database connection pool usage high'
description: 'Database has {{$value}} active connections'
# Pod crash loop
- alert: PodCrashLooping
expr: rate(kube_pod_container_status_restarts_total[15m]) > 0
for: 5m
labels:
severity: warning
annotations:
summary: 'Pod {{$labels.pod}} is crash looping'
description:
'Pod {{$labels.pod}} in namespace {{$labels.namespace}} is
restarting frequently'
AlertManager Configuration#
Routing and Notification Setup#
yaml
# alertmanager.yml
global:
smtp_smarthost: 'mail.lilith.com:587'
smtp_from: 'alerts@lilith.com'
route:
group_by: ['alertname', 'cluster', 'service']
group_wait: 10s
group_interval: 10s
repeat_interval: 12h
receiver: 'web.hook'
routes:
- match:
severity: critical
receiver: 'pagerduty-critical'
repeat_interval: 5m
- match:
severity: warning
receiver: 'slack-warnings'
repeat_interval: 30m
receivers:
- name: 'web.hook'
webhook_configs:
- url: 'http://webhook.lilith.com/alerts'
- name: 'pagerduty-critical'
pagerduty_configs:
- service_key: 'your-pagerduty-integration-key'
description: '{{range .Alerts}}{{.Annotations.summary}}{{end}}'
- name: 'slack-warnings'
slack_configs:
- api_url: 'YOUR_SLACK_WEBHOOK_URL'
channel: '#lilith-alerts'
title: 'Alert: {{.GroupLabels.alertname}}'
text: '{{range .Alerts}}{{.Annotations.description}}{{end}}'
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
equal: ['alertname', 'cluster', 'service']
Log Aggregation and Analysis#
Loki Configuration for Log Aggregation#
Loki Configuration#
yaml
# loki-config.yml
auth_enabled: false
server:
http_listen_port: 3100
ingester:
lifecycler:
address: 127.0.0.1
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: /loki/boltdb-shipper-active
cache_location: /loki/boltdb-shipper-cache
shared_store: filesystem
filesystem:
directory: /loki/chunks
limits_config:
enforce_metric_name: false
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s
Structured Logging Setup#
Application Logging Standards#
javascript
// Example: Structured logging in Node.js services
const winston = require('winston');
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(
winston.format.timestamp(),
winston.format.errors({ stack: true }),
winston.format.json()
),
defaultMeta: {
service: process.env.SERVICE_NAME || 'unknown',
version: process.env.SERVICE_VERSION || 'unknown',
environment: process.env.NODE_ENV || 'development',
},
transports: [
new winston.transports.Console(),
new winston.transports.File({
filename: '/var/log/app/error.log',
level: 'error',
}),
new winston.transports.File({ filename: '/var/log/app/combined.log' }),
],
});
// Usage examples
logger.info('User authentication successful', {
userId: '12345',
email: 'user@example.com',
duration: 150,
endpoint: '/api/auth/login',
});
logger.error('Database connection failed', {
error: error.message,
stack: error.stack,
database: 'primary',
retryAttempt: 3,
});
logger.warn('High response time detected', {
endpoint: '/api/chat/messages',
responseTime: 2500,
threshold: 2000,
});
Distributed Tracing Setup#
OpenTelemetry Configuration#
Trace Collection Setup#
javascript
// OpenTelemetry setup for Node.js services
const { NodeTracerProvider } = require('@opentelemetry/sdk-node');
const { Resource } = require('@opentelemetry/resources');
const {
SemanticResourceAttributes,
} = require('@opentelemetry/semantic-conventions');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');
const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base');
// Initialize tracing
const provider = new NodeTracerProvider({
resource: new Resource({
[SemanticResourceAttributes.SERVICE_NAME]: process.env.SERVICE_NAME,
[SemanticResourceAttributes.SERVICE_VERSION]: process.env.SERVICE_VERSION,
}),
});
// Configure Jaeger exporter
const jaegerExporter = new JaegerExporter({
endpoint: process.env.JAEGER_ENDPOINT || 'http://jaeger:14268/api/traces',
});
provider.addSpanProcessor(new BatchSpanProcessor(jaegerExporter));
provider.register();
// Auto-instrument HTTP requests
const {
getNodeAutoInstrumentations,
} = require('@opentelemetry/auto-instrumentations-node');
const { registerInstrumentations } = require('@opentelemetry/instrumentation');
registerInstrumentations({
instrumentations: [
getNodeAutoInstrumentations({
'@opentelemetry/instrumentation-fs': { enabled: false },
}),
],
});
Monitoring Automation Scripts#
Health Check Automation#
bash
#!/bin/bash
# Automated health check and alerting script
SERVICES=("bff" "auth" "conversation" "content" "media" "ai")
HEALTH_CHECK_LOG="/var/log/health-checks.log"
perform_health_check() {
local service=$1
local health_url="https://${service}.lilith.com/v1/health"
echo "$(date): Checking $service health..." >> $HEALTH_CHECK_LOG
response=$(curl -s -w "%{http_code}:%{time_total}" "$health_url")
http_code=$(echo $response | cut -d: -f1)
response_time=$(echo $response | cut -d: -f2)
if [ "$http_code" = "200" ]; then
echo "$(date): ✓ $service healthy (${response_time}s)" >> $HEALTH_CHECK_LOG
# Check if response time is acceptable
if (( $(echo "$response_time > 2.0" | bc -l) )); then
./scripts/send-alert.sh "⚠️ $service slow response time: ${response_time}s"
fi
else
echo "$(date): ✗ $service unhealthy (HTTP $http_code)" >> $HEALTH_CHECK_LOG
./scripts/send-alert.sh "🚨 $service health check failed (HTTP $http_code)"
fi
}
# Check all services
for service in "${SERVICES[@]}"; do
perform_health_check "$service" &
done
wait
echo "$(date): Health check cycle completed" >> $HEALTH_CHECK_LOG
Metrics Collection Validation#
bash
#!/bin/bash
# Validate that metrics are being collected properly
validate_metrics_collection() {
local service=$1
local prometheus_url="http://prometheus.lilith.com:9090"
echo "Validating metrics collection for $service..."
# Check if service is being scraped
scrape_status=$(curl -s "$prometheus_url/api/v1/query?query=up{job=\"$service\"}" | jq -r '.data.result[0].value[1]')
if [ "$scrape_status" = "1" ]; then
echo "✓ $service metrics are being scraped"
else
echo "✗ $service metrics scraping failed"
return 1
fi
# Check if custom metrics are present
custom_metrics=$(curl -s "$prometheus_url/api/v1/query?query=http_requests_total{job=\"$service\"}" | jq -r '.data.result | length')
if [ "$custom_metrics" -gt 0 ]; then
echo "✓ $service custom metrics are available"
else
echo "✗ $service custom metrics missing"
return 1
fi
echo "✓ Metrics validation completed for $service"
}
# Validate all services
for service in bff auth conversation content; do
validate_metrics_collection "$service"
done
Remember: Effective monitoring requires the right balance of coverage, noise reduction, and actionable alerts. Focus on metrics that indicate user impact and business value, not just technical health.