Database Monitoring: On-Prem (K8s & EC2)

Prometheus + Grafana stack for MySQL, PostgreSQL, Redis, MongoDB running on-prem or EC2/self-managed K8s.

Architecture

graph TD
    classDef blue fill:#3498db,stroke:#2980b9,color:#fff
    classDef orange fill:#e67e22,stroke:#d35400,color:#fff
    classDef green fill:#2ecc71,stroke:#27ae60,color:#fff
    classDef teal fill:#1abc9c,stroke:#16a085,color:#fff
    classDef red fill:#e74c3c,stroke:#c0392b,color:#fff
    classDef purple fill:#9b59b6,stroke:#8e44ad,color:#fff

    subgraph DBLayer["Database Layer — K8s pods or EC2 VMs"]
        PG["PostgreSQL<br/>native protocol :5432"]:::blue
        MY["MySQL<br/>native protocol :3306"]:::blue
        RD["Redis<br/>native protocol :6379"]:::blue
        MG["MongoDB<br/>native protocol :27017"]:::blue
    end

    subgraph Exporters["Exporters — sidecar or separate pod/process<br/>translate native stats into Prometheus text format"]
        PGE["postgres_exporter<br/>:9187"]:::teal
        MYE["mysqld_exporter<br/>:9104"]:::teal
        RDE["redis_exporter<br/>:9121"]:::teal
        MGE["mongodb_exporter<br/>:9216"]:::teal
    end

    subgraph Observability["Observability Stack"]
        PROM["Prometheus<br/>scrapes /metrics every 15s<br/>evaluates alert rules on the same data"]:::orange
        ALERT["Alertmanager<br/>dedupes, groups, routes firing alerts"]:::red
        GRAF["Grafana<br/>dashboards, queries Prometheus directly"]:::green
        LOKI["Loki (optional)<br/>log aggregation, correlate with metrics"]:::orange
    end

    subgraph Channels["Notification Channels"]
        PD["PagerDuty<br/>critical severity, pages on-call"]:::red
        SLACK["Slack<br/>warning severity, team channel"]:::purple
        EMAIL["Email<br/>low-urgency digest"]:::purple
    end

    PG -->|"read-only monitoring user<br/>queries pg_stat_* views"| PGE
    MY -->|"read-only monitoring user<br/>queries SHOW STATUS"| MYE
    RD -->|"AUTH + INFO command"| RDE
    MG -->|"read-only monitoring user<br/>runs serverStatus()"| MGE

    PGE & MYE & RDE & MGE -->|"scrape /metrics<br/>every 15s"| PROM
    PROM -->|"rule evaluation interval"| ALERT
    PROM -->|"PromQL queries"| GRAF
    ALERT -->|"critical"| PD
    ALERT -->|"warning"| SLACK
    ALERT -->|"info"| EMAIL

Exporters are the bridge — they query the DB using native protocol and expose Prometheus /metrics endpoint. None of these four databases speak Prometheus's text format on their own; the exporter's entire job is that one translation step.

Setup on Kubernetes

Prometheus + Grafana via kube-prometheus-stack

helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
  --namespace monitoring --create-namespace \
  --set grafana.adminPassword=changeme \
  --set prometheus.prometheusSpec.retention=30d \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName=gp3 \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=50Gi

This installs: Prometheus, Grafana, Alertmanager, node-exporter DaemonSet, kube-state-metrics.

PostgreSQL Exporter

# postgres-exporter deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: postgres-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: postgres-exporter
  template:
    metadata:
      labels:
        app: postgres-exporter
      annotations:
        prometheus.io/scrape: "true"   # auto-discovery
        prometheus.io/port: "9187"
    spec:
      containers:
      - name: exporter
        image: prometheuscommunity/postgres-exporter:v0.15.0
        env:
        - name: DATA_SOURCE_NAME
          valueFrom:
            secretKeyRef:
              name: postgres-exporter-secret
              key: dsn      # postgresql://monitor_user:pass@postgres-svc:5432/mydb?sslmode=disable
        ports:
        - containerPort: 9187
---
# Read-only monitoring user (least privilege)
# CREATE USER monitor WITH PASSWORD 'pass';
# GRANT pg_monitor TO monitor;   -- PostgreSQL 10+

# ServiceMonitor tells Prometheus to scrape this service
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: postgres-exporter
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: postgres-exporter
  endpoints:
  - port: metrics
    interval: 15s

Redis Exporter

apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis-exporter
  namespace: monitoring
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis-exporter
  template:
    metadata:
      labels:
        app: redis-exporter
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "9121"
    spec:
      containers:
      - name: exporter
        image: oliver006/redis_exporter:v1.55.0
        env:
        - name: REDIS_ADDR
          value: "redis://redis-svc:6379"
        - name: REDIS_PASSWORD
          valueFrom:
            secretKeyRef:
              name: redis-secret
              key: password
        ports:
        - containerPort: 9121

Setup on EC2 (bare metal / VM)

# Download and run postgres_exporter as a systemd service
wget https://github.com/prometheus-community/postgres_exporter/releases/download/v0.15.0/postgres_exporter-0.15.0.linux-amd64.tar.gz
tar xvf postgres_exporter*.tar.gz

# Create systemd unit
cat > /etc/systemd/system/postgres-exporter.service << EOF
[Unit]
Description=Postgres Exporter

[Service]
Environment="DATA_SOURCE_NAME=postgresql://monitor:pass@localhost:5432/mydb?sslmode=disable"
ExecStart=/usr/local/bin/postgres_exporter
Restart=always

[Install]
WantedBy=multi-user.target
EOF

systemctl enable --now postgres-exporter

# Tell Prometheus to scrape this EC2 instance
# In prometheus.yml on the Prometheus server:
# scrape_configs:
#   - job_name: postgres
#     static_configs:
#       - targets: ['10.0.1.50:9187']   # EC2 private IP

Key Metrics to Monitor

Same shape of question for all four engines — is it up, is it running out of headroom (connections/memory/disk), is it falling behind on replication, is it serving queries slowly — but each exposes that through its own metric names and its own notion of "healthy." Flip between them below.

Alerting Rules

# prometheus-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: database-alerts
  namespace: monitoring
spec:
  groups:
  - name: postgres
    rules:
    - alert: PostgresDown
      expr: pg_up == 0
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: "PostgreSQL is down on {{ $labels.instance }}"

    - alert: PostgresHighConnections
      expr: pg_stat_activity_count / pg_settings_max_connections > 0.8
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "PostgreSQL connections above 80% ({{ $value | humanizePercentage }})"

    - alert: PostgresLongRunningQuery
      expr: pg_stat_activity_max_tx_duration > 300
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: "Query running > 5min on {{ $labels.instance }}"

    - alert: PostgresReplicationLag
      expr: pg_stat_replication_pg_wal_lsn_diff > 52428800   # 50MB
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "Postgres replica lag {{ $value | humanize1024 }}B"

  - name: redis
    rules:
    - alert: RedisDown
      expr: redis_up == 0
      for: 1m
      labels:
        severity: critical

    - alert: RedisHighMemory
      expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.8
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: "Redis memory above 80% on {{ $labels.instance }}"

    - alert: RedisCacheHitRateLow
      expr: |
        rate(redis_keyspace_hits_total[5m]) /
        (rate(redis_keyspace_hits_total[5m]) + rate(redis_keyspace_misses_total[5m])) < 0.9
      for: 10m
      labels:
        severity: warning
      annotations:
        summary: "Redis hit rate below 90%: {{ $value | humanizePercentage }}"

Notice every rule pairs its expr with a for: duration before it's allowed to fire. That's a debounce: the condition has to hold continuously for the full window, not just be true on one 15s scrape, before Alertmanager sees it as firing. PostgresDown uses a short for: 1m because a real outage should page fast, while RedisCacheHitRateLow uses for: 10m because hit rate naturally dips for a few seconds under a traffic burst and that alone isn't worth waking anyone up. The severity label (critical vs warning) is what routes the alert to the right channel downstream in Alertmanager — it's data the rule attaches, not something Alertmanager infers on its own.

Grafana Dashboards

Import pre-built dashboards from grafana.com by ID:

# Import via Grafana API
curl -X POST http://admin:changeme@localhost:3000/api/dashboards/import \
  -H "Content-Type: application/json" \
  -d '{"gnetId": 9628, "overwrite": true, "folderId": 0}'

What to Put on the DB Overview Dashboard

graph TD
    classDef health fill:#e74c3c,stroke:#c0392b,color:#fff
    classDef perf fill:#3498db,stroke:#2980b9,color:#fff
    classDef res fill:#e67e22,stroke:#d35400,color:#fff
    classDef cache fill:#1abc9c,stroke:#16a085,color:#fff
    classDef alerts fill:#9b59b6,stroke:#8e44ad,color:#fff

    subgraph Row1["Row 1 — Health (is it even up?)"]
        H1["pg_up / redis_up / mysql_up<br/>single-stat, red when any = 0"]:::health
        H2["Active connections<br/>vs max_connections / maxclients"]:::health
        H3["Replication lag<br/>across all 4 engines, one panel"]:::health
    end

    subgraph Row2["Row 2 — Performance (is it keeping up?)"]
        P1["Queries/sec<br/>opcounters, ops_processed"]:::perf
        P2["Avg query latency<br/>op_latencies, exec_time"]:::perf
        P3["Slow queries/min<br/>slow_queries, long-running tx"]:::perf
    end

    subgraph Row3["Row 3 — Resources (is it about to run out?)"]
        R1["Memory used vs limit<br/>resident RAM, maxmemory"]:::res
        R2["Disk used %<br/>database_size_bytes"]:::res
        R3["CPU %<br/>from node-exporter, host-level"]:::res
    end

    subgraph Row4["Row 4 — Cache / Buffer (is it hitting disk too often?)"]
        C1["Buffer pool hit rate<br/>MySQL InnoDB"]:::cache
        C2["Redis hit rate<br/>keyspace_hits / (hits+misses)"]:::cache
        C3["Dead tuples<br/>PostgreSQL bloat, needs VACUUM"]:::cache
    end

    subgraph Row5["Row 5 — Alerts firing"]
        A1["Alertmanager panel<br/>everything currently above threshold"]:::alerts
    end

    Row1 --> Row2 --> Row3 --> Row4 --> Row5

The ordering is deliberate: health first because nothing else on the dashboard matters if the database is down, resource pressure before cache/buffer detail because a saturated resource explains a bad hit rate before you go hunting for a query-level cause, and Alertmanager last as the summary row that ties back to everything above it.