Advanced Observability

# Add to docker-compose.yml

  postgres-exporter:
    image: prometheuscommunity/postgres-exporter:latest
    container_name: postgres-exporter
    restart: unless-stopped
    environment:
      - DATA_SOURCE_NAME=postgresql://username:password@postgres:5432/authentik?sslmode=disable
    ports:
      - "9187:9187"
    networks:
      - monitoring

# Add to prometheus.yml scrape_configs:
  - job_name: 'postgres'
    static_configs:
      - targets: ['postgres-exporter:9187']

# Create blackbox.yml config
cat > ~/monitoring/blackbox/blackbox.yml << 'EOF'
modules:
  http_2xx:
    prober: http
    timeout: 5s
    http:
      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
      valid_status_codes: [200, 201, 301, 302]
      method: GET
      follow_redirects: true
      preferred_ip_protocol: "ip4"
  
  tcp_connect:
    prober: tcp
    timeout: 5s
EOF

# Docker Compose service
  blackbox-exporter:
    image: prom/blackbox-exporter:latest
    container_name: blackbox-exporter
    restart: unless-stopped
    ports:
      - "9115:9115"
    volumes:
      - ./blackbox/blackbox.yml:/etc/blackbox_exporter/config.yml:ro
    command:
      - '--config.file=/etc/blackbox_exporter/config.yml'
    networks:
      - monitoring

# Prometheus scrape config for HTTP probes
  - job_name: 'blackbox-http'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
          - https://authentik.yourdomain.com
          - https://vault.yourdomain.com
          - https://grafana.yourdomain.com
          - https://teleport.yourdomain.com
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

# Add to traefik.yml or command line:
metrics:
  prometheus:
    buckets:
      - 0.1
      - 0.3
      - 1.2
      - 5.0
    addEntryPointsLabels: true
    addRoutersLabels: true
    addServicesLabels: true
    entryPoint: metrics

entryPoints:
  metrics:
    address: ":8082"

# Prometheus scrape config
  - job_name: 'traefik'
    static_configs:
      - targets: ['traefik:8082']

# Add to docker-compose.yml

  alertmanager:
    image: prom/alertmanager:latest
    container_name: alertmanager
    restart: unless-stopped
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro
      - ./alertmanager/data:/alertmanager
    command:
      - '--config.file=/etc/alertmanager/alertmanager.yml'
      - '--storage.path=/alertmanager'
    networks:
      - monitoring

cat > ~/monitoring/alertmanager/alertmanager.yml << 'EOF'
global:
  resolve_timeout: 5m
  slack_api_url: 'YOUR_SLACK_WEBHOOK_URL'

route:
  receiver: 'default-receiver'
  group_by: ['alertname', 'severity', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  
  routes:
    # Critical alerts -> Slack immediately
    - match:
        severity: critical
      receiver: 'slack-critical'
      group_wait: 10s
      repeat_interval: 1h
    
    # High alerts -> Slack
    - match:
        severity: high
      receiver: 'slack-high'
    
    # Warning alerts -> Email
    - match:
        severity: warning
      receiver: 'email-warnings'
    
    # Security alerts -> Dedicated channel
    - match:
        service: wazuh
      receiver: 'slack-security'

receivers:
  - name: 'default-receiver'
    slack_configs:
      - channel: '#alerts'
        send_resolved: true
        title: '{{ .Status | toUpper }}: {{ .CommonAnnotations.summary }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

  - name: 'slack-critical'
    slack_configs:
      - channel: '#alerts-critical'
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'
        title: '🚨 CRITICAL: {{ .CommonAnnotations.summary }}'
        text: |
          *Alert:* {{ .CommonLabels.alertname }}
          *Description:* {{ .CommonAnnotations.description }}
          *Severity:* {{ .CommonLabels.severity }}
          {{ range .Alerts }}
          *Details:*
          {{ range .Labels.SortedPairs }} - {{ .Name }}: {{ .Value }}
          {{ end }}
          {{ end }}

  - name: 'slack-high'
    slack_configs:
      - channel: '#alerts'
        send_resolved: true
        color: 'warning'
        title: '⚠️ HIGH: {{ .CommonAnnotations.summary }}'

  - name: 'slack-security'
    slack_configs:
      - channel: '#security-alerts'
        send_resolved: true
        color: 'danger'
        title: '🛡️ Security: {{ .CommonAnnotations.summary }}'

  - name: 'email-warnings'
    email_configs:
      - to: 'alerts@yourdomain.com'
        from: 'alertmanager@yourdomain.com'
        smarthost: 'smtp.gmail.com:587'
        auth_username: 'your-email@gmail.com'
        auth_password: 'your-app-password'
        send_resolved: true

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'service']
EOF

cat > ~/monitoring/prometheus/alert-rules.yml << 'EOF'
groups:
  - name: infrastructure
    rules:
      # High CPU
      - alert: HighCPU
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
          service: infrastructure
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "CPU usage is {{ $value | printf \"%.1f\" }}%"

      # Critical CPU
      - alert: CriticalCPU
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 95
        for: 2m
        labels:
          severity: critical
          service: infrastructure
        annotations:
          summary: "Critical CPU on {{ $labels.instance }}"
          description: "CPU at {{ $value | printf \"%.1f\" }}% - immediate attention needed"

      # High Memory
      - alert: HighMemory
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
        for: 5m
        labels:
          severity: warning
          service: infrastructure
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
          description: "Memory usage is {{ $value | printf \"%.1f\" }}%"

      # Disk Almost Full
      - alert: DiskAlmostFull
        expr: (1 - (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"})) * 100 > 85
        for: 5m
        labels:
          severity: critical
          service: infrastructure
        annotations:
          summary: "Disk almost full on {{ $labels.instance }}"
          description: "Disk usage is {{ $value | printf \"%.1f\" }}%"

  - name: services
    rules:
      # Service Down
      - alert: ServiceDown
        expr: up{job=~"blackbox-http"} == 0
        for: 1m
        labels:
          severity: critical
          service: "{{ $labels.instance }}"
        annotations:
          summary: "Service {{ $labels.instance }} is DOWN"
          description: "The service has been unreachable for more than 1 minute"

      # High Response Time
      - alert: HighResponseTime
        expr: probe_http_duration_seconds{phase="transfer"} > 2
        for: 5m
        labels:
          severity: warning
          service: "{{ $labels.instance }}"
        annotations:
          summary: "Slow response from {{ $labels.instance }}"
          description: "Response time is {{ $value | printf \"%.2f\" }}s"

  - name: containers
    rules:
      # Container Down
      - alert: ContainerDown
        expr: absent(container_last_seen{name=~".+"})
        for: 1m
        labels:
          severity: critical
          service: docker
        annotations:
          summary: "Container {{ $labels.name }} is down"
          description: "Container has stopped running"

      # High Container Memory
      - alert: ContainerHighMemory
        expr: (container_memory_usage_bytes{name!=""} / container_spec_memory_limit_bytes{name!=""}) * 100 > 80
        for: 5m
        labels:
          severity: warning
          service: docker
        annotations:
          summary: "Container {{ $labels.name }} high memory"
          description: "Memory usage is {{ $value | printf \"%.1f\" }}%"
EOF

# Update prometheus.yml to include rules:
rule_files:
  - /etc/prometheus/alert-rules.yml

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

# Add MinIO to docker-compose.yml

  minio:
    image: minio/minio:latest
    container_name: minio
    restart: unless-stopped
    ports:
      - "9000:9000"
      - "9001:9001"
    environment:
      - MINIO_ROOT_USER=minioadmin
      - MINIO_ROOT_PASSWORD=YOUR_SECURE_PASSWORD
    command: server /data --console-address ":9001"
    volumes:
      - ./minio/data:/data
    networks:
      - monitoring

# Create bucket for Thanos
# Access MinIO console at http://SERVER_IP:9001
# Create bucket named "thanos"

# Create bucket config
cat > ~/monitoring/thanos/bucket.yml << 'EOF'
type: S3
config:
  bucket: thanos
  endpoint: minio:9000
  access_key: minioadmin
  secret_key: YOUR_SECURE_PASSWORD
  insecure: true
EOF

# Update Prometheus to allow Thanos access
# Add to prometheus command:
  prometheus:
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=2h'  # Short retention, Thanos handles long-term
      - '--storage.tsdb.min-block-duration=2h'
      - '--storage.tsdb.max-block-duration=2h'
      - '--web.enable-lifecycle'
      - '--web.enable-admin-api'

# Add Thanos Sidecar
  thanos-sidecar:
    image: quay.io/thanos/thanos:latest
    container_name: thanos-sidecar
    restart: unless-stopped
    command:
      - sidecar
      - --tsdb.path=/prometheus
      - --prometheus.url=http://prometheus:9090
      - --objstore.config-file=/etc/thanos/bucket.yml
      - --grpc-address=0.0.0.0:10901
      - --http-address=0.0.0.0:10902
    volumes:
      - ./prometheus/data:/prometheus:ro
      - ./thanos/bucket.yml:/etc/thanos/bucket.yml:ro
    depends_on:
      - prometheus
      - minio
    networks:
      - monitoring

# Add to docker-compose.yml

  thanos-store:
    image: quay.io/thanos/thanos:latest
    container_name: thanos-store
    restart: unless-stopped
    command:
      - store
      - --data-dir=/tmp/thanos/store
      - --objstore.config-file=/etc/thanos/bucket.yml
      - --grpc-address=0.0.0.0:10901
      - --http-address=0.0.0.0:10902
    volumes:
      - ./thanos/bucket.yml:/etc/thanos/bucket.yml:ro
      - ./thanos/store:/tmp/thanos/store
    depends_on:
      - minio
    networks:
      - monitoring

  thanos-query:
    image: quay.io/thanos/thanos:latest
    container_name: thanos-query
    restart: unless-stopped
    ports:
      - "10902:10902"
    command:
      - query
      - --http-address=0.0.0.0:10902
      - --grpc-address=0.0.0.0:10901
      - --store=thanos-sidecar:10901
      - --store=thanos-store:10901
    depends_on:
      - thanos-sidecar
      - thanos-store
    networks:
      - monitoring

  thanos-compactor:
    image: quay.io/thanos/thanos:latest
    container_name: thanos-compactor
    restart: unless-stopped
    command:
      - compact
      - --data-dir=/tmp/thanos/compact
      - --objstore.config-file=/etc/thanos/bucket.yml
      - --wait
    volumes:
      - ./thanos/bucket.yml:/etc/thanos/bucket.yml:ro
      - ./thanos/compact:/tmp/thanos/compact
    depends_on:
      - minio
    networks:
      - monitoring

# Update datasources.yml - change Prometheus URL to Thanos

  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://thanos-query:10902  # Changed from prometheus:9090
    isDefault: true
    editable: false
    jsonData:
      httpMethod: POST
      manageAlerts: true
      prometheusType: Thanos
      prometheusVersion: 0.31.0