Netcup Server-Ressourcen überwachen: CPU Steal (%st), NVMe-Latenz & Prometheus + Grafana

sudo apt update
sudo apt install -y sysstat iotop ioping vnstat htop btop

# 1. Historische sysstat-Aufzeichnung alle 10 Minuten aktivieren (/var/log/sysstat)
sudo sed -i 's/^ENABLED="false"/ENABLED="true"/' /etc/default/sysstat
sudo systemctl enable --now sysstat vnstat

# 2. CPU-Auslastung und Steal Time (%steal) pro vCore im 1-Sekunden-Takt messen (5 Messungen)
mpstat -P ALL 1 5

# 3. Linux-Kernel Pressure Stall Information (CPU-, Memory- und I/O-Stalls) auslesen
head -n 5 /proc/pressure/cpu /proc/pressure/memory /proc/pressure/io

Average:     CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
Average:     all    4.12    0.00    1.05    0.08    0.00    0.12    0.00    0.00    0.00   94.63

==> /proc/pressure/memory <==
some avg10=0.00 avg60=0.00 avg300=0.00 total=1420
full avg10=0.00 avg60=0.00 avg300=0.00 total=810

# 1. Erweiterte Block-Device-Statistiken prüfen (1-Sekunden-Intervall, 3 Berichte)
iostat -xz 1 3

# 2. Zerstörungsfreie 4-KiB-Direct-I/O-Latenz auf dem Root-Dateisystem messen (20 Anfragen)
sudo ioping -c 20 -i 0.1 -D /

# 3. Persistente tägliche und monatliche Netzwerk-Traffic-Summen in vnstat abfragen
vnstat -d 5
vnstat -m

Device            r/s     rkB/s   rrqm/s  %rrqm r_await rareq-sz     w/s     wkB/s   wrqm/s  %wrqm w_await wareq-sz  aqu-sz  %util
vda              8.50    112.00     0.00   0.00    0.28    13.18   24.20    384.50     4.10  14.49    0.41    15.89    0.01   1.20

--- / (ext4 /dev/vda1) ioping statistics ---
19 requests completed in 4.18 ms, 76 KiB read, 4.54 k iops, 17.7 MiB/s
min/avg/max/mdev = 164.2 us / 220.1 us / 389.4 us / 48.3 us

name: netcup_monitoring

services:
  prometheus:
    image: prom/prometheus:v2.54.1
    container_name: mon_prometheus
    restart: unless-stopped
    command:
      - "--config.file=/etc/prometheus/prometheus.yml"
      - "--storage.tsdb.path=/prometheus"
      - "--storage.tsdb.retention.time=30d"
      - "--storage.tsdb.retention.size=4GB"
      - "--web.listen-address=0.0.0.0:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
      - prom_data:/prometheus
    networks:
      - mon_internal
    deploy:
      resources:
        limits:
          cpus: "0.50"
          memory: 512M
    security_opt:
      - no-new-privileges:true

  node-exporter:
    image: prom/node-exporter:v1.8.2
    container_name: mon_node_exporter
    restart: unless-stopped
    command:
      - "--path.procfs=/host/proc"
      - "--path.sysfs=/host/sys"
      - "--path.rootfs=/rootfs"
      - "--collector.pressure"
      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    networks:
      - mon_internal
    deploy:
      resources:
        limits:
          cpus: "0.20"
          memory: 128M
    security_opt:
      - no-new-privileges:true

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.49.1
    container_name: mon_cadvisor
    restart: unless-stopped
    privileged: true
    devices:
      - /dev/kmsg:/dev/kmsg
    command:
      - "--docker_only=true"
      - "--housekeeping_interval=15s"
      - "--disable_metrics=percpu,sched,tcp,udp,diskIO,hugetlb,referenced_memory"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    networks:
      - mon_internal
    deploy:
      resources:
        limits:
          cpus: "0.30"
          memory: 256M

  grafana:
    image: grafana/grafana-oss:11.2.2
    container_name: mon_grafana
    restart: unless-stopped
    ports:
      - "127.0.0.1:3000:3000"
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-ChangeMeImmediately_2026!}
      GF_USERS_ALLOW_SIGN_UP: "false"
      GF_SERVER_HTTP_ADDR: "0.0.0.0"
    volumes:
      - grafana_data:/var/lib/grafana
    networks:
      - mon_internal
    depends_on:
      - prometheus
    deploy:
      resources:
        limits:
          cpus: "0.50"
          memory: 384M
    security_opt:
      - no-new-privileges:true

volumes:
  prom_data:
  grafana_data:

networks:
  mon_internal:
    driver: bridge

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "/etc/prometheus/alerts.yml"

scrape_configs:
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]

  - job_name: "netcup_host_node"
    static_configs:
      - targets: ["node-exporter:9100"]

  - job_name: "netcup_docker_containers"
    static_configs:
      - targets: ["cadvisor:8080"]

groups:
  - name: netcup_infrastructure_alerts
    rules:
      - alert: NetcupHighCpuStealTime
        expr: avg by (instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High Hypervisor CPU Steal (>10%) on {{ $labels.instance }}"
          description: "KVM CPU steal time has exceeded 10% for 10 minutes. Consider migrating workload to a dedicated-core Netcup Root Server (RS)."

      - alert: NetcupRootDiskSpaceLowForSnapshots
        expr: (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})) * 100 > 82
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Root Filesystem >82% Full (SCP Snapshots Risk)"
          description: "Disk usage exceeds 82%. Netcup SCP Copy-on-Write snapshots require free disk headroom to operate."

      - alert: NetcupHighNvmeReadAwaitLatency
        expr: (rate(node_disk_read_time_seconds_total{device="vda"}[5m]) / clamp_min(rate(node_disk_reads_completed_total{device="vda"}[5m]), 1)) * 1000 > 12
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Elevated NVMe Read Latency (>12ms) on /dev/vda"

      - alert: HostMemoryAvailableBelow10Percent
        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 < 10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Host Available RAM <10% on {{ $labels.instance }}"

# 1. Auf Ihrem Netcup-Server: Monitoring-Stack im Hintergrund starten
cd /opt/stacks/monitoring
sudo docker compose up -d
sudo docker compose ps

# 2. Auf Ihrer lokalen Workstation: SSH-Tunnel auf Port 3000 öffnen
ssh -N -L 3000:127.0.0.1:3000 adminuser@203.0.113.10
# Anschließend http://localhost:3000 im lokalen Browser öffnen