Logging & Metrics Stack¶

curl -O https://raw.githubusercontent.com/MDHMatt/mdhmedia/main/docker-pve1/monitoring/grafana/host-agents/install-alloy.sh
chmod +x install-alloy.sh
sudo ./install-alloy.sh                 # ships to 192.168.1.241 by default

# Or pass the collector IP explicitly
sudo ./install-alloy.sh 192.168.1.241

curl -O https://raw.githubusercontent.com/MDHMatt/mdhmedia/main/docker-pve1/monitoring/grafana/host-agents/install-alloy-caddy.sh
chmod +x install-alloy-caddy.sh
sudo ./install-alloy-caddy.sh 192.168.1.241

# Add Grafana repository (Debian/Ubuntu - DEB822 format)
sudo mkdir -p /etc/apt/keyrings
sudo wget -q -O /etc/apt/keyrings/grafana.asc https://apt.grafana.com/gpg-full.key
sudo chmod 644 /etc/apt/keyrings/grafana.asc

cat << 'EOF' | sudo tee /etc/apt/sources.list.d/grafana.sources
Types: deb
URIs: https://apt.grafana.com
Suites: stable
Components: main
Signed-By: /etc/apt/keyrings/grafana.asc
EOF

# Install
sudo apt-get update
sudo apt-get install alloy

# Configure and start
sudo systemctl enable --now alloy

local.file_match "myapp" {
  path_targets = [
    {
      __path__ = "/var/log/myapp/*.log",
      job      = "myapp",
    },
  ]
}

loki.source.file "myapp" {
  targets    = local.file_match.myapp.targets
  forward_to = [loki.write.default.receiver]
  tail_from_end = true
}

sudo systemctl reload alloy

prometheus.exporter.unix "node" {
  set_collectors = ["cpu", "meminfo", "diskstats", "filesystem", "loadavg", "netdev", "uname"]
  filesystem {
    fs_types_exclude = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
  }
}

prometheus.scrape "node" {
  targets    = prometheus.exporter.unix.node.targets
  forward_to = [prometheus.remote_write.default.receiver]
  scrape_interval = "15s"
  job_name   = "node"
}

local.file_match "docker" {
  path_targets = [
    {
      __path__         = "/var/lib/docker/containers/*/*.log",
      job              = "docker",
      __path_exclude__ = "/var/lib/docker/containers/*/*-json.log.*.gz",
    },
  ]
}

loki.source.file "docker" {
  targets    = local.file_match.docker.targets
  forward_to = [loki.write.default.receiver]
  tail_from_end = true
}

local.file_match "nginx" {
  path_targets = [
    {__path__ = "/var/log/nginx/access.log", job = "nginx", type = "access"},
    {__path__ = "/var/log/nginx/error.log", job = "nginx", type = "error"},
  ]
}

loki.source.file "nginx" {
  targets    = local.file_match.nginx.targets
  forward_to = [loki.write.default.receiver]
  tail_from_end = true
}

# All logs from a specific host
{host="docker1"}

# Systemd journal logs for a specific unit
{job="systemd-journal", unit="nginx.service"}

# Search for errors across all hosts
{env="homelab"} |= "error"

# Auth failures
{job="auth"} |= "Failed password"

# Top error-producing hosts (last hour)
sum by (host) (count_over_time({env="homelab"} |= "error" [1h]))

# SSH login attempts
{job="auth"} |~ "sshd.*Accepted|sshd.*Failed"

# Systemd service failures
{job="systemd-journal"} |= "Failed to start"

# Caddy requests by service
{job="caddy", request_host="grafana.mdhmedia.uk"}

# Caddy 5xx errors
{job="caddy", status=~"5.."}

# CPU usage by host (percentage)
100 - (avg by(host) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# Memory usage by host (percentage)
100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)

# Disk usage by host and mountpoint (percentage)
100 - (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100)

# Network traffic (bytes/sec)
rate(node_network_receive_bytes_total[5m])
rate(node_network_transmit_bytes_total[5m])

# System load average (1 minute)
node_load1

# Top 5 hosts by CPU usage
topk(5, 100 - (avg by(host) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100))

# Hosts with disk usage > 80%
node_filesystem_avail_bytes / node_filesystem_size_bytes * 100 < 20

# All metrics from a specific host
{host="caddy"}

sudo systemctl status alloy
sudo journalctl -u alloy -f
alloy fmt /etc/alloy/config.alloy  # Check syntax and format

loki.write "default" {
  endpoint {
    url = "http://192.168.1.241:3100/loki/api/v1/push"
    batch_size = "512KiB"
    batch_wait = "2s"
  }
}

sudo systemctl status loki
sudo journalctl -u loki -f
loki -config.file=/etc/loki/loki.yaml -verify-config

limits_config:
  retention_period: 720h  # 30 days

compactor:
  retention_enabled: true
  retention_delete_delay: 2h

ingester:
  chunk_idle_period: 30m
  max_chunk_age: 1h
  chunk_target_size: 1048576

sudo systemctl status prometheus
sudo journalctl -u prometheus -f
promtool check config /etc/prometheus/prometheus.yml

--storage.tsdb.retention.time=15d

sudo systemctl daemon-reload
sudo systemctl restart prometheus