Logs zentral sammeln: rsyslog, Loki und OpenSearch

[Journal]
Storage=persistent
Compress=yes
SystemMaxUse=2G
SystemMaxFileSize=128M
MaxRetentionSec=2week
RateLimitIntervalSec=30s
RateLimitBurst=20000
ForwardToSyslog=yes

sudo mkdir -p /var/log/journal
sudo systemd-tmpfiles --create --prefix /var/log/journal
sudo systemctl restart systemd-journald

journalctl --disk-usage
journalctl --verify
journalctl -b -p err                       # Fehler seit dem letzten Boot

module(load="omrelp")

action(type="omrelp"
       target="log01.intern.example"
       port="2514"
       tls="on"
       tls.caCert="/etc/pki/tls/certs/logging-ca.pem"
       tls.myCert="/etc/pki/tls/certs/client.pem"
       tls.myPrivKey="/etc/pki/tls/private/client.key"
       tls.authMode="name"
       tls.permittedPeer=["log01.intern.example"]
       action.resumeRetryCount="-1"
       queue.type="LinkedList"
       queue.filename="fwd_log01"
       queue.maxDiskSpace="1g"
       queue.saveOnShutdown="on")

# RHEL / Rocky / Alma
sudo dnf install -y rsyslog-relp rsyslog-gnutls
# Zertifikate unter /etc/pki/tls/{certs,private}

# Debian / Ubuntu
sudo apt install -y rsyslog-relp rsyslog-gnutls
# Zertifikate unter /etc/ssl/{certs,private}

sudo rsyslogd -N1                          # Syntaxprüfung, immer zuerst
sudo systemctl restart rsyslog
logger -p local3.info "logtest $(hostname) $(date +%s)"
ss -tnp | grep 2514

module(load="imrelp" ruleset="remote")

input(type="imrelp" port="2514" tls="on"
      tls.caCert="/etc/pki/tls/certs/logging-ca.pem"
      tls.myCert="/etc/pki/tls/certs/log01.pem"
      tls.myPrivKey="/etc/pki/tls/private/log01.key"
      tls.authMode="name"
      tls.permittedPeer=["*.intern.example"])

template(name="proHost" type="string"
         string="/var/log/remote/%HOSTNAME%/%$YEAR%-%$MONTH%-%$DAY%.log")

ruleset(name="remote") {
    action(type="omfile" dynaFile="proHost"
           dirCreateMode="0750" fileCreateMode="0640")
}

# RHEL / Rocky / Alma
sudo firewall-cmd --permanent --zone=internal --add-source=10.10.0.0/16
sudo firewall-cmd --permanent --zone=internal --add-port=2514/tcp
sudo firewall-cmd --reload

# Debian / Ubuntu mit nftables
sudo nft add rule inet filter input ip saddr 10.10.0.0/16 tcp dport 2514 accept

# Alte Tagesdateien wegräumen, Aufbewahrungsfrist einsetzen
# (mit chattr +a versiegelte Dateien vorher per chattr -a freigeben)
sudo find /var/log/remote -type f -name '*.log' -mtime +365 -print -delete

sudo dnf install -y audispd-plugins          # Debian/Ubuntu: sudo apt install audispd-plugins
sudoedit /etc/audit/plugins.d/au-remote.conf # active = yes
sudoedit /etc/audit/audisp-remote.conf
#   remote_server = log01.intern.example
#   port = 60
#   transport = tcp
#   network_failure_action = syslog

sudo service auditd restart
sudo auditctl -s
sudo ausearch -m USER_LOGIN -ts today | tail

sudo timedatectl set-timezone UTC
chronyc tracking | grep -E 'Reference ID|System time|Stratum'

# Abgeschlossene Tagesdatei versiegeln
sudo chattr +a /var/log/remote/web01/2026-07-24.log
lsattr /var/log/remote/web01/2026-07-24.log

# Journal zusätzlich lokal signieren (erkennt Manipulation, verhindert sie nicht)
sudo journalctl --setup-keys
journalctl --verify

loki.source.journal "system" {
  max_age    = "12h"
  labels     = { job = "systemd-journal", host = constants.hostname }
  forward_to = [loki.write.zentral.receiver]
}

loki.write "zentral" {
  endpoint {
    url = "https://loki.intern.example/loki/api/v1/push"
  }
}