Syslog Protocol: Centralized Logging Setup and Configuration

# Priority examples:
# kern.emerg    = 0*8 + 0 = 0   (kernel emergency)
# auth.warning  = 4*8 + 4 = 36  (auth warning)
# local0.info   = 16*8 + 6 = 134 (app info)

# /etc/rsyslog.conf

# Load modules
module(load="imuxsock")   # local Unix socket (most apps use this)
module(load="imklog")     # kernel log messages
module(load="imjournal")  # systemd journal (modern systems)

# Log rules: facility.severity  destination
# * = all, none = exclude

# All emergency messages to all users
*.emerg                         :omusrmsg:*

# Auth messages to auth.log
auth,authpriv.*                 /var/log/auth.log

# All except auth to syslog
*.*;auth,authpriv.none          /var/log/syslog

# Kernel messages
kern.*                          /var/log/kern.log

# Cron
cron.*                          /var/log/cron.log

# Debug (exclude mail, auth, cron)
*.debug;\
    mail,authpriv,cron.none     /var/log/debug

# /etc/rsyslog.conf — on the central log server

# Enable TCP input (reliable, recommended)
module(load="imtcp")
input(type="imtcp" port="514")

# Enable UDP input (legacy, less reliable)
module(load="imudp")
input(type="imudp" port="514")

# Store logs by hostname
$template RemoteLogs,"/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log"
*.* ?RemoteLogs

# Or store all remote logs in one file
*.* /var/log/remote/all.log

# Open firewall for syslog
sudo ufw allow 514/tcp
sudo ufw allow 514/udp

# Restart rsyslog
sudo systemctl restart rsyslog

# Verify it's listening
ss -tlnp | grep 514

# /etc/rsyslog.conf — on each client server

# Forward all logs via TCP (@@) to central server
*.* @@logserver.example.com:514

# Forward only errors and above
*.err @@logserver.example.com:514

# Forward specific facility
auth.* @@logserver.example.com:514

# UDP forwarding (single @)
*.* @logserver.example.com:514

# With queue for reliability (don't lose logs if server is down)
*.* action(type="omfwd"
    target="logserver.example.com"
    port="514"
    protocol="tcp"
    action.resumeRetryCount="100"
    queue.type="linkedList"
    queue.size="10000"
    queue.filename="fwdRule1"
    queue.saveOnShutdown="on")

# On the log server — create CA and server cert
mkdir -p /etc/rsyslog.d/certs && cd /etc/rsyslog.d/certs

# Create CA
openssl genrsa -out ca.key 4096
openssl req -new -x509 -days 3650 -key ca.key -out ca.crt \
    -subj "/CN=Syslog CA/O=MyOrg"

# Create server certificate
openssl genrsa -out server.key 4096
openssl req -new -key server.key -out server.csr \
    -subj "/CN=logserver.example.com/O=MyOrg"
openssl x509 -req -days 3650 -in server.csr \
    -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt

# Set permissions
chmod 600 server.key ca.key
chmod 644 server.crt ca.crt

# /etc/rsyslog.conf — log server

# Load TLS module
module(load="imtcp"
    StreamDriver.Name="gtls"
    StreamDriver.Mode="1"
    StreamDriver.AuthMode="anon")

# TLS certificates
global(
    DefaultNetstreamDriver="gtls"
    DefaultNetstreamDriverCAFile="/etc/rsyslog.d/certs/ca.crt"
    DefaultNetstreamDriverCertFile="/etc/rsyslog.d/certs/server.crt"
    DefaultNetstreamDriverKeyFile="/etc/rsyslog.d/certs/server.key"
)

# Listen on TLS port
input(type="imtcp" port="6514")

# /etc/rsyslog.conf — client

# Copy ca.crt from server to client
# scp logserver:/etc/rsyslog.d/certs/ca.crt /etc/rsyslog.d/certs/

global(
    DefaultNetstreamDriver="gtls"
    DefaultNetstreamDriverCAFile="/etc/rsyslog.d/certs/ca.crt"
)

# Forward with TLS
*.* action(type="omfwd"
    target="logserver.example.com"
    port="6514"
    protocol="tcp"
    StreamDriver="gtls"
    StreamDriverMode="1"
    StreamDriverAuthMode="anon")

# /etc/systemd/journald.conf
[Journal]
ForwardToSyslog=yes
MaxLevelSyslog=debug   # forward all levels

# Restart journald
sudo systemctl restart systemd-journald

# Verify forwarding
journalctl -f &
logger "test message"  # should appear in both journal and syslog

# /etc/rsyslog.d/json-app.conf

# Template to parse JSON from app logs
template(name="json-template" type="list") {
    property(name="timereported" dateFormat="rfc3339")
    constant(value=" ")
    property(name="hostname")
    constant(value=" ")
    property(name="$!msg")
    constant(value="\n")
}

# Parse JSON messages from myapp
if $programname == 'myapp' then {
    action(type="mmjsonparse")
    action(type="omfile"
        file="/var/log/myapp/structured.log"
        template="json-template")
    stop
}

import logging
import logging.handlers

# Send to local syslog
handler = logging.handlers.SysLogHandler(address='/dev/log')
handler.setFormatter(logging.Formatter('%(name)s: %(message)s'))

logger = logging.getLogger('myapp')
logger.addHandler(handler)
logger.setLevel(logging.INFO)

logger.info('Application started')
logger.error('Database connection failed', extra={'db_host': 'db01'})

import "log/syslog"

w, err := syslog.New(syslog.LOG_INFO|syslog.LOG_LOCAL0, "myapp")
if err != nil {
    panic(err)
}
defer w.Close()

w.Info("Application started")
w.Err("Database connection failed")

# logger command sends to syslog
logger "Backup completed successfully"
logger -p local0.err "Backup failed: disk full"
logger -t myapp -p daemon.warning "Config file missing"

# With structured data
logger -p local0.info -t myapp "user_login user=alice ip=192.168.1.1"

# /etc/logrotate.d/rsyslog
/var/log/syslog
/var/log/remote/*/*.log
{
    rotate 7
    daily
    missingok
    notifempty
    delaycompress
    compress
    postrotate
        /usr/lib/rsyslog/rsyslog-rotate
    endscript
}

# /etc/rsyslog.d/elasticsearch.conf
module(load="omelasticsearch")

template(name="es-template" type="list") {
    constant(value="{")
    constant(value="\"@timestamp\":\"")
    property(name="timereported" dateFormat="rfc3339")
    constant(value="\",\"host\":\"")
    property(name="hostname")
    constant(value="\",\"message\":\"")
    property(name="msg" format="json")
    constant(value="\",\"severity\":\"")
    property(name="syslogseverity-text")
    constant(value="\"}")
}

*.* action(type="omelasticsearch"
    server="elasticsearch.example.com"
    serverport="9200"
    template="es-template"
    searchIndex="syslog-%timereported:1:10:date-rfc3339%"
    bulkmode="on"
    queue.type="linkedList"
    queue.size="5000")

# Use promtail to forward syslog to Loki
# /etc/promtail/config.yml
scrape_configs:
  - job_name: syslog
    syslog:
      listen_address: 0.0.0.0:1514
      labels:
        job: syslog
    relabel_configs:
      - source_labels: [__syslog_message_hostname]
        target_label: host
      - source_labels: [__syslog_message_app_name]
        target_label: app

# Forward rsyslog to promtail
# /etc/rsyslog.d/loki.conf
*.* action(type="omfwd" target="127.0.0.1" port="1514" protocol="tcp"
    template="RSYSLOG_SyslogProtocol23Format")

# Test rsyslog config syntax
rsyslogd -N1

# Send a test message
logger -p local0.info "test message from $(hostname)"

# Watch syslog in real-time
tail -f /var/log/syslog

# Check rsyslog is running
systemctl status rsyslog

# Check what rsyslog is listening on
ss -tlnp | grep rsyslog

# Debug rsyslog (verbose output)
rsyslogd -dn 2>&1 | head -50