Files
LEAF/controller/leaf_controller/main.py
2026-11-15 17:41:40 +01:00

404 lines
18 KiB
Python

"""Durable alert decisions with a separate, serialized VyOS reconciliation worker."""
from __future__ import annotations
import argparse
import fcntl
import glob
import hashlib
import json
import logging
import os
import signal
import socket
import threading
import tempfile
import time
from .alertsock import AlertSocket
from .config import load_config
from .eve import EveTailer
from .metrics import Metrics
from .policy import Policy
from .state import State
from .vyos_client import VyosApiError, VyosClient
log = logging.getLogger('leaf')
class Audit:
def __init__(self, path, max_bytes=0, keep=3):
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
self.path, self.max_bytes, self.keep = path, int(max_bytes), int(keep)
self.fh = open(path, 'a')
self.lock = threading.Lock()
def write(self, obj):
with self.lock:
self.fh.write(json.dumps(dict(ts=time.time(), **obj)) + '\n')
self.fh.flush()
if self.max_bytes and self.fh.tell() >= self.max_bytes:
self._rotate_locked()
def _rotate_locked(self):
"""Rotate at the configured size and retain the newest archives."""
self.fh.close()
stamp = str(time.time_ns()).zfill(20)
dst = self.path + '.' + stamp
n = 2
while os.path.exists(dst): # tolerate a frozen/coarse clock
dst = self.path + '.' + stamp + f'.{n:08d}'
n += 1
os.replace(self.path, dst)
self.fh = open(self.path, 'a')
rotated = sorted(glob.glob(self.path + '.*'), key=lambda f: (os.stat(f).st_mtime_ns, f))
doomed = rotated[:-self.keep] if self.keep else rotated
for old in doomed:
try:
os.remove(old)
except FileNotFoundError:
pass
log.info('rotated audit log, keeping %d rotated file(s)', self.keep or 0)
def close(self):
self.fh.close()
class Controller:
def __init__(self, cfg, dry_run=False):
self.cfg, self.dry_run = cfg, dry_run
self._closed = False
self.group = cfg.get('block_group', 'IDS-BLOCK')
self._scratch = tempfile.TemporaryDirectory(prefix='leaf-dry-run-') if dry_run else None
state_path = os.path.join(self._scratch.name, 'state.sqlite') if self._scratch else cfg.get('state_db')
os.makedirs(os.path.dirname(os.path.abspath(state_path)), exist_ok=True)
self._lock = open(state_path + '.lock', 'a')
try:
fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
self._lock.close()
raise SystemExit('another controller owns this state database')
self.state = State(state_path)
self.policy = Policy(cfg, self.state)
self.audit = Audit(os.path.join(self._scratch.name, "audit.jsonl") if dry_run else cfg.get("audit_log"),
int(cfg.get("audit.max_bytes", 16777216)),
int(cfg.get("audit.keep", 3)))
# metrics: counting only; stats records never touch state/audit/API.
# health line once per full stats window, capture.log every
# metrics.capture_interval seconds (same bounded scheme as audit).
self.metrics = Metrics(float(cfg.get("metrics.reconcile_interval", 30.0)))
self._metrics_log = logging.getLogger("leaf.metrics")
self._metrics_log.setLevel(getattr(
logging, str(cfg.get("metrics.log_level", "INFO")).upper(), logging.INFO))
self._capture_path = (cfg.get("metrics.capture_log")
or os.path.join(os.path.dirname(
os.path.abspath(cfg.get("audit_log"))), "capture.log"))
if dry_run:
self._capture_path = os.path.join(self._scratch.name, "capture.jsonl")
self._capture_interval = max(0.0, float(cfg.get("metrics.capture_interval", 3600)))
self._capture_log = None
self._next_capture = 0.0
self.client = None if dry_run else VyosClient(
cfg.vyos_url(), cfg.vyos_key(), ca_bundle=cfg.get('vyos.ca_bundle'),
tls_verify=cfg.get('vyos.tls_verify', True),
cert_pin=cfg.get('vyos.tls_cert_pin') or None,
timeout=float(cfg.get('vyos.timeout', 75)),
retries=int(cfg.get('vyos.retries', 2)))
self.stop = threading.Event()
self.wake = threading.Event()
self._last_sync = 0.0
self._initial_sync = True
self._worker = None
# socket ingestion: suricata unix_stream output connects to our
# listener; empty path disables (offline/mock runs keep file tailing)
sock_path = cfg.get('alert_socket.path')
self._sock = None
if sock_path and not dry_run:
self._sock = AlertSocket(
sock_path, int(cfg.get('alert_socket.queue_size') or 1000))
def router_members(self):
return set(self.client.op_with_retry(self.client.group_members, self.group))
def startup(self):
if self.dry_run:
return
info = self.client.info()
log.info('connected to VyOS API: version=%s', info.get('version', '?'))
if self.cfg.get('ensure_primitives', True):
# Both metadata actions use one group; validate the corresponding hooks.
self.client.verify_enforcement(self.group, self.cfg.get('policy.block_src', False))
self.sync_router_state()
def sync_router_state(self):
"""One batched commit, then read back before acknowledging any change."""
if self.dry_run:
return
now = time.time()
members = self.router_members()
rows = {r.ip: r for r in self.state.all_blocks()}
# Only adopt on startup, never resurrect known expired entries.
foreign = members - rows.keys()
if self._initial_sync and self.cfg.get('adopt_foreign_members', True):
for ip in sorted(foreign):
if self.cfg.is_allowed_ip(ip):
log.warning('leaving protected foreign router member unowned: %s', ip)
continue
if self.state.count_active() >= int(self.cfg.get('policy.max_blocks', 50)):
log.warning('foreign member adoption capped; leaving unowned: %s', ip)
continue
self.state.upsert_hit(ip, 'dest', 0, float(self.cfg.get('policy.ttl_seconds', 3600)), now)
self.state.set_applied(ip)
self.audit.write({'action': 'adopt', 'ip': ip})
log.info('adopted foreign member %s into state', ip)
rows = {r.ip: r for r in self.state.all_blocks()}
self._initial_sync = False
active = {ip for ip, r in rows.items() if r.expires > now and not self.cfg.is_allowed_ip(ip)}
expired = rows.keys() - active
adds, deletes = active - members, set(expired) & members
if not self.cfg.get('adopt_foreign_members', True):
deletes |= foreign
ops = [self.client.member_op('set', self.group, ip) for ip in sorted(adds)]
ops += [self.client.member_op('delete', self.group, ip) for ip in sorted(deletes)]
if ops:
# A timeout may mean the commit succeeded: next pass reads before retrying.
try:
self.state.set_router_dirty(True)
self.client.configure(ops)
except Exception:
self.audit.write({'action': 'reconcile', 'ok': False,
'adds': sorted(adds), 'deletes': sorted(deletes)})
raise
members = self.router_members()
if not adds <= members or deletes & members:
raise VyosApiError('router readback does not match requested membership changes')
if self.cfg.get('save_config') and self.state.router_dirty():
self.client.save_config()
self.state.set_router_dirty(False)
for ip in active & members:
row = rows[ip]
if not row.applied or ip in adds:
self.state.set_applied(ip)
self.audit.write({'action': 'block', 'ip': ip, 'kind': row.kind,
'sigs': row.sigs, 'expires': row.expires,
'ok': True, 'verified': True})
log.info('BLOCKED %s (%s), router membership verified', ip, row.kind)
for ip in expired - members:
row = rows[ip]
# Do not delete a fresh decision made during the slow router request.
removed = self.state.remove(ip, expired_before=now)
if removed:
self.audit.write({'action': 'unblock', 'ip': ip, 'kind': row.kind,
'reason': 'ttl-expired', 'ok': True, 'verified': True})
log.info('UNBLOCKED %s, router absence verified', ip)
elif self.cfg.is_allowed_ip(ip):
# A newly protected address must not remain locally desired.
self.state.remove(ip)
else:
self.state.set_applied(ip, 0)
self.wake.set()
self._last_sync = time.time()
def _route(self, event):
"""Dispatch by event_type; only alerts can act. Stats records are
counting-only: they must never reach policy/state/audit/API."""
etype = event.get('event_type')
if etype == 'alert':
self.metrics.note_alert()
self.handle_event(event)
elif etype == 'stats':
try:
self.metrics.note_stats(event)
except (TypeError, ValueError, AttributeError, OverflowError, RecursionError):
log.warning('ignoring malformed stats record')
self.metrics.note_other()
else:
self.metrics.note_other()
def _sock_desc(self):
if self._sock is None:
return 'off(eve-file)'
c = self._sock.counts
return ('rec:%d conn:%d qdrop:%d perr:%d other:%d'
% (c['records'], c['connections'], c['queue_drops'],
c['parse_errors'], c['other']))
def _emit_metrics(self, now):
"""One health line per full stats window; capture.log line on cadence.
Read-only against controller state (count_active is a query)."""
rep = self.metrics.report(now)
if rep is None:
return
if rep['init']:
self._metrics_log.info('metrics baseline: %s',
json.dumps(rep['stats'], separators=(',', ':')))
return
flag = ' (counters reset: suricata restart)' if rep['restart'] else ''
self._metrics_log.info(
'health: alerts=%d cps=%.1f eps=%.1f dropped+=%d errors+=%d '
'invalid+=%d suppressed+=%d overflow+=%d blocks=%d uptime=%ds sock=%s%s',
rep['alerts'], rep['cps'], rep['eps'], rep['dropped'],
rep['delta']['capture_errors'], rep['delta']['invalid'],
rep['delta']['suppressed'], rep['delta']['queue_overflow'],
self.state.count_active(), rep['stats']['uptime'],
self._sock_desc(), flag)
if now >= self._next_capture:
self._next_capture = now + self._capture_interval
if self._capture_log is None:
self._capture_log = Audit(
self._capture_path,
int(self.cfg.get('metrics.capture_max_bytes', 10485760)),
int(self.cfg.get('metrics.capture_log_keep', 6)))
self._capture_log.write({'window': rep['window'],
'alerts': rep['alerts'],
'cps': rep['cps'], 'eps': rep['eps'],
'dropped': rep['dropped'],
'delta': rep['delta'],
'stats': rep['stats']})
def handle_event(self, event):
try:
dec = self.policy.evaluate(event)
except (TypeError, ValueError, AttributeError, OverflowError, RecursionError):
log.warning('ignoring malformed alert record')
return
if not dec:
return
existing = self.state.get(dec.ip)
if (not existing or existing.expires <= time.time()) and self.state.count_active() >= int(self.cfg.get('policy.max_blocks', 50)):
self.audit.write({'action': 'guard', 'ip': dec.ip, 'sid': dec.sid, 'reason': 'max_blocks'})
return
row = self.state.upsert_hit(dec.ip, dec.kind, dec.sid,
float(self.cfg.get('policy.ttl_seconds', 3600)))
if not row.applied:
self.audit.write({'action': 'decision', 'ip': dec.ip, 'kind': dec.kind,
'sid': dec.sid, 'expires': row.expires, 'dry_run': self.dry_run})
if self.dry_run:
if not row.applied:
self.audit.write({'action': 'block', 'ip': dec.ip, 'kind': dec.kind, 'dry_run': True})
self.state.set_applied(dec.ip)
else:
self.wake.set()
def housekeeping(self):
now = time.time()
self.state.prune_hits(float(self.cfg.get('policy.window_seconds', 300)) * 2, now)
if self.dry_run:
for row in self.state.expired(now):
self.state.remove(row.ip, expired_before=now)
return
if (self.cfg.get('save_config') and self.state.router_dirty()) or self.state.unapplied() or self.state.expired(now) or now - self._last_sync >= float(self.cfg.get('sync_interval', 300)):
self.sync_router_state()
def _router_loop(self):
ready = False
try:
while not self.stop.is_set():
try:
if not ready:
self.startup()
ready = True
else:
self.housekeeping()
except Exception as exc:
log.error('router reconciliation failed; decisions retained for retry: %s', exc)
self.wake.wait(float(self.cfg.get('router_interval', 2)))
self.wake.clear()
# A short batching window combines bursts into one commit.
self.stop.wait(float(self.cfg.get('batch_delay', 0.2)))
finally:
self.state.close()
def _save_checkpoint(self, tailer):
if self.dry_run or tailer._ino is None:
return
self.state.save_checkpoint(tailer._ino[0], tailer._ino[1], tailer._pos)
def run(self):
if self._sock:
# bind BEFORE suricata connects: the sensor unit's ordering
# guarantees it (Wants/After leaf-controller). a bind failure is
# fatal, not soft: a dead listener would mean silent alert loss.
try:
self._sock.bind()
except OSError as exc:
raise SystemExit('cannot bind alert socket %s: %s'
% (self._sock.path, exc))
self._sock.start()
log.info('listening for sensor alerts on %s', self._sock.path)
ckpt = self.state.get_checkpoint() if (not self.dry_run and self.cfg.get('eve.checkpoint', True)) else None
tailer = EveTailer(self.cfg.get('eve.path'), float(self.cfg.get('poll_interval', 0.2)),
from_start=self.dry_run, checkpoint=ckpt,
idle_wait=self._sock.available.wait if self._sock else self.stop.wait)
# Establish the live position before any slow API startup calls.
tailer._open()
self._save_checkpoint(tailer)
notify = os.environ.get('NOTIFY_SOCKET')
if notify:
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as ready:
ready.connect('\0' + notify[1:] if notify.startswith('@') else notify)
ready.sendall(b'READY=1')
if not self.dry_run:
self._worker = threading.Thread(target=self._router_loop, name='vyos-worker')
self._worker.start()
log.info('watching %s (mode=%s)', self.cfg.get('eve.path'), self.cfg.get('policy.mode'))
try:
for event in tailer.events():
if self.stop.is_set():
break
if self._sock:
for rec in self._sock.drain():
self._route(rec)
if event is not None:
# Socket mode has exactly one alert ingress; the file carries stats.
if not self._sock or event.get('event_type') != 'alert':
identity = hashlib.sha256(json.dumps(event, sort_keys=True).encode()).hexdigest()
self.state._local.event_key = '%s:%s:%s:%s' % (*tailer._ino, tailer._pos, identity)
try:
self._route(event)
finally:
self.state._local.event_key = None
self._emit_metrics(time.time())
# Offset follows the delivered record, never the unread batch.
# File mode is at-least-once across a crash before checkpoint.
self._save_checkpoint(tailer)
elif self.dry_run:
self.housekeeping()
finally:
tailer.close()
self.close()
def close(self):
if self._closed:
return
self._closed = True
if self._sock:
self._sock.close()
# Stop intake first, then persist every accepted queued alert.
for rec in self._sock.drain(limit=self._sock.q.qsize()):
self._route(rec)
self.stop.set()
self.wake.set()
if self._worker:
self._worker.join()
self._worker = None
self.state.close()
self.audit.close()
if self._capture_log:
self._capture_log.close()
self._lock.close()
if self._scratch:
self._scratch.cleanup()
def main(argv=None):
ap = argparse.ArgumentParser()
ap.add_argument('--config', default=os.path.join(os.path.dirname(__file__), '..', 'config.yaml'))
ap.add_argument('--dry-run', action='store_true')
ap.add_argument('-v', '--verbose', action='store_true')
args = ap.parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
format='%(asctime)s %(levelname)s %(name)s: %(message)s')
ctl = Controller(load_config(args.config), args.dry_run)
def shutdown(*args):
ctl.stop.set()
ctl.wake.set()
signal.signal(signal.SIGTERM, shutdown)
signal.signal(signal.SIGINT, shutdown)
ctl.run()
return 0
if __name__ == '__main__':
raise SystemExit(main())