"""Durable alert decisions with a separate, serialized VyOS reconciliation worker.""" from __future__ import annotations import argparse import fcntl import glob import hashlib import json import logging import os import signal import socket import threading import tempfile import time from .alertsock import AlertSocket from .config import load_config from .eve import EveTailer from .metrics import Metrics from .policy import Policy from .state import State from .vyos_client import VyosApiError, VyosClient log = logging.getLogger('leaf') class Audit: def __init__(self, path, max_bytes=0, keep=3): os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True) self.path, self.max_bytes, self.keep = path, int(max_bytes), int(keep) self.fh = open(path, 'a') self.lock = threading.Lock() def write(self, obj): with self.lock: self.fh.write(json.dumps(dict(ts=time.time(), **obj)) + '\n') self.fh.flush() if self.max_bytes and self.fh.tell() >= self.max_bytes: self._rotate_locked() def _rotate_locked(self): """Rotate at the configured size and retain the newest archives.""" self.fh.close() stamp = str(time.time_ns()).zfill(20) dst = self.path + '.' + stamp n = 2 while os.path.exists(dst): # tolerate a frozen/coarse clock dst = self.path + '.' + stamp + f'.{n:08d}' n += 1 os.replace(self.path, dst) self.fh = open(self.path, 'a') rotated = sorted(glob.glob(self.path + '.*'), key=lambda f: (os.stat(f).st_mtime_ns, f)) doomed = rotated[:-self.keep] if self.keep else rotated for old in doomed: try: os.remove(old) except FileNotFoundError: pass log.info('rotated audit log, keeping %d rotated file(s)', self.keep or 0) def close(self): self.fh.close() class Controller: def __init__(self, cfg, dry_run=False): self.cfg, self.dry_run = cfg, dry_run self._closed = False self.group = cfg.get('block_group', 'IDS-BLOCK') self._scratch = tempfile.TemporaryDirectory(prefix='leaf-dry-run-') if dry_run else None state_path = os.path.join(self._scratch.name, 'state.sqlite') if self._scratch else cfg.get('state_db') os.makedirs(os.path.dirname(os.path.abspath(state_path)), exist_ok=True) self._lock = open(state_path + '.lock', 'a') try: fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: self._lock.close() raise SystemExit('another controller owns this state database') self.state = State(state_path) self.policy = Policy(cfg, self.state) self.audit = Audit(os.path.join(self._scratch.name, "audit.jsonl") if dry_run else cfg.get("audit_log"), int(cfg.get("audit.max_bytes", 16777216)), int(cfg.get("audit.keep", 3))) # metrics: counting only; stats records never touch state/audit/API. # health line once per full stats window, capture.log every # metrics.capture_interval seconds (same bounded scheme as audit). self.metrics = Metrics(float(cfg.get("metrics.reconcile_interval", 30.0))) self._metrics_log = logging.getLogger("leaf.metrics") self._metrics_log.setLevel(getattr( logging, str(cfg.get("metrics.log_level", "INFO")).upper(), logging.INFO)) self._capture_path = (cfg.get("metrics.capture_log") or os.path.join(os.path.dirname( os.path.abspath(cfg.get("audit_log"))), "capture.log")) if dry_run: self._capture_path = os.path.join(self._scratch.name, "capture.jsonl") self._capture_interval = max(0.0, float(cfg.get("metrics.capture_interval", 3600))) self._capture_log = None self._next_capture = 0.0 self.client = None if dry_run else VyosClient( cfg.vyos_url(), cfg.vyos_key(), ca_bundle=cfg.get('vyos.ca_bundle'), tls_verify=cfg.get('vyos.tls_verify', True), cert_pin=cfg.get('vyos.tls_cert_pin') or None, timeout=float(cfg.get('vyos.timeout', 75)), retries=int(cfg.get('vyos.retries', 2))) self.stop = threading.Event() self.wake = threading.Event() self._last_sync = 0.0 self._initial_sync = True self._worker = None # socket ingestion: suricata unix_stream output connects to our # listener; empty path disables (offline/mock runs keep file tailing) sock_path = cfg.get('alert_socket.path') self._sock = None if sock_path and not dry_run: self._sock = AlertSocket( sock_path, int(cfg.get('alert_socket.queue_size') or 1000)) def router_members(self): return set(self.client.op_with_retry(self.client.group_members, self.group)) def startup(self): if self.dry_run: return info = self.client.info() log.info('connected to VyOS API: version=%s', info.get('version', '?')) if self.cfg.get('ensure_primitives', True): # Both metadata actions use one group; validate the corresponding hooks. self.client.verify_enforcement(self.group, self.cfg.get('policy.block_src', False)) self.sync_router_state() def sync_router_state(self): """One batched commit, then read back before acknowledging any change.""" if self.dry_run: return now = time.time() members = self.router_members() rows = {r.ip: r for r in self.state.all_blocks()} # Only adopt on startup, never resurrect known expired entries. foreign = members - rows.keys() if self._initial_sync and self.cfg.get('adopt_foreign_members', True): for ip in sorted(foreign): if self.cfg.is_allowed_ip(ip): log.warning('leaving protected foreign router member unowned: %s', ip) continue if self.state.count_active() >= int(self.cfg.get('policy.max_blocks', 50)): log.warning('foreign member adoption capped; leaving unowned: %s', ip) continue self.state.upsert_hit(ip, 'dest', 0, float(self.cfg.get('policy.ttl_seconds', 3600)), now) self.state.set_applied(ip) self.audit.write({'action': 'adopt', 'ip': ip}) log.info('adopted foreign member %s into state', ip) rows = {r.ip: r for r in self.state.all_blocks()} self._initial_sync = False active = {ip for ip, r in rows.items() if r.expires > now and not self.cfg.is_allowed_ip(ip)} expired = rows.keys() - active adds, deletes = active - members, set(expired) & members if not self.cfg.get('adopt_foreign_members', True): deletes |= foreign ops = [self.client.member_op('set', self.group, ip) for ip in sorted(adds)] ops += [self.client.member_op('delete', self.group, ip) for ip in sorted(deletes)] if ops: # A timeout may mean the commit succeeded: next pass reads before retrying. try: self.state.set_router_dirty(True) self.client.configure(ops) except Exception: self.audit.write({'action': 'reconcile', 'ok': False, 'adds': sorted(adds), 'deletes': sorted(deletes)}) raise members = self.router_members() if not adds <= members or deletes & members: raise VyosApiError('router readback does not match requested membership changes') if self.cfg.get('save_config') and self.state.router_dirty(): self.client.save_config() self.state.set_router_dirty(False) for ip in active & members: row = rows[ip] if not row.applied or ip in adds: self.state.set_applied(ip) self.audit.write({'action': 'block', 'ip': ip, 'kind': row.kind, 'sigs': row.sigs, 'expires': row.expires, 'ok': True, 'verified': True}) log.info('BLOCKED %s (%s), router membership verified', ip, row.kind) for ip in expired - members: row = rows[ip] # Do not delete a fresh decision made during the slow router request. removed = self.state.remove(ip, expired_before=now) if removed: self.audit.write({'action': 'unblock', 'ip': ip, 'kind': row.kind, 'reason': 'ttl-expired', 'ok': True, 'verified': True}) log.info('UNBLOCKED %s, router absence verified', ip) elif self.cfg.is_allowed_ip(ip): # A newly protected address must not remain locally desired. self.state.remove(ip) else: self.state.set_applied(ip, 0) self.wake.set() self._last_sync = time.time() def _route(self, event): """Dispatch by event_type; only alerts can act. Stats records are counting-only: they must never reach policy/state/audit/API.""" etype = event.get('event_type') if etype == 'alert': self.metrics.note_alert() self.handle_event(event) elif etype == 'stats': try: self.metrics.note_stats(event) except (TypeError, ValueError, AttributeError, OverflowError, RecursionError): log.warning('ignoring malformed stats record') self.metrics.note_other() else: self.metrics.note_other() def _sock_desc(self): if self._sock is None: return 'off(eve-file)' c = self._sock.counts return ('rec:%d conn:%d qdrop:%d perr:%d other:%d' % (c['records'], c['connections'], c['queue_drops'], c['parse_errors'], c['other'])) def _emit_metrics(self, now): """One health line per full stats window; capture.log line on cadence. Read-only against controller state (count_active is a query).""" rep = self.metrics.report(now) if rep is None: return if rep['init']: self._metrics_log.info('metrics baseline: %s', json.dumps(rep['stats'], separators=(',', ':'))) return flag = ' (counters reset: suricata restart)' if rep['restart'] else '' self._metrics_log.info( 'health: alerts=%d cps=%.1f eps=%.1f dropped+=%d errors+=%d ' 'invalid+=%d suppressed+=%d overflow+=%d blocks=%d uptime=%ds sock=%s%s', rep['alerts'], rep['cps'], rep['eps'], rep['dropped'], rep['delta']['capture_errors'], rep['delta']['invalid'], rep['delta']['suppressed'], rep['delta']['queue_overflow'], self.state.count_active(), rep['stats']['uptime'], self._sock_desc(), flag) if now >= self._next_capture: self._next_capture = now + self._capture_interval if self._capture_log is None: self._capture_log = Audit( self._capture_path, int(self.cfg.get('metrics.capture_max_bytes', 10485760)), int(self.cfg.get('metrics.capture_log_keep', 6))) self._capture_log.write({'window': rep['window'], 'alerts': rep['alerts'], 'cps': rep['cps'], 'eps': rep['eps'], 'dropped': rep['dropped'], 'delta': rep['delta'], 'stats': rep['stats']}) def handle_event(self, event): try: dec = self.policy.evaluate(event) except (TypeError, ValueError, AttributeError, OverflowError, RecursionError): log.warning('ignoring malformed alert record') return if not dec: return existing = self.state.get(dec.ip) if (not existing or existing.expires <= time.time()) and self.state.count_active() >= int(self.cfg.get('policy.max_blocks', 50)): self.audit.write({'action': 'guard', 'ip': dec.ip, 'sid': dec.sid, 'reason': 'max_blocks'}) return row = self.state.upsert_hit(dec.ip, dec.kind, dec.sid, float(self.cfg.get('policy.ttl_seconds', 3600))) if not row.applied: self.audit.write({'action': 'decision', 'ip': dec.ip, 'kind': dec.kind, 'sid': dec.sid, 'expires': row.expires, 'dry_run': self.dry_run}) if self.dry_run: if not row.applied: self.audit.write({'action': 'block', 'ip': dec.ip, 'kind': dec.kind, 'dry_run': True}) self.state.set_applied(dec.ip) else: self.wake.set() def housekeeping(self): now = time.time() self.state.prune_hits(float(self.cfg.get('policy.window_seconds', 300)) * 2, now) if self.dry_run: for row in self.state.expired(now): self.state.remove(row.ip, expired_before=now) return if (self.cfg.get('save_config') and self.state.router_dirty()) or self.state.unapplied() or self.state.expired(now) or now - self._last_sync >= float(self.cfg.get('sync_interval', 300)): self.sync_router_state() def _router_loop(self): ready = False try: while not self.stop.is_set(): try: if not ready: self.startup() ready = True else: self.housekeeping() except Exception as exc: log.error('router reconciliation failed; decisions retained for retry: %s', exc) self.wake.wait(float(self.cfg.get('router_interval', 2))) self.wake.clear() # A short batching window combines bursts into one commit. self.stop.wait(float(self.cfg.get('batch_delay', 0.2))) finally: self.state.close() def _save_checkpoint(self, tailer): if self.dry_run or tailer._ino is None: return self.state.save_checkpoint(tailer._ino[0], tailer._ino[1], tailer._pos) def run(self): if self._sock: # bind BEFORE suricata connects: the sensor unit's ordering # guarantees it (Wants/After leaf-controller). a bind failure is # fatal, not soft: a dead listener would mean silent alert loss. try: self._sock.bind() except OSError as exc: raise SystemExit('cannot bind alert socket %s: %s' % (self._sock.path, exc)) self._sock.start() log.info('listening for sensor alerts on %s', self._sock.path) ckpt = self.state.get_checkpoint() if (not self.dry_run and self.cfg.get('eve.checkpoint', True)) else None tailer = EveTailer(self.cfg.get('eve.path'), float(self.cfg.get('poll_interval', 0.2)), from_start=self.dry_run, checkpoint=ckpt, idle_wait=self._sock.available.wait if self._sock else self.stop.wait) # Establish the live position before any slow API startup calls. tailer._open() self._save_checkpoint(tailer) notify = os.environ.get('NOTIFY_SOCKET') if notify: with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as ready: ready.connect('\0' + notify[1:] if notify.startswith('@') else notify) ready.sendall(b'READY=1') if not self.dry_run: self._worker = threading.Thread(target=self._router_loop, name='vyos-worker') self._worker.start() log.info('watching %s (mode=%s)', self.cfg.get('eve.path'), self.cfg.get('policy.mode')) try: for event in tailer.events(): if self.stop.is_set(): break if self._sock: for rec in self._sock.drain(): self._route(rec) if event is not None: # Socket mode has exactly one alert ingress; the file carries stats. if not self._sock or event.get('event_type') != 'alert': identity = hashlib.sha256(json.dumps(event, sort_keys=True).encode()).hexdigest() self.state._local.event_key = '%s:%s:%s:%s' % (*tailer._ino, tailer._pos, identity) try: self._route(event) finally: self.state._local.event_key = None self._emit_metrics(time.time()) # Offset follows the delivered record, never the unread batch. # File mode is at-least-once across a crash before checkpoint. self._save_checkpoint(tailer) elif self.dry_run: self.housekeeping() finally: tailer.close() self.close() def close(self): if self._closed: return self._closed = True if self._sock: self._sock.close() # Stop intake first, then persist every accepted queued alert. for rec in self._sock.drain(limit=self._sock.q.qsize()): self._route(rec) self.stop.set() self.wake.set() if self._worker: self._worker.join() self._worker = None self.state.close() self.audit.close() if self._capture_log: self._capture_log.close() self._lock.close() if self._scratch: self._scratch.cleanup() def main(argv=None): ap = argparse.ArgumentParser() ap.add_argument('--config', default=os.path.join(os.path.dirname(__file__), '..', 'config.yaml')) ap.add_argument('--dry-run', action='store_true') ap.add_argument('-v', '--verbose', action='store_true') args = ap.parse_args(argv) logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, format='%(asctime)s %(levelname)s %(name)s: %(message)s') ctl = Controller(load_config(args.config), args.dry_run) def shutdown(*args): ctl.stop.set() ctl.wake.set() signal.signal(signal.SIGTERM, shutdown) signal.signal(signal.SIGINT, shutdown) ctl.run() return 0 if __name__ == '__main__': raise SystemExit(main())