Code
This commit is contained in:
403
controller/leaf_controller/main.py
Normal file
403
controller/leaf_controller/main.py
Normal file
@@ -0,0 +1,403 @@
|
||||
"""Durable alert decisions with a separate, serialized VyOS reconciliation worker."""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import fcntl
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import threading
|
||||
import tempfile
|
||||
import time
|
||||
from .alertsock import AlertSocket
|
||||
from .config import load_config
|
||||
from .eve import EveTailer
|
||||
from .metrics import Metrics
|
||||
from .policy import Policy
|
||||
from .state import State
|
||||
from .vyos_client import VyosApiError, VyosClient
|
||||
|
||||
log = logging.getLogger('leaf')
|
||||
|
||||
class Audit:
|
||||
def __init__(self, path, max_bytes=0, keep=3):
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
|
||||
self.path, self.max_bytes, self.keep = path, int(max_bytes), int(keep)
|
||||
self.fh = open(path, 'a')
|
||||
self.lock = threading.Lock()
|
||||
def write(self, obj):
|
||||
with self.lock:
|
||||
self.fh.write(json.dumps(dict(ts=time.time(), **obj)) + '\n')
|
||||
self.fh.flush()
|
||||
if self.max_bytes and self.fh.tell() >= self.max_bytes:
|
||||
self._rotate_locked()
|
||||
def _rotate_locked(self):
|
||||
"""Rotate at the configured size and retain the newest archives."""
|
||||
self.fh.close()
|
||||
stamp = str(time.time_ns()).zfill(20)
|
||||
dst = self.path + '.' + stamp
|
||||
n = 2
|
||||
while os.path.exists(dst): # tolerate a frozen/coarse clock
|
||||
dst = self.path + '.' + stamp + f'.{n:08d}'
|
||||
n += 1
|
||||
os.replace(self.path, dst)
|
||||
self.fh = open(self.path, 'a')
|
||||
rotated = sorted(glob.glob(self.path + '.*'), key=lambda f: (os.stat(f).st_mtime_ns, f))
|
||||
doomed = rotated[:-self.keep] if self.keep else rotated
|
||||
for old in doomed:
|
||||
try:
|
||||
os.remove(old)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
log.info('rotated audit log, keeping %d rotated file(s)', self.keep or 0)
|
||||
def close(self):
|
||||
self.fh.close()
|
||||
|
||||
class Controller:
|
||||
def __init__(self, cfg, dry_run=False):
|
||||
self.cfg, self.dry_run = cfg, dry_run
|
||||
self._closed = False
|
||||
self.group = cfg.get('block_group', 'IDS-BLOCK')
|
||||
self._scratch = tempfile.TemporaryDirectory(prefix='leaf-dry-run-') if dry_run else None
|
||||
state_path = os.path.join(self._scratch.name, 'state.sqlite') if self._scratch else cfg.get('state_db')
|
||||
os.makedirs(os.path.dirname(os.path.abspath(state_path)), exist_ok=True)
|
||||
self._lock = open(state_path + '.lock', 'a')
|
||||
try:
|
||||
fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
self._lock.close()
|
||||
raise SystemExit('another controller owns this state database')
|
||||
self.state = State(state_path)
|
||||
self.policy = Policy(cfg, self.state)
|
||||
self.audit = Audit(os.path.join(self._scratch.name, "audit.jsonl") if dry_run else cfg.get("audit_log"),
|
||||
int(cfg.get("audit.max_bytes", 16777216)),
|
||||
int(cfg.get("audit.keep", 3)))
|
||||
# metrics: counting only; stats records never touch state/audit/API.
|
||||
# health line once per full stats window, capture.log every
|
||||
# metrics.capture_interval seconds (same bounded scheme as audit).
|
||||
self.metrics = Metrics(float(cfg.get("metrics.reconcile_interval", 30.0)))
|
||||
self._metrics_log = logging.getLogger("leaf.metrics")
|
||||
self._metrics_log.setLevel(getattr(
|
||||
logging, str(cfg.get("metrics.log_level", "INFO")).upper(), logging.INFO))
|
||||
self._capture_path = (cfg.get("metrics.capture_log")
|
||||
or os.path.join(os.path.dirname(
|
||||
os.path.abspath(cfg.get("audit_log"))), "capture.log"))
|
||||
if dry_run:
|
||||
self._capture_path = os.path.join(self._scratch.name, "capture.jsonl")
|
||||
self._capture_interval = max(0.0, float(cfg.get("metrics.capture_interval", 3600)))
|
||||
self._capture_log = None
|
||||
self._next_capture = 0.0
|
||||
self.client = None if dry_run else VyosClient(
|
||||
cfg.vyos_url(), cfg.vyos_key(), ca_bundle=cfg.get('vyos.ca_bundle'),
|
||||
tls_verify=cfg.get('vyos.tls_verify', True),
|
||||
cert_pin=cfg.get('vyos.tls_cert_pin') or None,
|
||||
timeout=float(cfg.get('vyos.timeout', 75)),
|
||||
retries=int(cfg.get('vyos.retries', 2)))
|
||||
self.stop = threading.Event()
|
||||
self.wake = threading.Event()
|
||||
self._last_sync = 0.0
|
||||
self._initial_sync = True
|
||||
self._worker = None
|
||||
# socket ingestion: suricata unix_stream output connects to our
|
||||
# listener; empty path disables (offline/mock runs keep file tailing)
|
||||
sock_path = cfg.get('alert_socket.path')
|
||||
self._sock = None
|
||||
if sock_path and not dry_run:
|
||||
self._sock = AlertSocket(
|
||||
sock_path, int(cfg.get('alert_socket.queue_size') or 1000))
|
||||
|
||||
def router_members(self):
|
||||
return set(self.client.op_with_retry(self.client.group_members, self.group))
|
||||
|
||||
def startup(self):
|
||||
if self.dry_run:
|
||||
return
|
||||
info = self.client.info()
|
||||
log.info('connected to VyOS API: version=%s', info.get('version', '?'))
|
||||
if self.cfg.get('ensure_primitives', True):
|
||||
# Both metadata actions use one group; validate the corresponding hooks.
|
||||
self.client.verify_enforcement(self.group, self.cfg.get('policy.block_src', False))
|
||||
self.sync_router_state()
|
||||
|
||||
def sync_router_state(self):
|
||||
"""One batched commit, then read back before acknowledging any change."""
|
||||
if self.dry_run:
|
||||
return
|
||||
now = time.time()
|
||||
members = self.router_members()
|
||||
rows = {r.ip: r for r in self.state.all_blocks()}
|
||||
# Only adopt on startup, never resurrect known expired entries.
|
||||
foreign = members - rows.keys()
|
||||
if self._initial_sync and self.cfg.get('adopt_foreign_members', True):
|
||||
for ip in sorted(foreign):
|
||||
if self.cfg.is_allowed_ip(ip):
|
||||
log.warning('leaving protected foreign router member unowned: %s', ip)
|
||||
continue
|
||||
if self.state.count_active() >= int(self.cfg.get('policy.max_blocks', 50)):
|
||||
log.warning('foreign member adoption capped; leaving unowned: %s', ip)
|
||||
continue
|
||||
self.state.upsert_hit(ip, 'dest', 0, float(self.cfg.get('policy.ttl_seconds', 3600)), now)
|
||||
self.state.set_applied(ip)
|
||||
self.audit.write({'action': 'adopt', 'ip': ip})
|
||||
log.info('adopted foreign member %s into state', ip)
|
||||
rows = {r.ip: r for r in self.state.all_blocks()}
|
||||
self._initial_sync = False
|
||||
active = {ip for ip, r in rows.items() if r.expires > now and not self.cfg.is_allowed_ip(ip)}
|
||||
expired = rows.keys() - active
|
||||
adds, deletes = active - members, set(expired) & members
|
||||
if not self.cfg.get('adopt_foreign_members', True):
|
||||
deletes |= foreign
|
||||
ops = [self.client.member_op('set', self.group, ip) for ip in sorted(adds)]
|
||||
ops += [self.client.member_op('delete', self.group, ip) for ip in sorted(deletes)]
|
||||
if ops:
|
||||
# A timeout may mean the commit succeeded: next pass reads before retrying.
|
||||
try:
|
||||
self.state.set_router_dirty(True)
|
||||
self.client.configure(ops)
|
||||
except Exception:
|
||||
self.audit.write({'action': 'reconcile', 'ok': False,
|
||||
'adds': sorted(adds), 'deletes': sorted(deletes)})
|
||||
raise
|
||||
members = self.router_members()
|
||||
if not adds <= members or deletes & members:
|
||||
raise VyosApiError('router readback does not match requested membership changes')
|
||||
if self.cfg.get('save_config') and self.state.router_dirty():
|
||||
self.client.save_config()
|
||||
self.state.set_router_dirty(False)
|
||||
for ip in active & members:
|
||||
row = rows[ip]
|
||||
if not row.applied or ip in adds:
|
||||
self.state.set_applied(ip)
|
||||
self.audit.write({'action': 'block', 'ip': ip, 'kind': row.kind,
|
||||
'sigs': row.sigs, 'expires': row.expires,
|
||||
'ok': True, 'verified': True})
|
||||
log.info('BLOCKED %s (%s), router membership verified', ip, row.kind)
|
||||
for ip in expired - members:
|
||||
row = rows[ip]
|
||||
# Do not delete a fresh decision made during the slow router request.
|
||||
removed = self.state.remove(ip, expired_before=now)
|
||||
if removed:
|
||||
self.audit.write({'action': 'unblock', 'ip': ip, 'kind': row.kind,
|
||||
'reason': 'ttl-expired', 'ok': True, 'verified': True})
|
||||
log.info('UNBLOCKED %s, router absence verified', ip)
|
||||
elif self.cfg.is_allowed_ip(ip):
|
||||
# A newly protected address must not remain locally desired.
|
||||
self.state.remove(ip)
|
||||
else:
|
||||
self.state.set_applied(ip, 0)
|
||||
self.wake.set()
|
||||
self._last_sync = time.time()
|
||||
|
||||
def _route(self, event):
|
||||
"""Dispatch by event_type; only alerts can act. Stats records are
|
||||
counting-only: they must never reach policy/state/audit/API."""
|
||||
etype = event.get('event_type')
|
||||
if etype == 'alert':
|
||||
self.metrics.note_alert()
|
||||
self.handle_event(event)
|
||||
elif etype == 'stats':
|
||||
try:
|
||||
self.metrics.note_stats(event)
|
||||
except (TypeError, ValueError, AttributeError, OverflowError, RecursionError):
|
||||
log.warning('ignoring malformed stats record')
|
||||
self.metrics.note_other()
|
||||
else:
|
||||
self.metrics.note_other()
|
||||
|
||||
def _sock_desc(self):
|
||||
if self._sock is None:
|
||||
return 'off(eve-file)'
|
||||
c = self._sock.counts
|
||||
return ('rec:%d conn:%d qdrop:%d perr:%d other:%d'
|
||||
% (c['records'], c['connections'], c['queue_drops'],
|
||||
c['parse_errors'], c['other']))
|
||||
|
||||
def _emit_metrics(self, now):
|
||||
"""One health line per full stats window; capture.log line on cadence.
|
||||
Read-only against controller state (count_active is a query)."""
|
||||
rep = self.metrics.report(now)
|
||||
if rep is None:
|
||||
return
|
||||
if rep['init']:
|
||||
self._metrics_log.info('metrics baseline: %s',
|
||||
json.dumps(rep['stats'], separators=(',', ':')))
|
||||
return
|
||||
flag = ' (counters reset: suricata restart)' if rep['restart'] else ''
|
||||
self._metrics_log.info(
|
||||
'health: alerts=%d cps=%.1f eps=%.1f dropped+=%d errors+=%d '
|
||||
'invalid+=%d suppressed+=%d overflow+=%d blocks=%d uptime=%ds sock=%s%s',
|
||||
rep['alerts'], rep['cps'], rep['eps'], rep['dropped'],
|
||||
rep['delta']['capture_errors'], rep['delta']['invalid'],
|
||||
rep['delta']['suppressed'], rep['delta']['queue_overflow'],
|
||||
self.state.count_active(), rep['stats']['uptime'],
|
||||
self._sock_desc(), flag)
|
||||
if now >= self._next_capture:
|
||||
self._next_capture = now + self._capture_interval
|
||||
if self._capture_log is None:
|
||||
self._capture_log = Audit(
|
||||
self._capture_path,
|
||||
int(self.cfg.get('metrics.capture_max_bytes', 10485760)),
|
||||
int(self.cfg.get('metrics.capture_log_keep', 6)))
|
||||
self._capture_log.write({'window': rep['window'],
|
||||
'alerts': rep['alerts'],
|
||||
'cps': rep['cps'], 'eps': rep['eps'],
|
||||
'dropped': rep['dropped'],
|
||||
'delta': rep['delta'],
|
||||
'stats': rep['stats']})
|
||||
|
||||
def handle_event(self, event):
|
||||
try:
|
||||
dec = self.policy.evaluate(event)
|
||||
except (TypeError, ValueError, AttributeError, OverflowError, RecursionError):
|
||||
log.warning('ignoring malformed alert record')
|
||||
return
|
||||
if not dec:
|
||||
return
|
||||
existing = self.state.get(dec.ip)
|
||||
if (not existing or existing.expires <= time.time()) and self.state.count_active() >= int(self.cfg.get('policy.max_blocks', 50)):
|
||||
self.audit.write({'action': 'guard', 'ip': dec.ip, 'sid': dec.sid, 'reason': 'max_blocks'})
|
||||
return
|
||||
row = self.state.upsert_hit(dec.ip, dec.kind, dec.sid,
|
||||
float(self.cfg.get('policy.ttl_seconds', 3600)))
|
||||
if not row.applied:
|
||||
self.audit.write({'action': 'decision', 'ip': dec.ip, 'kind': dec.kind,
|
||||
'sid': dec.sid, 'expires': row.expires, 'dry_run': self.dry_run})
|
||||
if self.dry_run:
|
||||
if not row.applied:
|
||||
self.audit.write({'action': 'block', 'ip': dec.ip, 'kind': dec.kind, 'dry_run': True})
|
||||
self.state.set_applied(dec.ip)
|
||||
else:
|
||||
self.wake.set()
|
||||
|
||||
def housekeeping(self):
|
||||
now = time.time()
|
||||
self.state.prune_hits(float(self.cfg.get('policy.window_seconds', 300)) * 2, now)
|
||||
if self.dry_run:
|
||||
for row in self.state.expired(now):
|
||||
self.state.remove(row.ip, expired_before=now)
|
||||
return
|
||||
if (self.cfg.get('save_config') and self.state.router_dirty()) or self.state.unapplied() or self.state.expired(now) or now - self._last_sync >= float(self.cfg.get('sync_interval', 300)):
|
||||
self.sync_router_state()
|
||||
|
||||
def _router_loop(self):
|
||||
ready = False
|
||||
try:
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
if not ready:
|
||||
self.startup()
|
||||
ready = True
|
||||
else:
|
||||
self.housekeeping()
|
||||
except Exception as exc:
|
||||
log.error('router reconciliation failed; decisions retained for retry: %s', exc)
|
||||
self.wake.wait(float(self.cfg.get('router_interval', 2)))
|
||||
self.wake.clear()
|
||||
# A short batching window combines bursts into one commit.
|
||||
self.stop.wait(float(self.cfg.get('batch_delay', 0.2)))
|
||||
finally:
|
||||
self.state.close()
|
||||
|
||||
def _save_checkpoint(self, tailer):
|
||||
if self.dry_run or tailer._ino is None:
|
||||
return
|
||||
self.state.save_checkpoint(tailer._ino[0], tailer._ino[1], tailer._pos)
|
||||
|
||||
def run(self):
|
||||
if self._sock:
|
||||
# bind BEFORE suricata connects: the sensor unit's ordering
|
||||
# guarantees it (Wants/After leaf-controller). a bind failure is
|
||||
# fatal, not soft: a dead listener would mean silent alert loss.
|
||||
try:
|
||||
self._sock.bind()
|
||||
except OSError as exc:
|
||||
raise SystemExit('cannot bind alert socket %s: %s'
|
||||
% (self._sock.path, exc))
|
||||
self._sock.start()
|
||||
log.info('listening for sensor alerts on %s', self._sock.path)
|
||||
ckpt = self.state.get_checkpoint() if (not self.dry_run and self.cfg.get('eve.checkpoint', True)) else None
|
||||
tailer = EveTailer(self.cfg.get('eve.path'), float(self.cfg.get('poll_interval', 0.2)),
|
||||
from_start=self.dry_run, checkpoint=ckpt,
|
||||
idle_wait=self._sock.available.wait if self._sock else self.stop.wait)
|
||||
# Establish the live position before any slow API startup calls.
|
||||
tailer._open()
|
||||
self._save_checkpoint(tailer)
|
||||
notify = os.environ.get('NOTIFY_SOCKET')
|
||||
if notify:
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as ready:
|
||||
ready.connect('\0' + notify[1:] if notify.startswith('@') else notify)
|
||||
ready.sendall(b'READY=1')
|
||||
if not self.dry_run:
|
||||
self._worker = threading.Thread(target=self._router_loop, name='vyos-worker')
|
||||
self._worker.start()
|
||||
log.info('watching %s (mode=%s)', self.cfg.get('eve.path'), self.cfg.get('policy.mode'))
|
||||
try:
|
||||
for event in tailer.events():
|
||||
if self.stop.is_set():
|
||||
break
|
||||
if self._sock:
|
||||
for rec in self._sock.drain():
|
||||
self._route(rec)
|
||||
if event is not None:
|
||||
# Socket mode has exactly one alert ingress; the file carries stats.
|
||||
if not self._sock or event.get('event_type') != 'alert':
|
||||
identity = hashlib.sha256(json.dumps(event, sort_keys=True).encode()).hexdigest()
|
||||
self.state._local.event_key = '%s:%s:%s:%s' % (*tailer._ino, tailer._pos, identity)
|
||||
try:
|
||||
self._route(event)
|
||||
finally:
|
||||
self.state._local.event_key = None
|
||||
self._emit_metrics(time.time())
|
||||
# Offset follows the delivered record, never the unread batch.
|
||||
# File mode is at-least-once across a crash before checkpoint.
|
||||
self._save_checkpoint(tailer)
|
||||
elif self.dry_run:
|
||||
self.housekeeping()
|
||||
finally:
|
||||
tailer.close()
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
if self._sock:
|
||||
self._sock.close()
|
||||
# Stop intake first, then persist every accepted queued alert.
|
||||
for rec in self._sock.drain(limit=self._sock.q.qsize()):
|
||||
self._route(rec)
|
||||
self.stop.set()
|
||||
self.wake.set()
|
||||
if self._worker:
|
||||
self._worker.join()
|
||||
self._worker = None
|
||||
self.state.close()
|
||||
self.audit.close()
|
||||
if self._capture_log:
|
||||
self._capture_log.close()
|
||||
self._lock.close()
|
||||
if self._scratch:
|
||||
self._scratch.cleanup()
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--config', default=os.path.join(os.path.dirname(__file__), '..', 'config.yaml'))
|
||||
ap.add_argument('--dry-run', action='store_true')
|
||||
ap.add_argument('-v', '--verbose', action='store_true')
|
||||
args = ap.parse_args(argv)
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format='%(asctime)s %(levelname)s %(name)s: %(message)s')
|
||||
ctl = Controller(load_config(args.config), args.dry_run)
|
||||
def shutdown(*args):
|
||||
ctl.stop.set()
|
||||
ctl.wake.set()
|
||||
signal.signal(signal.SIGTERM, shutdown)
|
||||
signal.signal(signal.SIGINT, shutdown)
|
||||
ctl.run()
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user