Code
This commit is contained in:
32
controller/config.yaml
Normal file
32
controller/config.yaml
Normal file
@@ -0,0 +1,32 @@
|
||||
# Generic deployment example. The lab installer seeds config/prototype-vyos.yaml.
|
||||
vyos:
|
||||
url: https://10.66.0.1 # VyOS management address
|
||||
key_env: LEAF_VYOS_KEY # API key read from this env var (never stored here)
|
||||
ca_bundle: /etc/leaf-controller/vyos-ca.pem # CA that signed router cert (see snippet)
|
||||
tls_verify: true
|
||||
timeout: 10
|
||||
retries: 3
|
||||
|
||||
block_group: IDS-BLOCK
|
||||
ensure_primitives: true
|
||||
adopt_foreign_members: true
|
||||
save_config: false # true = persist config on router after each change
|
||||
|
||||
eve:
|
||||
path: /var/log/suricata-leaf/eve.json
|
||||
|
||||
policy:
|
||||
mode: threshold # threshold | immediate
|
||||
min_hits: 2
|
||||
window_seconds: 300
|
||||
ttl_seconds: 3600
|
||||
max_blocks: 50
|
||||
allow_cidrs: [] # additional never-block CIDRs
|
||||
sids_only: [] # non-empty = only act on listed sids
|
||||
sids_deny: []
|
||||
block_src: false # honor metadata leaf_action=block_src
|
||||
|
||||
state_db: /var/lib/leaf-controller/state.sqlite
|
||||
audit_log: /var/log/leaf-controller/decisions.jsonl
|
||||
poll_interval: 0.2
|
||||
sync_interval: 300.0
|
||||
30
controller/leaf-controller.service
Normal file
30
controller/leaf-controller.service
Normal file
@@ -0,0 +1,30 @@
|
||||
[Unit]
|
||||
Description=LEAF alert controller and VyOS reconciliation worker
|
||||
# ordering contract: the controller binds the alert socket BEFORE the
|
||||
# sensor unit starts (suricata is the unix_stream CLIENT); do not reorder.
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
RequiresMountsFor=/var/log/suricata-leaf
|
||||
|
||||
[Service]
|
||||
Type=notify
|
||||
NotifyAccess=main
|
||||
TimeoutStartSec=30
|
||||
EnvironmentFile=/etc/leaf-controller/env
|
||||
Environment=PYTHONDONTWRITEBYTECODE=1
|
||||
ExecStart=/usr/bin/python3 -m leaf_controller.main --config /etc/leaf-controller/config.yaml
|
||||
WorkingDirectory=/opt/leaf-controller/controller
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
TimeoutStopSec=180
|
||||
User=leafctrl
|
||||
Group=leafctrl
|
||||
UMask=0027
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
ReadWritePaths=/var/lib/leaf-controller /var/log/leaf-controller /run/suricata
|
||||
PrivateTmp=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
2
controller/leaf_controller/__init__.py
Normal file
2
controller/leaf_controller/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""leaf_controller — Suricata EVE -> VyOS dynamic block controller (prototype)."""
|
||||
__version__ = "0.1.0"
|
||||
170
controller/leaf_controller/alertsock.py
Normal file
170
controller/leaf_controller/alertsock.py
Normal file
@@ -0,0 +1,170 @@
|
||||
"""Bounded AF_UNIX EVE ingress. Accepted queues are volatile until policy persists them.
|
||||
|
||||
Suricata can also lose records before delivery on EAGAIN. Local counters do
|
||||
not prove end-to-end losslessness; this is a lab control plane, not a journal.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import fcntl
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import socket
|
||||
import stat
|
||||
import threading
|
||||
|
||||
log = logging.getLogger('leaf.sock')
|
||||
MAX_RECORD = 1 << 20
|
||||
|
||||
|
||||
class AlertSocket:
|
||||
def __init__(self, path, queue_size=1000, rcvbuf=1 << 20):
|
||||
if int(queue_size) <= 0:
|
||||
raise ValueError('alert_socket.queue_size must be positive')
|
||||
self.path = path
|
||||
self.q = queue.Queue(maxsize=int(queue_size))
|
||||
self.counts = dict(connections=0, records=0, parse_errors=0, queue_drops=0, other=0)
|
||||
self.available = threading.Event()
|
||||
self._rcvbuf = int(rcvbuf)
|
||||
self._sock = self._conn = self._thread = self._lock = self._identity = None
|
||||
self._stop = threading.Event()
|
||||
|
||||
def bind(self):
|
||||
os.makedirs(os.path.dirname(os.path.abspath(self.path)), exist_ok=True)
|
||||
self._lock = open(self.path + '.lock', 'a')
|
||||
try:
|
||||
fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if os.path.lexists(self.path):
|
||||
st = os.lstat(self.path)
|
||||
if not stat.S_ISSOCK(st.st_mode):
|
||||
raise OSError('refusing to replace a non-socket path')
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as probe:
|
||||
probe.settimeout(0.2)
|
||||
try:
|
||||
probe.connect(self.path)
|
||||
except (ConnectionRefusedError, FileNotFoundError):
|
||||
os.unlink(self.path)
|
||||
else:
|
||||
raise OSError('another listener owns this socket')
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
self._sock = sock
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self._rcvbuf)
|
||||
sock.settimeout(0.2)
|
||||
sock.bind(self.path)
|
||||
st = os.lstat(self.path)
|
||||
self._identity = (st.st_dev, st.st_ino)
|
||||
os.chmod(self.path, 0o660)
|
||||
sock.listen(1)
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def start(self):
|
||||
self._thread = threading.Thread(target=self._accept_loop, name='alert-sock', daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def _accept_loop(self):
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
conn, _ = self._sock.accept()
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
break
|
||||
self._conn = conn
|
||||
self.counts['connections'] += 1
|
||||
try:
|
||||
conn.settimeout(0.2)
|
||||
conn.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self._rcvbuf)
|
||||
self._read_conn(conn)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
conn.close()
|
||||
self._conn = None
|
||||
|
||||
def _read_conn(self, conn):
|
||||
pending = bytearray()
|
||||
oversized = False
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
data = conn.recv(65536)
|
||||
except socket.timeout:
|
||||
continue
|
||||
if not data:
|
||||
if pending or oversized:
|
||||
self.counts['parse_errors'] += 1
|
||||
return
|
||||
for part in data.splitlines(keepends=True):
|
||||
complete = part.endswith(b'\n')
|
||||
if not oversized:
|
||||
pending.extend(part)
|
||||
if len(pending) > MAX_RECORD:
|
||||
pending.clear()
|
||||
oversized = True
|
||||
if complete:
|
||||
if oversized:
|
||||
self.counts['parse_errors'] += 1
|
||||
else:
|
||||
self._enqueue(pending)
|
||||
pending.clear()
|
||||
oversized = False
|
||||
|
||||
def _enqueue(self, line):
|
||||
if len(line) > MAX_RECORD:
|
||||
self.counts['parse_errors'] += 1
|
||||
return
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except (ValueError, RecursionError, UnicodeError):
|
||||
self.counts['parse_errors'] += 1
|
||||
return
|
||||
if not isinstance(rec, dict) or rec.get('event_type') != 'alert':
|
||||
self.counts['other'] += 1
|
||||
return
|
||||
try:
|
||||
self.q.put_nowait(rec)
|
||||
self.counts['records'] += 1
|
||||
self.available.set()
|
||||
except queue.Full:
|
||||
self.counts['queue_drops'] += 1
|
||||
log.error('alert queue full: dropped %d', self.counts['queue_drops'])
|
||||
|
||||
def drain(self, limit=256):
|
||||
self.available.clear()
|
||||
out = []
|
||||
while len(out) < limit:
|
||||
try:
|
||||
out.append(self.q.get_nowait())
|
||||
except queue.Empty:
|
||||
break
|
||||
if not self.q.empty():
|
||||
self.available.set()
|
||||
return out
|
||||
|
||||
def close(self):
|
||||
self._stop.set()
|
||||
if self._conn:
|
||||
try:
|
||||
self._conn.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
if self._sock:
|
||||
self._sock.close()
|
||||
if self._thread:
|
||||
self._thread.join(timeout=2)
|
||||
if self._thread.is_alive():
|
||||
raise RuntimeError('socket reader did not stop')
|
||||
self._thread = None
|
||||
self._sock = None
|
||||
if self._identity:
|
||||
try:
|
||||
st = os.lstat(self.path)
|
||||
if (st.st_dev, st.st_ino) == self._identity:
|
||||
os.unlink(self.path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
self._identity = None
|
||||
if self._lock:
|
||||
self._lock.close()
|
||||
self._lock = None
|
||||
155
controller/leaf_controller/config.py
Normal file
155
controller/leaf_controller/config.py
Normal file
@@ -0,0 +1,155 @@
|
||||
"""Config loading + defaults. YAML file, secrets via env var only."""
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
DEFAULTS: dict[str, Any] = {
|
||||
"vyos": {
|
||||
"url": "https://10.66.0.1",
|
||||
"key_env": "LEAF_VYOS_KEY",
|
||||
"ca_bundle": None, # path to CA/pinned cert; None -> system CAs
|
||||
"tls_verify": True,
|
||||
"tls_cert_pin": "", # sha256 cert fingerprint; pins TLS trust (IP-hosted self-signed router cert)
|
||||
"timeout": 10.0,
|
||||
"retries": 3,
|
||||
},
|
||||
"block_group": "IDS-BLOCK",
|
||||
"ensure_primitives": True, # verify group/rule hook exists on startup
|
||||
"adopt_foreign_members": True, # members found on router but not in state db
|
||||
"save_config": False, # POST /config-file save after every mutation
|
||||
"eve": {
|
||||
"path": "/var/log/suricata-leaf/eve.json",
|
||||
"checkpoint": True, # resume at persisted offset across restarts
|
||||
},
|
||||
"alert_socket": {
|
||||
# AF_UNIX listener for suricata's unix_stream output: alerts arrive
|
||||
# through the socket instead of eve.json. empty path disables the
|
||||
# socket path entirely (offline/mock runs keep pure file tailing).
|
||||
"path": "/run/suricata/alerts.sock",
|
||||
"queue_size": 1000,
|
||||
},
|
||||
"audit": {
|
||||
"max_bytes": 16777216, # rotate decisions.jsonl at 16 MiB
|
||||
"keep": 3, # rotated files retained
|
||||
},
|
||||
"metrics": {
|
||||
# stats records are counting-only: never reach state/audit/API.
|
||||
"reconcile_interval": 30.0, # one journal line per stats window
|
||||
"capture_interval": 3600.0, # capture.log line cadence
|
||||
"capture_log": None, # default: capture.log next to audit_log
|
||||
"capture_max_bytes": 10485760, # rotate capture.log at 10 MiB...
|
||||
"capture_log_keep": 6, # ... keeping six rotated files
|
||||
"log_level": "INFO",
|
||||
},
|
||||
"policy": {
|
||||
"mode": "threshold", # threshold | immediate
|
||||
"min_hits": 2,
|
||||
"window_seconds": 300,
|
||||
"ttl_seconds": 3600,
|
||||
"max_blocks": 50,
|
||||
"allow_cidrs": [], # extra never-block networks
|
||||
"sids_deny": [], # never act on these sids
|
||||
"sids_only": [], # if non-empty: act ONLY on these sids
|
||||
},
|
||||
"always_allow_cidrs": [
|
||||
# hard floor: never block routers/LAN/mgmt/mcast-ish space no matter what
|
||||
"10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16",
|
||||
"0.0.0.0/8", "127.0.0.0/8", "240.0.0.0/4",
|
||||
"100.64.0.0/10", "169.254.0.0/16", "224.0.0.0/4",
|
||||
"fc00::/7", "fe80::/10",
|
||||
],
|
||||
"state_db": "/var/lib/leaf-controller/state.sqlite",
|
||||
"audit_log": "/var/log/leaf-controller/decisions.jsonl",
|
||||
"poll_interval": 0.2,
|
||||
"sync_interval": 300.0,
|
||||
}
|
||||
|
||||
|
||||
def _merge(base: dict, override: dict) -> dict:
|
||||
out = dict(base)
|
||||
for k, v in override.items():
|
||||
if isinstance(v, dict) and isinstance(out.get(k), dict):
|
||||
out[k] = _merge(out[k], v)
|
||||
else:
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
raw: dict = field(default_factory=dict)
|
||||
|
||||
def get(self, dotted: str, default: Any = None) -> Any:
|
||||
cur: Any = self.raw
|
||||
for part in dotted.split("."):
|
||||
if not isinstance(cur, dict) or part not in cur:
|
||||
return default
|
||||
cur = cur[part]
|
||||
return cur
|
||||
|
||||
# --- derived helpers ---
|
||||
def vyos_url(self) -> str:
|
||||
return self.get("vyos.url").rstrip("/")
|
||||
|
||||
def vyos_key(self) -> str:
|
||||
env = self.get("vyos.key_env", "LEAF_VYOS_KEY")
|
||||
key = os.environ.get(env, "")
|
||||
if not key and not self.get("dry_run"):
|
||||
raise SystemExit(f"missing API key: set env var {env}")
|
||||
return key
|
||||
|
||||
def allow_networks(self) -> list:
|
||||
nets = list(DEFAULTS["always_allow_cidrs"]) + list(self.get("always_allow_cidrs", [])) + list(self.get("policy.allow_cidrs", []))
|
||||
return [ipaddress.ip_network(c, strict=False) for c in nets]
|
||||
|
||||
def is_allowed_ip(self, ip: str) -> bool:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return True # malformed / non-ip (eg:66 etc) -> treat as allowed, never block
|
||||
return any(addr in net for net in self.allow_networks())
|
||||
|
||||
|
||||
def load_config(path: str) -> Config:
|
||||
with open(path) as fh:
|
||||
user = yaml.safe_load(fh) or {}
|
||||
if not isinstance(user, dict):
|
||||
raise SystemExit(f"{path}: expected a mapping")
|
||||
cfg = Config(raw=_merge(DEFAULTS, user))
|
||||
import math
|
||||
for name in ('alert_socket.queue_size', 'policy.min_hits', 'policy.max_blocks',
|
||||
'policy.window_seconds', 'policy.ttl_seconds', 'poll_interval',
|
||||
'sync_interval', 'router_interval', 'vyos.timeout', 'vyos.retries'):
|
||||
value = cfg.get(name, 2 if name == 'router_interval' else None)
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
|
||||
raise ValueError(name + ' must be a positive finite number')
|
||||
for name in ('alert_socket.queue_size', 'policy.min_hits', 'policy.max_blocks', 'vyos.retries'):
|
||||
if int(cfg.get(name)) != cfg.get(name):
|
||||
raise ValueError(name + ' must be an integer')
|
||||
for name in ('audit.max_bytes', 'audit.keep', 'metrics.capture_max_bytes', 'metrics.capture_log_keep'):
|
||||
v = cfg.get(name)
|
||||
if isinstance(v,bool) or not isinstance(v,int) or v < 0:
|
||||
raise ValueError(name + ' must be a non-negative integer')
|
||||
if cfg.get('policy.mode') not in ('threshold', 'immediate'):
|
||||
raise ValueError('policy.mode must be threshold or immediate')
|
||||
cfg.allow_networks() # Validate every configured CIDR at startup.
|
||||
return cfg
|
||||
|
||||
|
||||
def read_key_env(path, name='LEAF_VYOS_KEY'):
|
||||
"""Read the simple quoted assignment emitted by the credential tool; never eval."""
|
||||
import shlex
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
key, sep, value = line.strip().partition('=')
|
||||
if sep and key == name:
|
||||
parts = shlex.split(value, comments=True)
|
||||
if len(parts) != 1 or not parts[0]:
|
||||
raise ValueError('invalid API key assignment')
|
||||
return parts[0]
|
||||
raise ValueError('API key assignment missing')
|
||||
106
controller/leaf_controller/eve.py
Normal file
106
controller/leaf_controller/eve.py
Normal file
@@ -0,0 +1,106 @@
|
||||
"""Read complete EVE records, preserving partial writes and handling rotation."""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
log = logging.getLogger("leaf.eve")
|
||||
|
||||
class EveTailer:
|
||||
def __init__(self, path, poll_interval=0.2, from_start=False, checkpoint=None, idle_wait=None):
|
||||
self.path, self.poll, self.from_start = path, poll_interval, from_start
|
||||
# (dev, ino, pos) persisted while the controller was running; if it
|
||||
# matches the file on disk we resume there instead of skipping ahead.
|
||||
self.checkpoint = checkpoint
|
||||
self.idle_wait = idle_wait or time.sleep
|
||||
self._fh = None
|
||||
self._ino = None
|
||||
self._pos = 0
|
||||
|
||||
def _open(self):
|
||||
try:
|
||||
fh = open(self.path, "rb")
|
||||
except FileNotFoundError:
|
||||
# A file created after startup must be read from its beginning.
|
||||
self.from_start = True
|
||||
return None
|
||||
st = os.fstat(fh.fileno())
|
||||
self._ino = (st.st_dev, st.st_ino)
|
||||
if not self.from_start:
|
||||
ck = self.checkpoint
|
||||
if ck and ck[:2] != self._ino:
|
||||
log.warning("eve checkpoint inode mismatch on %s; processing "
|
||||
"current file from start", self.path)
|
||||
fh.seek(0)
|
||||
elif ck and ck[2] <= st.st_size:
|
||||
fh.seek(ck[2])
|
||||
elif ck:
|
||||
log.warning("eve checkpoint pos %d beyond size %d; rereading %s from 0",
|
||||
ck[2], st.st_size, self.path)
|
||||
fh.seek(0)
|
||||
else:
|
||||
# First run ever: never replay pre-existing history.
|
||||
fh.seek(0, os.SEEK_END)
|
||||
self.from_start = True
|
||||
self._fh, self._pos = fh, fh.tell()
|
||||
log.info("opened eve file %s (pos=%d)", self.path, self._pos)
|
||||
return fh
|
||||
|
||||
def _needs_reopen(self):
|
||||
try:
|
||||
st = os.stat(self.path)
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
return (st.st_dev, st.st_ino) != self._ino or st.st_size < (self._fh.tell() if self._fh else self._pos)
|
||||
|
||||
def _read_ready(self, fh):
|
||||
# Never consume a partial record: the writer may append its remainder.
|
||||
out = []
|
||||
for _ in range(1000):
|
||||
pos = fh.tell()
|
||||
line = fh.readline()
|
||||
if not line.endswith(b"\n"):
|
||||
fh.seek(pos)
|
||||
break
|
||||
out.append((line.decode("utf-8", errors="replace"), fh.tell()))
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _parse(line):
|
||||
if isinstance(line, tuple):
|
||||
line = line[0]
|
||||
try:
|
||||
event = json.loads(line)
|
||||
return event if isinstance(event, dict) else None
|
||||
except (ValueError, RecursionError):
|
||||
log.warning("skipping invalid EVE JSON record")
|
||||
return None
|
||||
|
||||
def events(self):
|
||||
try:
|
||||
while True:
|
||||
fh = self._fh or self._open()
|
||||
if fh is None:
|
||||
self.idle_wait(self.poll)
|
||||
yield None
|
||||
continue
|
||||
lines = self._read_ready(fh)
|
||||
for line, end in lines:
|
||||
self._pos = end
|
||||
parsed = self._parse(line)
|
||||
if parsed is not None:
|
||||
yield parsed
|
||||
if self._needs_reopen() and not lines:
|
||||
fh.close()
|
||||
self._fh = None
|
||||
continue
|
||||
if not lines:
|
||||
self.idle_wait(self.poll)
|
||||
yield None
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
if self._fh:
|
||||
self._fh.close()
|
||||
self._fh = None
|
||||
403
controller/leaf_controller/main.py
Normal file
403
controller/leaf_controller/main.py
Normal file
@@ -0,0 +1,403 @@
|
||||
"""Durable alert decisions with a separate, serialized VyOS reconciliation worker."""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import fcntl
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import threading
|
||||
import tempfile
|
||||
import time
|
||||
from .alertsock import AlertSocket
|
||||
from .config import load_config
|
||||
from .eve import EveTailer
|
||||
from .metrics import Metrics
|
||||
from .policy import Policy
|
||||
from .state import State
|
||||
from .vyos_client import VyosApiError, VyosClient
|
||||
|
||||
log = logging.getLogger('leaf')
|
||||
|
||||
class Audit:
|
||||
def __init__(self, path, max_bytes=0, keep=3):
|
||||
os.makedirs(os.path.dirname(os.path.abspath(path)), exist_ok=True)
|
||||
self.path, self.max_bytes, self.keep = path, int(max_bytes), int(keep)
|
||||
self.fh = open(path, 'a')
|
||||
self.lock = threading.Lock()
|
||||
def write(self, obj):
|
||||
with self.lock:
|
||||
self.fh.write(json.dumps(dict(ts=time.time(), **obj)) + '\n')
|
||||
self.fh.flush()
|
||||
if self.max_bytes and self.fh.tell() >= self.max_bytes:
|
||||
self._rotate_locked()
|
||||
def _rotate_locked(self):
|
||||
"""Rotate at the configured size and retain the newest archives."""
|
||||
self.fh.close()
|
||||
stamp = str(time.time_ns()).zfill(20)
|
||||
dst = self.path + '.' + stamp
|
||||
n = 2
|
||||
while os.path.exists(dst): # tolerate a frozen/coarse clock
|
||||
dst = self.path + '.' + stamp + f'.{n:08d}'
|
||||
n += 1
|
||||
os.replace(self.path, dst)
|
||||
self.fh = open(self.path, 'a')
|
||||
rotated = sorted(glob.glob(self.path + '.*'), key=lambda f: (os.stat(f).st_mtime_ns, f))
|
||||
doomed = rotated[:-self.keep] if self.keep else rotated
|
||||
for old in doomed:
|
||||
try:
|
||||
os.remove(old)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
log.info('rotated audit log, keeping %d rotated file(s)', self.keep or 0)
|
||||
def close(self):
|
||||
self.fh.close()
|
||||
|
||||
class Controller:
|
||||
def __init__(self, cfg, dry_run=False):
|
||||
self.cfg, self.dry_run = cfg, dry_run
|
||||
self._closed = False
|
||||
self.group = cfg.get('block_group', 'IDS-BLOCK')
|
||||
self._scratch = tempfile.TemporaryDirectory(prefix='leaf-dry-run-') if dry_run else None
|
||||
state_path = os.path.join(self._scratch.name, 'state.sqlite') if self._scratch else cfg.get('state_db')
|
||||
os.makedirs(os.path.dirname(os.path.abspath(state_path)), exist_ok=True)
|
||||
self._lock = open(state_path + '.lock', 'a')
|
||||
try:
|
||||
fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
self._lock.close()
|
||||
raise SystemExit('another controller owns this state database')
|
||||
self.state = State(state_path)
|
||||
self.policy = Policy(cfg, self.state)
|
||||
self.audit = Audit(os.path.join(self._scratch.name, "audit.jsonl") if dry_run else cfg.get("audit_log"),
|
||||
int(cfg.get("audit.max_bytes", 16777216)),
|
||||
int(cfg.get("audit.keep", 3)))
|
||||
# metrics: counting only; stats records never touch state/audit/API.
|
||||
# health line once per full stats window, capture.log every
|
||||
# metrics.capture_interval seconds (same bounded scheme as audit).
|
||||
self.metrics = Metrics(float(cfg.get("metrics.reconcile_interval", 30.0)))
|
||||
self._metrics_log = logging.getLogger("leaf.metrics")
|
||||
self._metrics_log.setLevel(getattr(
|
||||
logging, str(cfg.get("metrics.log_level", "INFO")).upper(), logging.INFO))
|
||||
self._capture_path = (cfg.get("metrics.capture_log")
|
||||
or os.path.join(os.path.dirname(
|
||||
os.path.abspath(cfg.get("audit_log"))), "capture.log"))
|
||||
if dry_run:
|
||||
self._capture_path = os.path.join(self._scratch.name, "capture.jsonl")
|
||||
self._capture_interval = max(0.0, float(cfg.get("metrics.capture_interval", 3600)))
|
||||
self._capture_log = None
|
||||
self._next_capture = 0.0
|
||||
self.client = None if dry_run else VyosClient(
|
||||
cfg.vyos_url(), cfg.vyos_key(), ca_bundle=cfg.get('vyos.ca_bundle'),
|
||||
tls_verify=cfg.get('vyos.tls_verify', True),
|
||||
cert_pin=cfg.get('vyos.tls_cert_pin') or None,
|
||||
timeout=float(cfg.get('vyos.timeout', 75)),
|
||||
retries=int(cfg.get('vyos.retries', 2)))
|
||||
self.stop = threading.Event()
|
||||
self.wake = threading.Event()
|
||||
self._last_sync = 0.0
|
||||
self._initial_sync = True
|
||||
self._worker = None
|
||||
# socket ingestion: suricata unix_stream output connects to our
|
||||
# listener; empty path disables (offline/mock runs keep file tailing)
|
||||
sock_path = cfg.get('alert_socket.path')
|
||||
self._sock = None
|
||||
if sock_path and not dry_run:
|
||||
self._sock = AlertSocket(
|
||||
sock_path, int(cfg.get('alert_socket.queue_size') or 1000))
|
||||
|
||||
def router_members(self):
|
||||
return set(self.client.op_with_retry(self.client.group_members, self.group))
|
||||
|
||||
def startup(self):
|
||||
if self.dry_run:
|
||||
return
|
||||
info = self.client.info()
|
||||
log.info('connected to VyOS API: version=%s', info.get('version', '?'))
|
||||
if self.cfg.get('ensure_primitives', True):
|
||||
# Both metadata actions use one group; validate the corresponding hooks.
|
||||
self.client.verify_enforcement(self.group, self.cfg.get('policy.block_src', False))
|
||||
self.sync_router_state()
|
||||
|
||||
def sync_router_state(self):
|
||||
"""One batched commit, then read back before acknowledging any change."""
|
||||
if self.dry_run:
|
||||
return
|
||||
now = time.time()
|
||||
members = self.router_members()
|
||||
rows = {r.ip: r for r in self.state.all_blocks()}
|
||||
# Only adopt on startup, never resurrect known expired entries.
|
||||
foreign = members - rows.keys()
|
||||
if self._initial_sync and self.cfg.get('adopt_foreign_members', True):
|
||||
for ip in sorted(foreign):
|
||||
if self.cfg.is_allowed_ip(ip):
|
||||
log.warning('leaving protected foreign router member unowned: %s', ip)
|
||||
continue
|
||||
if self.state.count_active() >= int(self.cfg.get('policy.max_blocks', 50)):
|
||||
log.warning('foreign member adoption capped; leaving unowned: %s', ip)
|
||||
continue
|
||||
self.state.upsert_hit(ip, 'dest', 0, float(self.cfg.get('policy.ttl_seconds', 3600)), now)
|
||||
self.state.set_applied(ip)
|
||||
self.audit.write({'action': 'adopt', 'ip': ip})
|
||||
log.info('adopted foreign member %s into state', ip)
|
||||
rows = {r.ip: r for r in self.state.all_blocks()}
|
||||
self._initial_sync = False
|
||||
active = {ip for ip, r in rows.items() if r.expires > now and not self.cfg.is_allowed_ip(ip)}
|
||||
expired = rows.keys() - active
|
||||
adds, deletes = active - members, set(expired) & members
|
||||
if not self.cfg.get('adopt_foreign_members', True):
|
||||
deletes |= foreign
|
||||
ops = [self.client.member_op('set', self.group, ip) for ip in sorted(adds)]
|
||||
ops += [self.client.member_op('delete', self.group, ip) for ip in sorted(deletes)]
|
||||
if ops:
|
||||
# A timeout may mean the commit succeeded: next pass reads before retrying.
|
||||
try:
|
||||
self.state.set_router_dirty(True)
|
||||
self.client.configure(ops)
|
||||
except Exception:
|
||||
self.audit.write({'action': 'reconcile', 'ok': False,
|
||||
'adds': sorted(adds), 'deletes': sorted(deletes)})
|
||||
raise
|
||||
members = self.router_members()
|
||||
if not adds <= members or deletes & members:
|
||||
raise VyosApiError('router readback does not match requested membership changes')
|
||||
if self.cfg.get('save_config') and self.state.router_dirty():
|
||||
self.client.save_config()
|
||||
self.state.set_router_dirty(False)
|
||||
for ip in active & members:
|
||||
row = rows[ip]
|
||||
if not row.applied or ip in adds:
|
||||
self.state.set_applied(ip)
|
||||
self.audit.write({'action': 'block', 'ip': ip, 'kind': row.kind,
|
||||
'sigs': row.sigs, 'expires': row.expires,
|
||||
'ok': True, 'verified': True})
|
||||
log.info('BLOCKED %s (%s), router membership verified', ip, row.kind)
|
||||
for ip in expired - members:
|
||||
row = rows[ip]
|
||||
# Do not delete a fresh decision made during the slow router request.
|
||||
removed = self.state.remove(ip, expired_before=now)
|
||||
if removed:
|
||||
self.audit.write({'action': 'unblock', 'ip': ip, 'kind': row.kind,
|
||||
'reason': 'ttl-expired', 'ok': True, 'verified': True})
|
||||
log.info('UNBLOCKED %s, router absence verified', ip)
|
||||
elif self.cfg.is_allowed_ip(ip):
|
||||
# A newly protected address must not remain locally desired.
|
||||
self.state.remove(ip)
|
||||
else:
|
||||
self.state.set_applied(ip, 0)
|
||||
self.wake.set()
|
||||
self._last_sync = time.time()
|
||||
|
||||
def _route(self, event):
|
||||
"""Dispatch by event_type; only alerts can act. Stats records are
|
||||
counting-only: they must never reach policy/state/audit/API."""
|
||||
etype = event.get('event_type')
|
||||
if etype == 'alert':
|
||||
self.metrics.note_alert()
|
||||
self.handle_event(event)
|
||||
elif etype == 'stats':
|
||||
try:
|
||||
self.metrics.note_stats(event)
|
||||
except (TypeError, ValueError, AttributeError, OverflowError, RecursionError):
|
||||
log.warning('ignoring malformed stats record')
|
||||
self.metrics.note_other()
|
||||
else:
|
||||
self.metrics.note_other()
|
||||
|
||||
def _sock_desc(self):
|
||||
if self._sock is None:
|
||||
return 'off(eve-file)'
|
||||
c = self._sock.counts
|
||||
return ('rec:%d conn:%d qdrop:%d perr:%d other:%d'
|
||||
% (c['records'], c['connections'], c['queue_drops'],
|
||||
c['parse_errors'], c['other']))
|
||||
|
||||
def _emit_metrics(self, now):
|
||||
"""One health line per full stats window; capture.log line on cadence.
|
||||
Read-only against controller state (count_active is a query)."""
|
||||
rep = self.metrics.report(now)
|
||||
if rep is None:
|
||||
return
|
||||
if rep['init']:
|
||||
self._metrics_log.info('metrics baseline: %s',
|
||||
json.dumps(rep['stats'], separators=(',', ':')))
|
||||
return
|
||||
flag = ' (counters reset: suricata restart)' if rep['restart'] else ''
|
||||
self._metrics_log.info(
|
||||
'health: alerts=%d cps=%.1f eps=%.1f dropped+=%d errors+=%d '
|
||||
'invalid+=%d suppressed+=%d overflow+=%d blocks=%d uptime=%ds sock=%s%s',
|
||||
rep['alerts'], rep['cps'], rep['eps'], rep['dropped'],
|
||||
rep['delta']['capture_errors'], rep['delta']['invalid'],
|
||||
rep['delta']['suppressed'], rep['delta']['queue_overflow'],
|
||||
self.state.count_active(), rep['stats']['uptime'],
|
||||
self._sock_desc(), flag)
|
||||
if now >= self._next_capture:
|
||||
self._next_capture = now + self._capture_interval
|
||||
if self._capture_log is None:
|
||||
self._capture_log = Audit(
|
||||
self._capture_path,
|
||||
int(self.cfg.get('metrics.capture_max_bytes', 10485760)),
|
||||
int(self.cfg.get('metrics.capture_log_keep', 6)))
|
||||
self._capture_log.write({'window': rep['window'],
|
||||
'alerts': rep['alerts'],
|
||||
'cps': rep['cps'], 'eps': rep['eps'],
|
||||
'dropped': rep['dropped'],
|
||||
'delta': rep['delta'],
|
||||
'stats': rep['stats']})
|
||||
|
||||
def handle_event(self, event):
|
||||
try:
|
||||
dec = self.policy.evaluate(event)
|
||||
except (TypeError, ValueError, AttributeError, OverflowError, RecursionError):
|
||||
log.warning('ignoring malformed alert record')
|
||||
return
|
||||
if not dec:
|
||||
return
|
||||
existing = self.state.get(dec.ip)
|
||||
if (not existing or existing.expires <= time.time()) and self.state.count_active() >= int(self.cfg.get('policy.max_blocks', 50)):
|
||||
self.audit.write({'action': 'guard', 'ip': dec.ip, 'sid': dec.sid, 'reason': 'max_blocks'})
|
||||
return
|
||||
row = self.state.upsert_hit(dec.ip, dec.kind, dec.sid,
|
||||
float(self.cfg.get('policy.ttl_seconds', 3600)))
|
||||
if not row.applied:
|
||||
self.audit.write({'action': 'decision', 'ip': dec.ip, 'kind': dec.kind,
|
||||
'sid': dec.sid, 'expires': row.expires, 'dry_run': self.dry_run})
|
||||
if self.dry_run:
|
||||
if not row.applied:
|
||||
self.audit.write({'action': 'block', 'ip': dec.ip, 'kind': dec.kind, 'dry_run': True})
|
||||
self.state.set_applied(dec.ip)
|
||||
else:
|
||||
self.wake.set()
|
||||
|
||||
def housekeeping(self):
|
||||
now = time.time()
|
||||
self.state.prune_hits(float(self.cfg.get('policy.window_seconds', 300)) * 2, now)
|
||||
if self.dry_run:
|
||||
for row in self.state.expired(now):
|
||||
self.state.remove(row.ip, expired_before=now)
|
||||
return
|
||||
if (self.cfg.get('save_config') and self.state.router_dirty()) or self.state.unapplied() or self.state.expired(now) or now - self._last_sync >= float(self.cfg.get('sync_interval', 300)):
|
||||
self.sync_router_state()
|
||||
|
||||
def _router_loop(self):
|
||||
ready = False
|
||||
try:
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
if not ready:
|
||||
self.startup()
|
||||
ready = True
|
||||
else:
|
||||
self.housekeeping()
|
||||
except Exception as exc:
|
||||
log.error('router reconciliation failed; decisions retained for retry: %s', exc)
|
||||
self.wake.wait(float(self.cfg.get('router_interval', 2)))
|
||||
self.wake.clear()
|
||||
# A short batching window combines bursts into one commit.
|
||||
self.stop.wait(float(self.cfg.get('batch_delay', 0.2)))
|
||||
finally:
|
||||
self.state.close()
|
||||
|
||||
def _save_checkpoint(self, tailer):
|
||||
if self.dry_run or tailer._ino is None:
|
||||
return
|
||||
self.state.save_checkpoint(tailer._ino[0], tailer._ino[1], tailer._pos)
|
||||
|
||||
def run(self):
|
||||
if self._sock:
|
||||
# bind BEFORE suricata connects: the sensor unit's ordering
|
||||
# guarantees it (Wants/After leaf-controller). a bind failure is
|
||||
# fatal, not soft: a dead listener would mean silent alert loss.
|
||||
try:
|
||||
self._sock.bind()
|
||||
except OSError as exc:
|
||||
raise SystemExit('cannot bind alert socket %s: %s'
|
||||
% (self._sock.path, exc))
|
||||
self._sock.start()
|
||||
log.info('listening for sensor alerts on %s', self._sock.path)
|
||||
ckpt = self.state.get_checkpoint() if (not self.dry_run and self.cfg.get('eve.checkpoint', True)) else None
|
||||
tailer = EveTailer(self.cfg.get('eve.path'), float(self.cfg.get('poll_interval', 0.2)),
|
||||
from_start=self.dry_run, checkpoint=ckpt,
|
||||
idle_wait=self._sock.available.wait if self._sock else self.stop.wait)
|
||||
# Establish the live position before any slow API startup calls.
|
||||
tailer._open()
|
||||
self._save_checkpoint(tailer)
|
||||
notify = os.environ.get('NOTIFY_SOCKET')
|
||||
if notify:
|
||||
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as ready:
|
||||
ready.connect('\0' + notify[1:] if notify.startswith('@') else notify)
|
||||
ready.sendall(b'READY=1')
|
||||
if not self.dry_run:
|
||||
self._worker = threading.Thread(target=self._router_loop, name='vyos-worker')
|
||||
self._worker.start()
|
||||
log.info('watching %s (mode=%s)', self.cfg.get('eve.path'), self.cfg.get('policy.mode'))
|
||||
try:
|
||||
for event in tailer.events():
|
||||
if self.stop.is_set():
|
||||
break
|
||||
if self._sock:
|
||||
for rec in self._sock.drain():
|
||||
self._route(rec)
|
||||
if event is not None:
|
||||
# Socket mode has exactly one alert ingress; the file carries stats.
|
||||
if not self._sock or event.get('event_type') != 'alert':
|
||||
identity = hashlib.sha256(json.dumps(event, sort_keys=True).encode()).hexdigest()
|
||||
self.state._local.event_key = '%s:%s:%s:%s' % (*tailer._ino, tailer._pos, identity)
|
||||
try:
|
||||
self._route(event)
|
||||
finally:
|
||||
self.state._local.event_key = None
|
||||
self._emit_metrics(time.time())
|
||||
# Offset follows the delivered record, never the unread batch.
|
||||
# File mode is at-least-once across a crash before checkpoint.
|
||||
self._save_checkpoint(tailer)
|
||||
elif self.dry_run:
|
||||
self.housekeeping()
|
||||
finally:
|
||||
tailer.close()
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
if self._sock:
|
||||
self._sock.close()
|
||||
# Stop intake first, then persist every accepted queued alert.
|
||||
for rec in self._sock.drain(limit=self._sock.q.qsize()):
|
||||
self._route(rec)
|
||||
self.stop.set()
|
||||
self.wake.set()
|
||||
if self._worker:
|
||||
self._worker.join()
|
||||
self._worker = None
|
||||
self.state.close()
|
||||
self.audit.close()
|
||||
if self._capture_log:
|
||||
self._capture_log.close()
|
||||
self._lock.close()
|
||||
if self._scratch:
|
||||
self._scratch.cleanup()
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--config', default=os.path.join(os.path.dirname(__file__), '..', 'config.yaml'))
|
||||
ap.add_argument('--dry-run', action='store_true')
|
||||
ap.add_argument('-v', '--verbose', action='store_true')
|
||||
args = ap.parse_args(argv)
|
||||
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format='%(asctime)s %(levelname)s %(name)s: %(message)s')
|
||||
ctl = Controller(load_config(args.config), args.dry_run)
|
||||
def shutdown(*args):
|
||||
ctl.stop.set()
|
||||
ctl.wake.set()
|
||||
signal.signal(signal.SIGTERM, shutdown)
|
||||
signal.signal(signal.SIGINT, shutdown)
|
||||
ctl.run()
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
105
controller/leaf_controller/metrics.py
Normal file
105
controller/leaf_controller/metrics.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""Counting-only metrics from Suricata stats records.
|
||||
|
||||
Stats records update counters without changing policy, SQLite state or
|
||||
router configuration. The caller controls when reports are logged.
|
||||
|
||||
Live eve.json (Suricata 7, threads: no + totals: yes) carries
|
||||
capture/decoder/detect sub-objects on each stats record; there is no
|
||||
'ifaces' object at totals level, so capture packets come from
|
||||
capture.kernel_packets (the real field on the wire).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class Metrics:
|
||||
"""Cumulative Suricata counters; one delta report per full window.
|
||||
|
||||
The report gate exists because stats records arrive every
|
||||
metrics.reconcile_interval seconds; a backlog drain can deliver two
|
||||
stats records back-to-back, and the journal must still see at most one
|
||||
line per window.
|
||||
"""
|
||||
|
||||
def __init__(self, reconcile_interval: float = 30.0):
|
||||
self.interval = max(1.0, float(reconcile_interval))
|
||||
self.alerts = 0
|
||||
self.non_alert_events = 0
|
||||
self.stats_seen = 0
|
||||
self._totals: dict[str, int] = {}
|
||||
self._baseline: dict[str, int] | None = None
|
||||
self._last = 0.0
|
||||
|
||||
def note_alert(self) -> None:
|
||||
self.alerts += 1
|
||||
|
||||
def note_other(self) -> None:
|
||||
self.non_alert_events += 1
|
||||
|
||||
def note_stats(self, event: dict) -> None:
|
||||
"""Extract the cumulative counters; nothing else is logged or stored.
|
||||
|
||||
Live suricata emits both thread records (flush=False: per-thread
|
||||
counters, 'engine' key) and a totals record (uptime/memcap_pressure
|
||||
keys, 'detect' key). Totals are authoritative for rates; thread
|
||||
records are only accepted if no totals have ever been seen.
|
||||
"""
|
||||
stats = event.get("stats") or {}
|
||||
cap = stats.get("capture") or {}
|
||||
dec = stats.get("decoder") or {}
|
||||
det = stats.get("detect") or stats.get("engine") or {}
|
||||
totals = {
|
||||
"uptime": int(stats.get("uptime", 0)),
|
||||
"memcap_pressure": int(stats.get("memcap_pressure", 0)),
|
||||
"packets": int(cap.get("kernel_packets", 0)),
|
||||
"dropped": int(cap.get("kernel_drops", 0)),
|
||||
"capture_errors": int(cap.get("errors", 0)),
|
||||
"decoded": int(dec.get("pkts", 0)),
|
||||
"bytes": int(dec.get("bytes", 0)),
|
||||
"invalid": int(dec.get("invalid", 0)),
|
||||
"engine_alerts": int(det.get("alert", 0)),
|
||||
"suppressed": int(det.get("alerts_suppressed", 0)),
|
||||
"queue_overflow": int(det.get("alert_queue_overflow", 0)),
|
||||
}
|
||||
if "uptime" in stats or not self._totals:
|
||||
self._totals = totals
|
||||
self.stats_seen += 1
|
||||
|
||||
def report(self, now: float) -> dict | None:
|
||||
"""None unless a full flush window has passed since the last report.
|
||||
|
||||
First stats record only adopts the cumulative baseline: deltas are
|
||||
meaningless before the second stats tick (counters run since process
|
||||
start), so the first window must not pretend to be a measured one.
|
||||
Suricata's counters reset on restart: deltas never go negative, the
|
||||
reset is flagged instead.
|
||||
"""
|
||||
if not self._totals:
|
||||
return None
|
||||
if self._baseline is None:
|
||||
self._baseline = dict(self._totals)
|
||||
self._last = now
|
||||
return {"init": True, "alerts": self.alerts, "stats": dict(self._totals)}
|
||||
if now - self._last < self.interval * 0.75: # tolerate flush jitter
|
||||
return None
|
||||
window = now - self._last
|
||||
delta, restart = {}, False
|
||||
for key, value in self._totals.items():
|
||||
n = value - self._baseline.get(key, 0)
|
||||
if n < 0:
|
||||
restart = True
|
||||
n = value
|
||||
delta[key] = n
|
||||
self._baseline = dict(self._totals)
|
||||
self._last = now
|
||||
return {
|
||||
"init": False,
|
||||
"restart": restart,
|
||||
"window": round(window, 1),
|
||||
"alerts": self.alerts,
|
||||
"engine_alerts_delta": delta["engine_alerts"],
|
||||
"cps": round(delta["packets"] / window, 1), # captured pps
|
||||
"eps": round(delta["decoded"] / window, 1), # decoded pps
|
||||
"dropped": delta["dropped"],
|
||||
"delta": delta,
|
||||
"stats": dict(self._totals),
|
||||
}
|
||||
22
controller/leaf_controller/ops.py
Normal file
22
controller/leaf_controller/ops.py
Normal file
@@ -0,0 +1,22 @@
|
||||
"""Sensor-side operational helpers sharing the installed controller trust settings."""
|
||||
import os
|
||||
from .config import load_config, read_key_env
|
||||
from .vyos_client import VyosClient
|
||||
|
||||
|
||||
def configured_client():
|
||||
cfg=load_config('/etc/leaf-controller/config.yaml')
|
||||
name=cfg.get('vyos.key_env','LEAF_VYOS_KEY')
|
||||
key=os.environ.get(name) or read_key_env('/etc/leaf-controller/env', name)
|
||||
client=VyosClient(cfg.vyos_url(),key,cert_pin=cfg.get('vyos.tls_cert_pin') or None,
|
||||
ca_bundle=cfg.get('vyos.ca_bundle'),tls_verify=cfg.get('vyos.tls_verify',True),
|
||||
timeout=float(cfg.get('vyos.timeout',75)),retries=int(cfg.get('vyos.retries',2)))
|
||||
return cfg,client
|
||||
|
||||
|
||||
def redact(node):
|
||||
if isinstance(node,dict):
|
||||
return {k: '[REDACTED]' if any(s in k.lower() for s in ('key','password','secret','token','private'))
|
||||
else redact(v) for k,v in node.items()}
|
||||
if isinstance(node,list):return [redact(v) for v in node]
|
||||
return node
|
||||
91
controller/leaf_controller/policy.py
Normal file
91
controller/leaf_controller/policy.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""Alert -> decision. Rule metadata leaf_action drives intent, policy gates it."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
log = logging.getLogger("leaf.policy")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Decision:
|
||||
ip: str
|
||||
kind: str # dest | src
|
||||
sid: int
|
||||
reason: str
|
||||
sig: str = ""
|
||||
hits: int = 1
|
||||
|
||||
|
||||
class Policy:
|
||||
def __init__(self, cfg, state):
|
||||
self.cfg = cfg
|
||||
self.state = state
|
||||
|
||||
@staticmethod
|
||||
def metadata_actions(event: dict) -> list[str]:
|
||||
md = event.get("alert", {}).get("metadata", {}) or {}
|
||||
vals = md.get("leaf_action", []) or []
|
||||
if isinstance(vals, str):
|
||||
vals = [vals]
|
||||
return [str(v).strip().lower() for v in vals]
|
||||
|
||||
def evaluate(self, event: dict, now: float | None = None) -> Decision | None:
|
||||
"""Return Decision to block, or None. `event` = eve alert record."""
|
||||
now = now or time.time()
|
||||
if event.get("event_type") != "alert":
|
||||
return None
|
||||
alert = event.get("alert", {})
|
||||
sid = int(alert.get("signature_id", -1))
|
||||
sig = str(alert.get("signature", ""))
|
||||
actions = self.metadata_actions(event)
|
||||
if not actions:
|
||||
return None
|
||||
|
||||
sids_only = [int(s) for s in self.cfg.get("policy.sids_only", [])]
|
||||
sids_deny = [int(s) for s in self.cfg.get("policy.sids_deny", [])]
|
||||
if sids_only and sid not in sids_only:
|
||||
return None
|
||||
if sid in sids_deny:
|
||||
return None
|
||||
|
||||
mode = self.cfg.get("policy.mode", "threshold")
|
||||
src_ip = str(event.get("src_ip", ""))
|
||||
dest_ip = str(event.get("dest_ip", ""))
|
||||
|
||||
candidates: list[tuple[str, str]] = []
|
||||
if "block" in actions:
|
||||
candidates.append((dest_ip, "dest"))
|
||||
if "block_src" in actions and self.cfg.get("policy.block_src", False):
|
||||
candidates.append((src_ip, "src"))
|
||||
|
||||
for ip, kind in candidates:
|
||||
dec = self._gate(ip, kind, sid, sig, now, mode)
|
||||
if dec:
|
||||
return dec
|
||||
return None
|
||||
|
||||
def _gate(self, ip: str, kind: str, sid: int, sig: str,
|
||||
now: float, mode: str) -> Decision | None:
|
||||
if not ip:
|
||||
return None
|
||||
import ipaddress
|
||||
try:
|
||||
if ipaddress.ip_address(ip).version != 4:
|
||||
return None
|
||||
except ValueError:
|
||||
return None
|
||||
if self.cfg.is_allowed_ip(ip):
|
||||
log.debug("allowlisted, not blocking: %s (sid=%s)", ip, sid)
|
||||
return None
|
||||
self.state.record_hit(sid, ip, now)
|
||||
min_hits = int(self.cfg.get("policy.min_hits", 1))
|
||||
window = float(self.cfg.get("policy.window_seconds", 300))
|
||||
hits = self.state.hit_count(sid, ip, window, now)
|
||||
if mode == "immediate" or hits >= min_hits:
|
||||
return Decision(ip=ip, kind=kind, sid=sid, sig=sig, hits=hits,
|
||||
reason=f"{mode}:{hits} hit(s) sid={sid} in {window:.0f}s")
|
||||
log.info("hit %d/%d for %s (sid=%s) — not yet blocking", hits, min_hits, ip, sid)
|
||||
return None
|
||||
176
controller/leaf_controller/state.py
Normal file
176
controller/leaf_controller/state.py
Normal file
@@ -0,0 +1,176 @@
|
||||
"""sqlite-backed state: block list w/ TTL, hit windowing, applied tracking."""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import time
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS controller_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
CREATE TABLE IF NOT EXISTS blocks (
|
||||
ip TEXT PRIMARY KEY, -- ip/cidr as pushed to VyOS (plain ip = /32 semantics)
|
||||
kind TEXT NOT NULL, -- dest|src
|
||||
sigs TEXT NOT NULL DEFAULT '',-- csv of triggering sids
|
||||
first_seen REAL NOT NULL,
|
||||
last_hit REAL NOT NULL,
|
||||
expires REAL NOT NULL,
|
||||
applied INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS hits (
|
||||
sid INTEGER NOT NULL,
|
||||
ip TEXT NOT NULL,
|
||||
ts REAL NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS hits_idx ON hits (sid, ip, ts);
|
||||
CREATE TABLE IF NOT EXISTS eve_ckpt (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
dev INTEGER NOT NULL, -- st_dev/st_ino of the EVE file last read
|
||||
ino INTEGER NOT NULL,
|
||||
pos INTEGER NOT NULL, -- byte offset of the next unread record
|
||||
updated_at REAL NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class BlockRow:
|
||||
ip: str
|
||||
kind: str
|
||||
sigs: str
|
||||
first_seen: float
|
||||
last_hit: float
|
||||
expires: float
|
||||
applied: int
|
||||
|
||||
|
||||
class State:
|
||||
def __init__(self, path: str):
|
||||
self.path = path
|
||||
self._local = threading.local()
|
||||
self.con.executescript(SCHEMA)
|
||||
columns = {r[1] for r in self.con.execute("PRAGMA table_info(hits)")}
|
||||
if "event_key" not in columns:
|
||||
self.con.execute("ALTER TABLE hits ADD COLUMN event_key TEXT")
|
||||
self.con.execute("CREATE UNIQUE INDEX IF NOT EXISTS hits_event ON hits(event_key)")
|
||||
self.con.commit()
|
||||
|
||||
@property
|
||||
def con(self):
|
||||
# Alert processing and router I/O have independent connections.
|
||||
if not hasattr(self._local, "con"):
|
||||
con = sqlite3.connect(self.path, timeout=10)
|
||||
con.execute("PRAGMA journal_mode=WAL")
|
||||
con.execute("PRAGMA busy_timeout=10000")
|
||||
self._local.con = con
|
||||
return self._local.con
|
||||
|
||||
def close(self):
|
||||
if hasattr(self._local, "con"):
|
||||
self._local.con.close()
|
||||
del self._local.con
|
||||
|
||||
# ---- hit windowing (threshold mode) ----
|
||||
def record_hit(self, sid: int, ip: str, ts: float | None = None) -> None:
|
||||
event_key = getattr(self._local, "event_key", None)
|
||||
if event_key is not None:
|
||||
event_key = f'{event_key}:{sid}:{ip}'
|
||||
self.con.execute("INSERT OR IGNORE INTO hits (sid, ip, ts, event_key) VALUES (?,?,?,?)",
|
||||
(sid, ip, ts or time.time(), event_key))
|
||||
self.con.commit()
|
||||
|
||||
def hit_count(self, sid: int, ip: str, window: float, now: float | None = None) -> int:
|
||||
now = now or time.time()
|
||||
(n,) = self.con.execute(
|
||||
"SELECT count(*) FROM hits WHERE sid=? AND ip=? AND ts>=?",
|
||||
(sid, ip, now - window)).fetchone()
|
||||
return n
|
||||
|
||||
def prune_hits(self, max_age: float, now: float | None = None) -> None:
|
||||
now = now or time.time()
|
||||
self.con.execute("DELETE FROM hits WHERE ts < ?", (now - max_age,))
|
||||
self.con.commit()
|
||||
|
||||
# ---- blocks ----
|
||||
def _row(self, r: sqlite3.Row | tuple) -> BlockRow:
|
||||
return BlockRow(*r)
|
||||
|
||||
def get(self, ip: str) -> BlockRow | None:
|
||||
r = self.con.execute("SELECT * FROM blocks WHERE ip=?", (ip,)).fetchone()
|
||||
return self._row(r) if r else None
|
||||
|
||||
def upsert_hit(self, ip: str, kind: str, sid: int, ttl: float,
|
||||
now: float | None = None) -> BlockRow:
|
||||
"""Create block entry or refresh expiry/last_hit of existing one."""
|
||||
now = now or time.time()
|
||||
# Serialize the read/modify/write with expiry cleanup in the worker.
|
||||
with self.con:
|
||||
self.con.execute("BEGIN IMMEDIATE")
|
||||
row = self.get(ip)
|
||||
if row:
|
||||
sigs = row.sigs
|
||||
if str(sid) not in sigs.split(","):
|
||||
sigs = f"{sigs},{sid}" if sigs else str(sid)
|
||||
self.con.execute(
|
||||
"UPDATE blocks SET expires=?, last_hit=?, sigs=? WHERE ip=?",
|
||||
(now + ttl, now, sigs, ip))
|
||||
else:
|
||||
self.con.execute("INSERT INTO blocks VALUES (?,?,?,?,?,?,0)",
|
||||
(ip, kind, str(sid), now, now, now + ttl))
|
||||
result = self.get(ip)
|
||||
return result
|
||||
|
||||
def set_applied(self, ip: str, applied: int = 1) -> None:
|
||||
self.con.execute("UPDATE blocks SET applied=? WHERE ip=?", (applied, ip))
|
||||
self.con.commit()
|
||||
|
||||
def remove(self, ip: str, expired_before: float | None = None) -> bool:
|
||||
# A new alert may refresh a block while the router commit is in flight.
|
||||
with self.con:
|
||||
if expired_before is None:
|
||||
cur = self.con.execute("DELETE FROM blocks WHERE ip=?", (ip,))
|
||||
else:
|
||||
cur = self.con.execute("DELETE FROM blocks WHERE ip=? AND expires<=?",
|
||||
(ip, expired_before))
|
||||
if cur.rowcount:
|
||||
self.con.execute("DELETE FROM hits WHERE ip=?", (ip,))
|
||||
return bool(cur.rowcount)
|
||||
|
||||
def all_blocks(self) -> list[BlockRow]:
|
||||
return [self._row(r) for r in self.con.execute("SELECT * FROM blocks")]
|
||||
|
||||
def expired(self, now: float | None = None) -> list[BlockRow]:
|
||||
now = now or time.time()
|
||||
return [self._row(r) for r in
|
||||
self.con.execute("SELECT * FROM blocks WHERE expires<=?", (now,))]
|
||||
|
||||
def unapplied(self) -> list[BlockRow]:
|
||||
return [self._row(r) for r in
|
||||
self.con.execute("SELECT * FROM blocks WHERE applied=0")]
|
||||
|
||||
def count_active(self, now: float | None = None) -> int:
|
||||
now = now or time.time()
|
||||
(n,) = self.con.execute("SELECT count(*) FROM blocks WHERE expires>?", (now,)).fetchone()
|
||||
return n
|
||||
|
||||
# ---- EVE read checkpoint (survives controller restarts) ----
|
||||
def get_checkpoint(self) -> tuple | None:
|
||||
r = self.con.execute("SELECT dev, ino, pos FROM eve_ckpt WHERE id=1").fetchone()
|
||||
return tuple(r) if r else None
|
||||
|
||||
def save_checkpoint(self, dev: int, ino: int, pos: int, now: float | None = None) -> None:
|
||||
with self.con:
|
||||
self.con.execute(
|
||||
"INSERT INTO eve_ckpt VALUES (1,?,?,?,?) "
|
||||
"ON CONFLICT(id) DO UPDATE SET dev=excluded.dev, ino=excluded.ino, "
|
||||
"pos=excluded.pos, updated_at=excluded.updated_at",
|
||||
(dev, ino, pos, now or time.time()))
|
||||
|
||||
def router_dirty(self):
|
||||
r = self.con.execute("SELECT value FROM controller_meta WHERE key='router_dirty'").fetchone()
|
||||
return bool(r and r[0] == '1')
|
||||
|
||||
def set_router_dirty(self, dirty):
|
||||
with self.con:
|
||||
self.con.execute("INSERT OR REPLACE INTO controller_meta VALUES ('router_dirty', ?)",
|
||||
('1' if dirty else '0',))
|
||||
236
controller/leaf_controller/vyos_client.py
Normal file
236
controller/leaf_controller/vyos_client.py
Normal file
@@ -0,0 +1,236 @@
|
||||
"""Minimal VyOS HTTP API client (Stream 2026.03 / rolling: service https api rest).
|
||||
|
||||
Protocol (docs: automation/vyos-api.html):
|
||||
POST {base}/configure form fields: data=<json>, key=<key>
|
||||
POST {base}/retrieve {"op":"returnValues","path":[tokens]} -> {"returnValues":[...]}
|
||||
POST {base}/show {"op":"show","path":[...]}
|
||||
POST {base}/config-file {"op":"save"}
|
||||
GET {base}/info version/health, no auth
|
||||
Multipart/form-data encoding, exactly what curl -F sends.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.connection import HTTPSConnection
|
||||
from urllib3.connectionpool import HTTPSConnectionPool
|
||||
|
||||
log = logging.getLogger("leaf.vyos")
|
||||
|
||||
|
||||
class VyosApiError(RuntimeError):
|
||||
def __init__(self, msg: str, status: int | None = None, body: Any = None):
|
||||
super().__init__(msg)
|
||||
self.status = status
|
||||
self.body = body
|
||||
|
||||
|
||||
def normalize_pin(pin: str) -> str:
|
||||
"""Accept openssl-style 'AA:BB:..' or plain hex; return lowercase hex digest."""
|
||||
p = pin.strip().lower().removeprefix("sha256:").replace(":", "")
|
||||
if len(p) != 64 or any(c not in "0123456789abcdef" for c in p):
|
||||
raise ValueError("tls_cert_pin must be a sha256 certificate fingerprint (hex)")
|
||||
return p
|
||||
|
||||
|
||||
class PinMismatchError(VyosApiError):
|
||||
pass
|
||||
|
||||
|
||||
class _PinnedHTTPSConnection(HTTPSConnection):
|
||||
"""TLS to an IP-hosted self-signed endpoint, anchored on a pinned cert.
|
||||
|
||||
Hostname verification is impossible for this router's cert (CN=localhost,
|
||||
no IP SAN), so trust comes from comparing the peer's SHA-256 fingerprint
|
||||
against the configured pin. CERT_NONE is safe *because* the pin is the
|
||||
trust anchor: a mismatch aborts the connection before any request.
|
||||
"""
|
||||
leaf_pin: str | None = None
|
||||
|
||||
def connect(self):
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
sock = socket.create_connection((self.host, self.port), self.timeout,
|
||||
self.source_address)
|
||||
self.sock = ctx.wrap_socket(sock, server_hostname=self.host)
|
||||
der = self.sock.getpeercert(binary_form=True)
|
||||
if der is None:
|
||||
raise PinMismatchError("TLS handshake returned no peer certificate")
|
||||
digest = hashlib.sha256(der).hexdigest()
|
||||
if digest != self.leaf_pin:
|
||||
self.sock.close()
|
||||
raise PinMismatchError(
|
||||
f"TLS cert fingerprint mismatch for {self.host}; peer is "
|
||||
f"{digest}, pinned is {self.leaf_pin} (possible MiTM?)")
|
||||
# Pin matched: this connection IS authenticated to the pinned anchor.
|
||||
self.is_verified = True
|
||||
return None
|
||||
|
||||
|
||||
class VyosClient:
|
||||
def __init__(self, url: str, key: str, ca_bundle: str | None = None,
|
||||
tls_verify: bool = True, timeout: float = 10.0, retries: int = 3,
|
||||
cert_pin: str | None = None):
|
||||
self.url = url.rstrip("/")
|
||||
self.key = key
|
||||
self.timeout = timeout
|
||||
self.retries = retries
|
||||
self.sess = requests.Session()
|
||||
# Do not bypass the pinned adapter through an environment HTTP proxy.
|
||||
self.sess.trust_env = False
|
||||
if cert_pin:
|
||||
pin = normalize_pin(cert_pin)
|
||||
|
||||
class PinAdapter(HTTPAdapter):
|
||||
def init_poolmanager(self, *args, **kwargs):
|
||||
super().init_poolmanager(*args, **kwargs)
|
||||
conn_cls = type(
|
||||
"PinnedConn", (_PinnedHTTPSConnection,),
|
||||
{"leaf_pin": pin})
|
||||
self.poolmanager.pool_classes_by_scheme = dict(self.poolmanager.pool_classes_by_scheme)
|
||||
self.poolmanager.pool_classes_by_scheme["https"] = type(
|
||||
"PinnedPool", (HTTPSConnectionPool,),
|
||||
{"ConnectionCls": conn_cls})
|
||||
return self.poolmanager
|
||||
|
||||
self.sess.mount("https://", PinAdapter())
|
||||
elif ca_bundle:
|
||||
self.sess.verify = ca_bundle
|
||||
else:
|
||||
self.sess.verify = bool(tls_verify)
|
||||
if self.sess.verify is False:
|
||||
requests.packages.urllib3.disable_warnings() # explicit opt-in only
|
||||
|
||||
# ------------------------------------------------------------------ core
|
||||
def _post(self, endpoint: str, payload: dict, auth: bool = True) -> dict:
|
||||
# multipart/form-data exactly like `curl -F "data=..." -F "key=..."`:
|
||||
# part tuples (None, value) -> no filename -> plain form field
|
||||
fields = {"data": (None, json.dumps(payload))}
|
||||
if auth:
|
||||
fields["key"] = (None, self.key)
|
||||
resp = self.sess.post(f"{self.url}{endpoint}", files=fields, timeout=self.timeout)
|
||||
try:
|
||||
body = resp.json()
|
||||
except ValueError:
|
||||
body = resp.text
|
||||
if resp.status_code >= 400:
|
||||
raise VyosApiError(f"{endpoint} -> HTTP {resp.status_code}: {body!r}",
|
||||
status=resp.status_code, body=body)
|
||||
# VyOS returns success:false + error for semantic failures (often 200/400)
|
||||
if isinstance(body, dict) and body.get("success") is False:
|
||||
raise VyosApiError(f"{endpoint} rejected: {body.get('error')!r}",
|
||||
status=resp.status_code, body=body)
|
||||
if not isinstance(body, dict) or body.get("success") is not True:
|
||||
raise VyosApiError(f"{endpoint}: missing success acknowledgement")
|
||||
return body
|
||||
|
||||
# ------------------------------------------------------------- public API
|
||||
def info(self) -> dict:
|
||||
resp = self.sess.get(f"{self.url}/info", timeout=self.timeout)
|
||||
if resp.status_code >= 400:
|
||||
raise VyosApiError(f"/info -> HTTP {resp.status_code}", status=resp.status_code)
|
||||
body = resp.json()
|
||||
if isinstance(body, dict) and isinstance(body.get("data"), dict):
|
||||
return body["data"]
|
||||
return body
|
||||
|
||||
def configure(self, ops: list[dict] | dict) -> dict:
|
||||
"""One commit: a single op dict or a list of ops (atomic multi-op)."""
|
||||
payload = ops if isinstance(ops, list) else ops
|
||||
return self._post("/configure", payload)
|
||||
|
||||
def retrieve(self, op: str, path: list[str]) -> dict:
|
||||
return self._post("/retrieve", {"op": op, "path": path})
|
||||
|
||||
def group_members(self, group: str) -> list[str]:
|
||||
body = self.retrieve("returnValues",
|
||||
["firewall", "group", "address-group", group, "address"])
|
||||
# live 2026.03 returns {"success":true,"data":[...]}; older builds used
|
||||
# {"returnValues":[...]}. Accept both, never silently hide members.
|
||||
vals = body.get("data")
|
||||
if vals is None:
|
||||
if "returnValues" not in body and "data" not in body:
|
||||
raise VyosApiError("group member response contains no values field")
|
||||
vals = body.get("returnValues") or []
|
||||
if not isinstance(vals, list):
|
||||
vals = [vals] if vals else []
|
||||
return [str(v) for v in vals]
|
||||
|
||||
@staticmethod
|
||||
def member_op(op, group, ip):
|
||||
import ipaddress
|
||||
ipaddress.IPv4Address(ip) # This prototype manages IPv4 address groups only.
|
||||
return {"op": op, "path": ["firewall", "group", "address-group", group, "address", ip]}
|
||||
|
||||
def set_group_member(self, group: str, ip: str) -> dict:
|
||||
return self.configure(self.member_op("set", group, ip))
|
||||
|
||||
def del_group_member(self, group: str, ip: str) -> dict:
|
||||
return self.configure(self.member_op("delete", group, ip))
|
||||
|
||||
def verify_enforcement(self, group, require_src=False):
|
||||
fw = self.retrieve("showConfig", ["firewall"]).get("data", {})
|
||||
if not isinstance(fw, dict):
|
||||
raise VyosApiError("cannot inspect firewall enforcement hooks")
|
||||
directions = set()
|
||||
def walk(obj):
|
||||
if not isinstance(obj, dict):
|
||||
return
|
||||
if obj.get("action") == "drop":
|
||||
for side in ("source", "destination"):
|
||||
if obj.get(side, {}).get("group", {}).get("address-group") == group:
|
||||
directions.add(side)
|
||||
for value in obj.values():
|
||||
if isinstance(value, dict):
|
||||
walk(value)
|
||||
ipv4 = fw.get("ipv4", {})
|
||||
walk(ipv4.get("forward", {}))
|
||||
attached = set()
|
||||
def attachments(obj):
|
||||
if not isinstance(obj, dict):
|
||||
return
|
||||
hook = obj.get("firewall", {})
|
||||
if isinstance(hook, dict) and isinstance(hook.get("name"), str):
|
||||
attached.add(hook["name"])
|
||||
for value in obj.values():
|
||||
if isinstance(value, dict):
|
||||
attachments(value)
|
||||
attachments(fw.get("zone", {}))
|
||||
for name in attached:
|
||||
walk(ipv4.get("name", {}).get(name, {}))
|
||||
required = {"destination", "source"} if require_src else {"destination"}
|
||||
if not required <= directions:
|
||||
raise VyosApiError("missing drop hooks for " + ", ".join(sorted(required - directions)))
|
||||
|
||||
def show(self, command: list[str]) -> str:
|
||||
body = self._post("/show", {"op": "show", "path": command})
|
||||
out = body.get("output")
|
||||
if out is None:
|
||||
out = body.get("data", "")
|
||||
return str(out)
|
||||
|
||||
def save_config(self) -> dict:
|
||||
return self._post("/config-file", {"op": "save"})
|
||||
|
||||
# ----------------------------------------------------- retrying operations
|
||||
def op_with_retry(self, fn, *args, **kwargs) -> Any:
|
||||
last: Exception | None = None
|
||||
for attempt in range(1, self.retries + 1):
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
except (requests.RequestException, VyosApiError) as exc:
|
||||
last = exc
|
||||
wait = min(2 ** attempt, 30)
|
||||
log.warning("vyos op failed (attempt %d/%d): %s — retry in %ss",
|
||||
attempt, self.retries, exc, wait)
|
||||
time.sleep(wait)
|
||||
raise VyosApiError(f"vyos op failed after {self.retries} attempts: {last}")
|
||||
2
controller/requirements.txt
Normal file
2
controller/requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
requests>=2.31
|
||||
PyYAML>=6.0
|
||||
Reference in New Issue
Block a user