171 lines
5.8 KiB
Python
171 lines
5.8 KiB
Python
|
|
"""Bounded AF_UNIX EVE ingress. Accepted queues are volatile until policy persists them.
|
||
|
|
|
||
|
|
Suricata can also lose records before delivery on EAGAIN. Local counters do
|
||
|
|
not prove end-to-end losslessness; this is a lab control plane, not a journal.
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
import fcntl
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import queue
|
||
|
|
import socket
|
||
|
|
import stat
|
||
|
|
import threading
|
||
|
|
|
||
|
|
log = logging.getLogger('leaf.sock')
|
||
|
|
MAX_RECORD = 1 << 20
|
||
|
|
|
||
|
|
|
||
|
|
class AlertSocket:
|
||
|
|
def __init__(self, path, queue_size=1000, rcvbuf=1 << 20):
|
||
|
|
if int(queue_size) <= 0:
|
||
|
|
raise ValueError('alert_socket.queue_size must be positive')
|
||
|
|
self.path = path
|
||
|
|
self.q = queue.Queue(maxsize=int(queue_size))
|
||
|
|
self.counts = dict(connections=0, records=0, parse_errors=0, queue_drops=0, other=0)
|
||
|
|
self.available = threading.Event()
|
||
|
|
self._rcvbuf = int(rcvbuf)
|
||
|
|
self._sock = self._conn = self._thread = self._lock = self._identity = None
|
||
|
|
self._stop = threading.Event()
|
||
|
|
|
||
|
|
def bind(self):
|
||
|
|
os.makedirs(os.path.dirname(os.path.abspath(self.path)), exist_ok=True)
|
||
|
|
self._lock = open(self.path + '.lock', 'a')
|
||
|
|
try:
|
||
|
|
fcntl.flock(self._lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||
|
|
if os.path.lexists(self.path):
|
||
|
|
st = os.lstat(self.path)
|
||
|
|
if not stat.S_ISSOCK(st.st_mode):
|
||
|
|
raise OSError('refusing to replace a non-socket path')
|
||
|
|
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as probe:
|
||
|
|
probe.settimeout(0.2)
|
||
|
|
try:
|
||
|
|
probe.connect(self.path)
|
||
|
|
except (ConnectionRefusedError, FileNotFoundError):
|
||
|
|
os.unlink(self.path)
|
||
|
|
else:
|
||
|
|
raise OSError('another listener owns this socket')
|
||
|
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||
|
|
self._sock = sock
|
||
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self._rcvbuf)
|
||
|
|
sock.settimeout(0.2)
|
||
|
|
sock.bind(self.path)
|
||
|
|
st = os.lstat(self.path)
|
||
|
|
self._identity = (st.st_dev, st.st_ino)
|
||
|
|
os.chmod(self.path, 0o660)
|
||
|
|
sock.listen(1)
|
||
|
|
except BaseException:
|
||
|
|
self.close()
|
||
|
|
raise
|
||
|
|
|
||
|
|
def start(self):
|
||
|
|
self._thread = threading.Thread(target=self._accept_loop, name='alert-sock', daemon=True)
|
||
|
|
self._thread.start()
|
||
|
|
|
||
|
|
def _accept_loop(self):
|
||
|
|
while not self._stop.is_set():
|
||
|
|
try:
|
||
|
|
conn, _ = self._sock.accept()
|
||
|
|
except socket.timeout:
|
||
|
|
continue
|
||
|
|
except OSError:
|
||
|
|
break
|
||
|
|
self._conn = conn
|
||
|
|
self.counts['connections'] += 1
|
||
|
|
try:
|
||
|
|
conn.settimeout(0.2)
|
||
|
|
conn.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self._rcvbuf)
|
||
|
|
self._read_conn(conn)
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
finally:
|
||
|
|
conn.close()
|
||
|
|
self._conn = None
|
||
|
|
|
||
|
|
def _read_conn(self, conn):
|
||
|
|
pending = bytearray()
|
||
|
|
oversized = False
|
||
|
|
while not self._stop.is_set():
|
||
|
|
try:
|
||
|
|
data = conn.recv(65536)
|
||
|
|
except socket.timeout:
|
||
|
|
continue
|
||
|
|
if not data:
|
||
|
|
if pending or oversized:
|
||
|
|
self.counts['parse_errors'] += 1
|
||
|
|
return
|
||
|
|
for part in data.splitlines(keepends=True):
|
||
|
|
complete = part.endswith(b'\n')
|
||
|
|
if not oversized:
|
||
|
|
pending.extend(part)
|
||
|
|
if len(pending) > MAX_RECORD:
|
||
|
|
pending.clear()
|
||
|
|
oversized = True
|
||
|
|
if complete:
|
||
|
|
if oversized:
|
||
|
|
self.counts['parse_errors'] += 1
|
||
|
|
else:
|
||
|
|
self._enqueue(pending)
|
||
|
|
pending.clear()
|
||
|
|
oversized = False
|
||
|
|
|
||
|
|
def _enqueue(self, line):
|
||
|
|
if len(line) > MAX_RECORD:
|
||
|
|
self.counts['parse_errors'] += 1
|
||
|
|
return
|
||
|
|
try:
|
||
|
|
rec = json.loads(line)
|
||
|
|
except (ValueError, RecursionError, UnicodeError):
|
||
|
|
self.counts['parse_errors'] += 1
|
||
|
|
return
|
||
|
|
if not isinstance(rec, dict) or rec.get('event_type') != 'alert':
|
||
|
|
self.counts['other'] += 1
|
||
|
|
return
|
||
|
|
try:
|
||
|
|
self.q.put_nowait(rec)
|
||
|
|
self.counts['records'] += 1
|
||
|
|
self.available.set()
|
||
|
|
except queue.Full:
|
||
|
|
self.counts['queue_drops'] += 1
|
||
|
|
log.error('alert queue full: dropped %d', self.counts['queue_drops'])
|
||
|
|
|
||
|
|
def drain(self, limit=256):
|
||
|
|
self.available.clear()
|
||
|
|
out = []
|
||
|
|
while len(out) < limit:
|
||
|
|
try:
|
||
|
|
out.append(self.q.get_nowait())
|
||
|
|
except queue.Empty:
|
||
|
|
break
|
||
|
|
if not self.q.empty():
|
||
|
|
self.available.set()
|
||
|
|
return out
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
self._stop.set()
|
||
|
|
if self._conn:
|
||
|
|
try:
|
||
|
|
self._conn.shutdown(socket.SHUT_RDWR)
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
if self._sock:
|
||
|
|
self._sock.close()
|
||
|
|
if self._thread:
|
||
|
|
self._thread.join(timeout=2)
|
||
|
|
if self._thread.is_alive():
|
||
|
|
raise RuntimeError('socket reader did not stop')
|
||
|
|
self._thread = None
|
||
|
|
self._sock = None
|
||
|
|
if self._identity:
|
||
|
|
try:
|
||
|
|
st = os.lstat(self.path)
|
||
|
|
if (st.st_dev, st.st_ino) == self._identity:
|
||
|
|
os.unlink(self.path)
|
||
|
|
except FileNotFoundError:
|
||
|
|
pass
|
||
|
|
self._identity = None
|
||
|
|
if self._lock:
|
||
|
|
self._lock.close()
|
||
|
|
self._lock = None
|