Files
LEAF/tests/test_controller.py
2026-11-15 17:42:55 +01:00

449 lines
21 KiB
Python

"""Offline regression tests. No lab access or router writes."""
import copy
import json
import os
from pathlib import Path
import sys
import tempfile
import threading
import time
import unittest
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'controller'))
from leaf_controller.config import Config, DEFAULTS
from leaf_controller.eve import EveTailer
from leaf_controller.main import Controller
from leaf_controller.vyos_client import (
PinMismatchError, VyosClient, VyosApiError, normalize_pin)
class Router:
member_op = staticmethod(VyosClient.member_op)
def __init__(self, members=()):
self.members = set(members)
self.commits = []
self.fail = False
self.after_commit = None
def op_with_retry(self, fn, *args): return fn(*args)
def group_members(self, group): return sorted(self.members)
def configure(self, ops):
self.commits.append(ops)
for op in ops:
ip = op['path'][-1]
if op['op'] == 'set': self.members.add(ip)
else: self.members.discard(ip)
if self.after_commit: self.after_commit()
if self.fail: raise TimeoutError('commit acknowledged too late')
return {'success': True}
def save_config(self): pass
class Tests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
raw = copy.deepcopy(DEFAULTS)
raw.update(state_db=str(self.root/'state.sqlite'), audit_log=str(self.root/'audit.jsonl'))
raw['policy'].update(mode='immediate', block_src=True)
with patch.object(Config, 'vyos_key', return_value='offline-test'):
self.ctl = Controller(Config(raw), dry_run=False)
self.router = self.ctl.client = Router()
def tearDown(self):
self.ctl.close()
self.tmp.cleanup()
def alert(self, ip='192.0.2.77', action='block', src='203.0.113.11'):
return {'event_type':'alert','src_ip':src,'dest_ip':ip,
'alert':{'signature_id':1000001,'metadata':{'leaf_action':[action]}}}
def test_decision_is_durable_without_api(self):
self.ctl.handle_event(self.alert())
self.assertIsNotNone(self.ctl.state.get('192.0.2.77'))
self.assertFalse(self.router.commits)
def test_batch_dedup_readback_and_expiry(self):
for ip in ('192.0.2.77','192.0.2.78','192.0.2.77'):
self.ctl.handle_event(self.alert(ip))
self.ctl.sync_router_state()
self.assertEqual(len(self.router.commits),1)
self.assertEqual(len(self.router.commits[0]),2)
self.assertTrue(self.ctl.state.get('192.0.2.77').applied)
self.ctl.state.con.execute('UPDATE blocks SET expires=?',(time.time()-1,))
self.ctl.state.con.commit()
self.ctl.sync_router_state()
self.assertEqual(self.router.members,set())
self.assertEqual(self.ctl.state.all_blocks(),[])
def test_expired_startup_not_adopted(self):
self.ctl.state.upsert_hit('192.0.2.77','dest',1,1,now=time.time()-10)
self.router.members.add('192.0.2.77')
self.ctl.sync_router_state()
self.assertIsNone(self.ctl.state.get('192.0.2.77'))
self.assertFalse(self.router.members)
def test_state_owned_group_removes_stale_router_members(self):
self.ctl.cfg.raw['adopt_foreign_members']=False
self.router.members.add('192.0.2.77')
self.ctl.sync_router_state()
self.assertFalse(self.router.members)
self.assertEqual(self.ctl.state.all_blocks(),[])
def test_timeout_verified_before_resubmit(self):
self.ctl.handle_event(self.alert())
self.router.fail=True
with self.assertRaises(TimeoutError): self.ctl.sync_router_state()
self.router.fail=False
self.ctl.sync_router_state()
self.assertEqual(len(self.router.commits),1)
self.assertTrue(self.ctl.state.get('192.0.2.77').applied)
def test_refresh_during_delete_is_retained(self):
self.ctl.state.upsert_hit('192.0.2.77','dest',1,1,now=time.time()-10)
self.router.members.add('192.0.2.77')
self.router.after_commit=lambda: self.ctl.handle_event(self.alert())
self.ctl.sync_router_state()
self.assertGreater(self.ctl.state.get('192.0.2.77').expires,time.time())
self.assertFalse(self.ctl.state.get('192.0.2.77').applied)
self.router.after_commit=None
self.ctl.sync_router_state()
self.assertIn('192.0.2.77',self.router.members)
def test_slow_commit_does_not_block_decisions(self):
self.ctl.handle_event(self.alert())
started, release = threading.Event(), threading.Event()
self.router.after_commit=lambda: (started.set(),release.wait(3))
def worker():
try: self.ctl.sync_router_state()
finally: self.ctl.state.close()
t=threading.Thread(target=worker); t.start()
self.assertTrue(started.wait(2))
t0=time.monotonic()
for n in range(10): self.ctl.handle_event(self.alert(f'192.0.2.{n+100}'))
elapsed=time.monotonic()-t0
release.set(); t.join(3)
self.assertLess(elapsed,1)
self.assertEqual(self.ctl.state.count_active(),11)
def test_floor_src_ipv6_and_bad_events(self):
for ip in ('10.0.249.0','172.19.32.3','192.168.100.83','127.0.0.1','0.0.0.0','255.255.255.255','::1','bogus'):
self.ctl.handle_event(self.alert(ip))
for e in ({'event_type':'alert','alert':None}, {'event_type':'alert','alert':{'signature_id':'bad'}}, {'event_type':'tls'}):
self.ctl.handle_event(e)
self.assertEqual(self.ctl.state.count_active(),0)
self.ctl.handle_event(self.alert(action='block_src'))
self.assertIsNotNone(self.ctl.state.get('203.0.113.11'))
def test_dry_run_cannot_seed_live_blocks(self):
dry = Controller(self.ctl.cfg, dry_run=True)
try:
dry.handle_event(self.alert())
self.assertIsNotNone(dry.state.get('192.0.2.77'))
self.assertIsNone(self.ctl.state.get('192.0.2.77'))
finally: dry.close()
def test_cap(self):
self.ctl.cfg.raw['policy']['max_blocks']=1
self.ctl.handle_event(self.alert())
self.ctl.handle_event(self.alert('192.0.2.78'))
self.assertEqual(self.ctl.state.count_active(),1)
def test_single_instance_and_wal(self):
with self.assertRaises(SystemExit): Controller(self.ctl.cfg,dry_run=False)
self.assertEqual(self.ctl.state.con.execute('pragma journal_mode').fetchone()[0],'wal')
def test_drift_repaired(self):
self.ctl.handle_event(self.alert()); self.ctl.sync_router_state()
self.router.members.clear(); self.ctl.sync_router_state()
self.assertIn('192.0.2.77',self.router.members)
def test_partial_write_rotation_truncation(self):
f=self.root/'eve.json'; f.write_bytes(b'')
tail=EveTailer(str(f),from_start=True); fh=tail._open()
f.write_bytes(b'{"event_type":')
self.assertEqual(tail._read_ready(fh),[])
with f.open('ab') as w: w.write(b'"alert"}\n')
lines=tail._read_ready(fh)
self.assertEqual(tail._parse(lines[0]),{'event_type':'alert'})
f.write_bytes(b'')
self.assertTrue(tail._needs_reopen())
f.rename(self.root/'eve.old'); f.write_text('{}\n')
self.assertTrue(tail._needs_reopen())
tail.close()
def test_rotation_drains_more_than_one_batch(self):
f=self.root/'eve.json'; f.write_text('{}\n'*1100)
tail=EveTailer(str(f),poll_interval=.001,from_start=True)
tail._open()
f.rename(self.root/'eve.old'); f.write_text('{"new":true}\n')
events=tail.events()
received=[next(events) for _ in range(1101)]
self.assertEqual(received[-1],{'new':True})
self.assertEqual(received[:-1],[{}]*1100)
events.close()
def test_checkpoint_resumes_alerts_written_during_shutdown(self):
f=self.root/'eve.json'; f.write_text('')
tail=EveTailer(str(f)); tail._open()
self.ctl.state.save_checkpoint(*tail._ino, tail._pos)
tail.close()
# controller is down; suricata keeps appending
with f.open('ab') as w: w.write(b'{"event_type":"alert","missed":true}\n')
ck=self.ctl.state.get_checkpoint()
resumed=EveTailer(str(f), checkpoint=ck); fh=resumed._open()
self.assertEqual([resumed._parse(l) for l in resumed._read_ready(fh)],
[{'event_type':'alert','missed':True}])
resumed.close()
def test_checkpoint_ignored_on_first_run_never_replays_history(self):
f=self.root/'eve.json'; f.write_text('{"ancient":true}\n')
t=EveTailer(str(f)); fh=t._open()
self.assertEqual(t._read_ready(fh),[])
self.assertIsNone(self.ctl.state.get_checkpoint())
t.close()
def test_checkpoint_across_rotation_reads_new_file_from_start(self):
f=self.root/'eve.json'; f.write_text('{"old":true}\n')
t=EveTailer(str(f)); t._open()
self.ctl.state.save_checkpoint(*t._ino, t._pos); t.close()
f.rename(self.root/'eve.old'); f.write_text('{"fresh":true}\n')
r=EveTailer(str(f), checkpoint=self.ctl.state.get_checkpoint()); fh=r._open()
self.assertEqual([r._parse(l) for l in r._read_ready(fh)],[{'fresh':True}])
r.close()
def test_audit_log_rotation_bounded(self):
from leaf_controller.main import Audit
p=str(self.root/'dec.jsonl')
a=Audit(p, max_bytes=50, keep=2)
for i in range(20): a.write({'i':i,'pad':'x'*80})
a.close()
rotated=list(self.root.glob('dec.jsonl.*'))
self.assertEqual(len(rotated),2) # exactly 'keep' retained
for old in rotated:
self.assertTrue(old.stat().st_size>0)
# every rotated line is a valid decision record
recs=[]
for f in rotated:
with open(f) as fh:
recs += [json.loads(l) for l in fh]
self.assertTrue(all(r['pad']=='x'*80 for r in recs))
def test_new_file_after_startup_is_not_skipped(self):
f=self.root/'missing-eve.json'
tail=EveTailer(str(f))
self.assertIsNone(tail._open())
f.write_text('{"event_type":"alert"}\n')
fh=tail._open()
self.assertEqual(len(tail._read_ready(fh)),1)
tail.close()
def test_idle_reader_waits(self):
f=self.root/'eve.json'; f.touch()
tail=EveTailer(str(f),poll_interval=.05)
events=tail.events(); start=time.monotonic()
self.assertIsNone(next(events))
self.assertGreaterEqual(time.monotonic()-start,.04)
events.close()
def test_packet_headers_and_checksums(self):
import pathlib, sys
from syn_packets import packet, checksum
import socket, struct
frame=packet('203.0.113.12','192.0.2.201',22,0)
ip,tcp=frame[14:34],frame[34:]
self.assertEqual(len(frame),54)
self.assertEqual(struct.unpack('!H',ip[6:8])[0],0)
self.assertEqual(ip[8:10],bytes([64,6]))
self.assertEqual(tcp[13],2)
self.assertEqual(checksum(ip),0)
pseudo=socket.inet_aton('203.0.113.12')+socket.inet_aton('192.0.2.201')+struct.pack('!BBH',0,6,20)
self.assertEqual(checksum(pseudo+tcp),0)
def test_unattached_firewall_hook_rejected(self):
client=VyosClient('http://example.invalid','test')
hook={'action':'drop','source':{'group':{'address-group':'IDS-BLOCK'}}}
fw={'ipv4':{'forward':{'filter':{'rule':{'10':{'action':'drop','destination':{'group':{'address-group':'IDS-BLOCK'}}}}}},'name':{'unused':{'rule':{'9':hook}}}}}
with patch.object(client,'retrieve',return_value={'data':fw}):
with self.assertRaises(VyosApiError): client.verify_enforcement('IDS-BLOCK',True)
fw['zone']={'zone':{'MAOSPF':{'from':{'VEXLAN':{'firewall':{'name':'unused'}}}}}}
with patch.object(client,'retrieve',return_value={'data':fw}): client.verify_enforcement('IDS-BLOCK',True)
def test_show_uses_path(self):
client=VyosClient('http://example.invalid','test')
with patch.object(client,'_post',return_value={'success':True,'data':'counters'}) as post:
self.assertEqual(client.show(['firewall']), 'counters')
post.assert_called_once_with('/show',{'op':'show','path':['firewall']})
def test_ambiguous_api_success_rejected(self):
client=VyosClient('http://example.invalid','test')
response=type('Response',(),{'status_code':200,'json':lambda _: {}})()
with patch.object(client.sess,'post',return_value=response):
with self.assertRaises(VyosApiError): client.configure([])
def test_readback_failure_keeps_pending(self):
self.ctl.handle_event(self.alert())
self.router.group_members=lambda group: []
with self.assertRaises(VyosApiError): self.ctl.sync_router_state()
self.assertFalse(self.ctl.state.get('192.0.2.77').applied)
class MetricsTests(unittest.TestCase):
"""Stats records update metrics without entering the policy path."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name)
raw = copy.deepcopy(DEFAULTS)
raw.update(state_db=str(self.root/'state.sqlite'),
audit_log=str(self.root/'audit.jsonl'))
raw['metrics'].update(capture_interval=0, capture_max_bytes=200,
capture_log_keep=2)
with patch.object(Config, 'vyos_key', return_value='offline-test'):
self.ctl = Controller(Config(raw), dry_run=False)
self.router = self.ctl.client = Router()
def tearDown(self):
self.ctl.close()
self.tmp.cleanup()
def stats(self, packets=1000, dropped=0, engine_alerts=3, uptime=120):
"""Mirrors the live eve.json shape: capture/decoder/detect sub-objects,
no 'ifaces' at totals level (threads: no + totals: yes)."""
return {'event_type': 'stats', 'stats': {
'uptime': uptime, 'memcap_pressure': 0,
'capture': {'kernel_packets': packets, 'kernel_drops': dropped,
'errors': 0},
'decoder': {'pkts': packets, 'bytes': packets * 200, 'invalid': 0},
'detect': {'alert': engine_alerts, 'alerts_suppressed': 0,
'alert_queue_overflow': 0}}}
def thread_stats(self, packets=500):
"""Live thread records (flush=False): per-thread counters, no uptime."""
return {'event_type': 'stats', 'stats': {
'capture': {'kernel_packets': packets, 'kernel_drops': 0,
'errors': 0},
'decoder': {'pkts': packets, 'bytes': packets * 100, 'invalid': 0},
'engine': {'alert': 1, 'alerts_suppressed': 0,
'alert_queue_overflow': 0}}}
def test_stats_never_touch_state_router_or_audit(self):
# Verify that stats cannot produce policy decisions:
# 1) routed stats can only ever be counted
for i in range(20):
self.ctl._route(self.stats(packets=100 * i))
self.ctl._route({'event_type': 'dns', 'src_ip': '192.0.2.77'})
self.ctl._route({'event_type': 'http', 'src_ip': '192.0.2.78'})
# 2) even a stats record handed straight to handle_event cannot act
self.ctl.handle_event(self.stats())
self.assertEqual(self.ctl.state.all_blocks(), [])
self.assertEqual(self.ctl.state.count_active(), 0)
self.assertFalse(self.router.commits)
self.assertFalse(self.router.members)
self.assertEqual((self.root/'audit.jsonl').read_text(), '')
self.assertFalse(list(self.root.glob('capture.log*')))
def test_health_line_once_per_full_window(self):
self.ctl._route(self.stats(packets=1000))
self.ctl._emit_metrics(1000.0) # baseline adoption only
self.ctl._route(self.stats(packets=2000, dropped=1))
with self.assertLogs('leaf.metrics', level='INFO') as logs:
self.ctl._emit_metrics(1030.0) # a full window: one line
self.ctl._route(self.stats(packets=2100))
self.ctl._emit_metrics(1040.0) # partial window: stays silent
health = [l for l in logs.output if 'health:' in l]
self.assertEqual(len(health), 1)
self.assertIn('cps=33.3', health[0]) # 1000 pkts / 30 s window
self.assertIn('dropped+=1', health[0])
def test_counter_restart_never_goes_negative(self):
self.ctl._route(self.stats(packets=5000))
self.ctl._emit_metrics(1000.0)
self.ctl._route(self.stats(packets=100, uptime=30)) # suricata restart
rep = self.ctl.metrics.report(1030.0)
self.assertTrue(rep['restart'])
self.assertEqual(rep['delta']['packets'], 100)
self.assertGreaterEqual(rep['cps'], 0)
def test_totals_authoritative_over_thread_records(self):
# live emits alternating thread(flush=False)+totals; mixing the two
# counter sets would produce bogus rates
self.ctl._route(self.thread_stats(packets=500)) # only kind seen: adopted
self.ctl._emit_metrics(1000.0) # baseline
self.ctl._route(self.stats(packets=2000)) # totals authoritative
self.ctl._route(self.thread_stats(packets=100)) # must not override
rep = self.ctl.metrics.report(1030.0)
self.assertFalse(rep['restart'])
self.assertEqual(rep['delta']['packets'], 1500)
def test_capture_log_bounded_by_size(self):
t = 10000.0
for i in range(1, 40):
self.ctl._route(self.stats(packets=1000 * i, dropped=i))
self.ctl._emit_metrics(t)
t += 40
current = self.root/'capture.log'
rotated = sorted(self.root.glob('capture.log.*'))
# bounded growth is the point: current + at most keep rotated files.
# Audit rotates write-then-check, so the newest line may itself sit in
# the most recent rotated file; current can legitimately be empty.
self.assertLessEqual(len(rotated), 2)
self.assertTrue(current.exists())
recs = []
for p in [current] + rotated:
for l in p.read_text().splitlines():
rec = json.loads(l)
self.assertIn('dropped', rec)
self.assertIn('delta', rec)
recs.append(rec)
assert recs
# Each retained line is a complete window snapshot.
# This check covers the size bound; retention order is tested separately.
self.assertTrue(any(r['dropped'] > 0 for r in recs))
# audit rotation is: append, size-check, rotate, prune-to-keep — so an
# over-cap write can land on disk briefly before the prune settles:
# worst settled state is current (~cap) + keep rotated (~cap each)
disk = sum(x.stat().st_size for x in [current] + rotated)
self.assertLess(disk, 200 * 4 + 400) # (keep+2) x cap + slop
class TLSPinTests(unittest.TestCase):
"""Cert-pin trust against a local TLS server with a throwaway cert."""
@classmethod
def setUpClass(cls):
import shutil, ssl, subprocess
from http.server import BaseHTTPRequestHandler, HTTPServer
if shutil.which('openssl') is None:
raise unittest.SkipTest('openssl required for TLS pin tests')
cls.tmp = tempfile.TemporaryDirectory()
key, crt = cls.tmp.name + '/k.pem', cls.tmp.name + '/c.pem'
subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes',
'-keyout', key, '-out', crt, '-days', '1',
'-subj', '/CN=localhost'],
check=True, capture_output=True)
out = subprocess.run(['openssl', 'x509', '-in', crt, '-noout',
'-fingerprint', '-sha256'],
check=True, capture_output=True, text=True).stdout
cls.pin = out.split('=')[1].strip()
class H(BaseHTTPRequestHandler):
def do_GET(self):
body = b'{"success": true, "data": {"proto": "pin-test"}}'
self.send_response(200)
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, *a):
pass
cls.srv = HTTPServer(('127.0.0.1', 0), H)
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
ctx.load_cert_chain(crt, key)
cls.srv.socket = ctx.wrap_socket(cls.srv.socket, server_side=True)
cls.port = cls.srv.socket.getsockname()[1]
threading.Thread(target=cls.srv.serve_forever, daemon=True).start()
cls.url = f'https://127.0.0.1:{cls.port}'
@classmethod
def tearDownClass(cls):
cls.srv.shutdown(); cls.srv.server_close(); cls.tmp.cleanup()
def test_pin_normalizes_colon_hex(self):
self.assertEqual(normalize_pin(self.pin), self.pin.lower().replace(':', ''))
self.assertEqual(normalize_pin('sha256:' + self.pin.upper()),
self.pin.lower().replace(':', ''))
with self.assertRaises(ValueError):
normalize_pin('not-a-pin')
def test_pinned_client_connects(self):
c = VyosClient(self.url, 'k', cert_pin=self.pin, timeout=10, retries=1)
self.assertEqual(c.info().get('proto'), 'pin-test')
def test_wrong_pin_aborts_before_request(self):
wrong = 'ab' * 32
c = VyosClient(self.url, 'k', cert_pin=wrong, timeout=10, retries=1)
with self.assertRaises(PinMismatchError):
c.info()
if __name__=='__main__': unittest.main()