118 lines
7.1 KiB
Python
118 lines
7.1 KiB
Python
|
|
#!/usr/bin/env python3
|
||
|
|
"""Bounded lab proof: real packets, fresh alerts, router drop counters, TTL cleanup.
|
||
|
|
Run on workstation: python3 tests/live_e2e.py --output /path/to/evidence.json
|
||
|
|
Requires deployed LEAF services. Temporarily uses TTL=180; restores config on exit.
|
||
|
|
"""
|
||
|
|
import argparse,json,subprocess,time
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
def main():
|
||
|
|
ap=argparse.ArgumentParser(description=__doc__)
|
||
|
|
ap.add_argument('--sensor',default='yuru@192.168.100.84') # lease drifts; verify by hostname LEAF-PROTO
|
||
|
|
ap.add_argument('--lxc',default='root@172.19.32.3')
|
||
|
|
ap.add_argument('--second-lxc',default='root@172.19.32.5')
|
||
|
|
ap.add_argument('--ssh-key',default=str(Path.home()/'.ssh/id_ed25519_alice'))
|
||
|
|
ap.add_argument('--output',type=Path,required=True)
|
||
|
|
a=ap.parse_args(); root=Path(__file__).resolve().parents[1]
|
||
|
|
ssh=['ssh','-F','/dev/null','-i',a.ssh_key,'-o','BatchMode=yes','-o','ConnectTimeout=8']
|
||
|
|
scp=['scp','-F','/dev/null','-i',a.ssh_key,'-o','BatchMode=yes']
|
||
|
|
def run(host,cmd):
|
||
|
|
return subprocess.run(ssh+[host,cmd],capture_output=True,text=True,check=True,timeout=150).stdout
|
||
|
|
def status(since=0.0):
|
||
|
|
return json.loads(run(a.sensor,f'sudo -n python3 /tmp/leaf-live-helper.py status --since {since}'))
|
||
|
|
def syn(n):
|
||
|
|
print(run(a.lxc,f'python3 /tmp/leaf-send-syn.py 203.0.113.12 192.0.2.201 22 {n}').strip(),flush=True)
|
||
|
|
def ping(n):
|
||
|
|
# Different source hosts guarantee distinct Suricata ICMP flows.
|
||
|
|
for i in range(n):
|
||
|
|
host=a.lxc if i%2==0 else a.second_lxc
|
||
|
|
run(host,'ping -c 1 -W 1 192.0.2.202 >/dev/null 2>&1 || test $? -eq 1')
|
||
|
|
evidence={'started':time.time(),'passed':False}
|
||
|
|
changed=False
|
||
|
|
try:
|
||
|
|
subprocess.run(scp+[str(root/'tests/live_helper.py'),a.sensor+':/tmp/leaf-live-helper.py'],check=True)
|
||
|
|
subprocess.run(scp+[str(root/'tests/syn_packets.py'),a.lxc+':/tmp/leaf-send-syn.py'],check=True)
|
||
|
|
baseline=status(); evidence['baseline']=baseline
|
||
|
|
targets={'203.0.113.12','192.0.2.202'}
|
||
|
|
if baseline['members'] or baseline['blocks']: raise RuntimeError('live proof requires an idle empty block group/state for counter attribution')
|
||
|
|
run(a.sensor,'sudo -n python3 /tmp/leaf-live-helper.py short-ttl')
|
||
|
|
changed=True
|
||
|
|
run(a.sensor,'sudo -n systemctl restart leaf-controller')
|
||
|
|
t0=time.time();evidence['traffic_started']=t0
|
||
|
|
capture=subprocess.Popen(ssh+[a.sensor,f"sudo -n timeout 15 tcpdump -ni eth1 -c 24 -w /tmp/leaf-live-wave-{int(t0)}.pcap 'tcp and src host 203.0.113.12'"],stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
||
|
|
time.sleep(1);syn(24);ping(2)
|
||
|
|
out,err=capture.communicate(timeout=20)
|
||
|
|
evidence['capture']={'exit':capture.returncode,'summary':err.strip()}
|
||
|
|
if capture.returncode!=0 or '24 packets captured' not in err: raise RuntimeError('capture did not prove 24 packets')
|
||
|
|
deadline=time.time()+180
|
||
|
|
while time.time()<deadline:
|
||
|
|
current=status(t0)
|
||
|
|
applied={row['ip'] for row in current['blocks'] if row['applied']}
|
||
|
|
verified={row['ip'] for row in current['audit'] if row.get('action')=='block' and row.get('verified')}
|
||
|
|
if targets <= set(current['members']) & applied & verified: break
|
||
|
|
print('waiting for router commits',flush=True);time.sleep(3)
|
||
|
|
else: raise RuntimeError('both blocks were not verified')
|
||
|
|
evidence['blocked']=current
|
||
|
|
# Detection evidence comes from the controller's own audit trail
|
||
|
|
# (decisions.jsonl, surfaced by the helper as audit rows), NOT from
|
||
|
|
# /var/log/suricata-leaf/eve.json: since the unix_stream migration,
|
||
|
|
# the eve FILE only carries stats; alerts flow over the socket and
|
||
|
|
# land as 'decision' rows with sid. Both sids must be represented.
|
||
|
|
ids=[r['sid'] for r in current['audit'] if r.get('action')=='decision']
|
||
|
|
if not {1000030,1000001} <= set(ids):
|
||
|
|
# 'decision' rows are written the moment the alert lands on the
|
||
|
|
# socket; the block row lands up to ~60 s later (VyOS commit). If
|
||
|
|
# the reads disagree, re-poll briefly before declaring failure.
|
||
|
|
grace=time.time()+30
|
||
|
|
while not {1000030,1000001} <= set(ids):
|
||
|
|
if time.time()>=grace: raise RuntimeError('fresh detection evidence missing')
|
||
|
|
time.sleep(5)
|
||
|
|
current=status(t0); evidence['blocked']=current
|
||
|
|
ids=[r['sid'] for r in current['audit'] if r.get('action')=='decision']
|
||
|
|
print('Both blocks verified; testing real drop counters',flush=True)
|
||
|
|
before=current['drop_counters'];syn(3);ping(1)
|
||
|
|
current=status(t0); evidence['enforced']=current
|
||
|
|
for rule in ('8','9'):
|
||
|
|
if current['drop_counters'].get(rule,0)<=before.get(rule,0): raise RuntimeError('drop counter did not increment for rule '+rule)
|
||
|
|
print('Source and destination drop counters increased; waiting for TTL cleanup',flush=True)
|
||
|
|
deadline=time.time()+300
|
||
|
|
while time.time()<deadline:
|
||
|
|
current=status(t0)
|
||
|
|
tracked={row['ip'] for row in current['blocks']}
|
||
|
|
if not targets & (set(current['members']) | tracked): break
|
||
|
|
time.sleep(5)
|
||
|
|
else: raise RuntimeError('TTL cleanup did not remove test members')
|
||
|
|
evidence['expired']=current
|
||
|
|
# With no remaining IDS-BLOCK members, post-expiry probes must not
|
||
|
|
# increase either IDS drop counter.
|
||
|
|
before=current['drop_counters'].get('8',0); before9=current['drop_counters'].get('9',0)
|
||
|
|
syn(3);ping(1)
|
||
|
|
time.sleep(2)
|
||
|
|
current=status(t0);evidence['after_expiry_probes']=current
|
||
|
|
if current['drop_counters'].get('8',0)>before or current['drop_counters'].get('9',0)>before9:
|
||
|
|
raise RuntimeError('test drop counters still increment after expiry')
|
||
|
|
unblocked={e['ip'] for e in current['audit'] if e.get('action')=='unblock' and e.get('verified')}
|
||
|
|
if not targets<=unblocked: raise RuntimeError('verified unblock audit missing')
|
||
|
|
final=status();evidence['final']=final
|
||
|
|
if final['protected_hashes']!=baseline['protected_hashes']: raise RuntimeError('protected router configuration changed')
|
||
|
|
evidence['passed']=True
|
||
|
|
print('LIVE E2E PASS: capture -> alert -> durable decisions -> verified router blocks -> drop counters -> TTL cleanup',flush=True)
|
||
|
|
finally:
|
||
|
|
try:
|
||
|
|
if changed:
|
||
|
|
try:
|
||
|
|
if not evidence['passed']:
|
||
|
|
run(a.sensor,f'sudo -n python3 /tmp/leaf-live-helper.py cleanup --since {evidence.get("traffic_started",evidence["started"])}')
|
||
|
|
finally:
|
||
|
|
print(run(a.sensor,'sudo -n python3 /tmp/leaf-live-helper.py restore && sudo -n systemctl restart leaf-controller').strip(),flush=True)
|
||
|
|
except Exception as exc:
|
||
|
|
evidence['restoration_error']=str(exc)
|
||
|
|
evidence['passed']=False
|
||
|
|
raise
|
||
|
|
finally:
|
||
|
|
evidence['finished']=time.time()
|
||
|
|
a.output.parent.mkdir(parents=True,exist_ok=True)
|
||
|
|
a.output.write_text(json.dumps(evidence,indent=2)+'\n')
|
||
|
|
|
||
|
|
if __name__=='__main__':main()
|