107 lines
3.7 KiB
Python
107 lines
3.7 KiB
Python
|
|
"""Read complete EVE records, preserving partial writes and handling rotation."""
|
||
|
|
from __future__ import annotations
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import time
|
||
|
|
log = logging.getLogger("leaf.eve")
|
||
|
|
|
||
|
|
class EveTailer:
|
||
|
|
def __init__(self, path, poll_interval=0.2, from_start=False, checkpoint=None, idle_wait=None):
|
||
|
|
self.path, self.poll, self.from_start = path, poll_interval, from_start
|
||
|
|
# (dev, ino, pos) persisted while the controller was running; if it
|
||
|
|
# matches the file on disk we resume there instead of skipping ahead.
|
||
|
|
self.checkpoint = checkpoint
|
||
|
|
self.idle_wait = idle_wait or time.sleep
|
||
|
|
self._fh = None
|
||
|
|
self._ino = None
|
||
|
|
self._pos = 0
|
||
|
|
|
||
|
|
def _open(self):
|
||
|
|
try:
|
||
|
|
fh = open(self.path, "rb")
|
||
|
|
except FileNotFoundError:
|
||
|
|
# A file created after startup must be read from its beginning.
|
||
|
|
self.from_start = True
|
||
|
|
return None
|
||
|
|
st = os.fstat(fh.fileno())
|
||
|
|
self._ino = (st.st_dev, st.st_ino)
|
||
|
|
if not self.from_start:
|
||
|
|
ck = self.checkpoint
|
||
|
|
if ck and ck[:2] != self._ino:
|
||
|
|
log.warning("eve checkpoint inode mismatch on %s; processing "
|
||
|
|
"current file from start", self.path)
|
||
|
|
fh.seek(0)
|
||
|
|
elif ck and ck[2] <= st.st_size:
|
||
|
|
fh.seek(ck[2])
|
||
|
|
elif ck:
|
||
|
|
log.warning("eve checkpoint pos %d beyond size %d; rereading %s from 0",
|
||
|
|
ck[2], st.st_size, self.path)
|
||
|
|
fh.seek(0)
|
||
|
|
else:
|
||
|
|
# First run ever: never replay pre-existing history.
|
||
|
|
fh.seek(0, os.SEEK_END)
|
||
|
|
self.from_start = True
|
||
|
|
self._fh, self._pos = fh, fh.tell()
|
||
|
|
log.info("opened eve file %s (pos=%d)", self.path, self._pos)
|
||
|
|
return fh
|
||
|
|
|
||
|
|
def _needs_reopen(self):
|
||
|
|
try:
|
||
|
|
st = os.stat(self.path)
|
||
|
|
except FileNotFoundError:
|
||
|
|
return True
|
||
|
|
return (st.st_dev, st.st_ino) != self._ino or st.st_size < (self._fh.tell() if self._fh else self._pos)
|
||
|
|
|
||
|
|
def _read_ready(self, fh):
|
||
|
|
# Never consume a partial record: the writer may append its remainder.
|
||
|
|
out = []
|
||
|
|
for _ in range(1000):
|
||
|
|
pos = fh.tell()
|
||
|
|
line = fh.readline()
|
||
|
|
if not line.endswith(b"\n"):
|
||
|
|
fh.seek(pos)
|
||
|
|
break
|
||
|
|
out.append((line.decode("utf-8", errors="replace"), fh.tell()))
|
||
|
|
return out
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _parse(line):
|
||
|
|
if isinstance(line, tuple):
|
||
|
|
line = line[0]
|
||
|
|
try:
|
||
|
|
event = json.loads(line)
|
||
|
|
return event if isinstance(event, dict) else None
|
||
|
|
except (ValueError, RecursionError):
|
||
|
|
log.warning("skipping invalid EVE JSON record")
|
||
|
|
return None
|
||
|
|
|
||
|
|
def events(self):
|
||
|
|
try:
|
||
|
|
while True:
|
||
|
|
fh = self._fh or self._open()
|
||
|
|
if fh is None:
|
||
|
|
self.idle_wait(self.poll)
|
||
|
|
yield None
|
||
|
|
continue
|
||
|
|
lines = self._read_ready(fh)
|
||
|
|
for line, end in lines:
|
||
|
|
self._pos = end
|
||
|
|
parsed = self._parse(line)
|
||
|
|
if parsed is not None:
|
||
|
|
yield parsed
|
||
|
|
if self._needs_reopen() and not lines:
|
||
|
|
fh.close()
|
||
|
|
self._fh = None
|
||
|
|
continue
|
||
|
|
if not lines:
|
||
|
|
self.idle_wait(self.poll)
|
||
|
|
yield None
|
||
|
|
finally:
|
||
|
|
self.close()
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
if self._fh:
|
||
|
|
self._fh.close()
|
||
|
|
self._fh = None
|