diff --git a/.gitignore b/.gitignore index e5baa77..31d308a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ venv __pycache__ -IP2Location \ No newline at end of file +IP2Location +flows +old \ No newline at end of file diff --git a/INFLUXDB.py.bck b/INFLUXDB.py.bck deleted file mode 100644 index 1ca1c16..0000000 --- a/INFLUXDB.py.bck +++ /dev/null @@ -1,104 +0,0 @@ -import netflow, socket, json, time, os, influxdb_client, ipaddress -from influxdb_client import InfluxDBClient, Point, WritePrecision -from influxdb_client.client.write_api import SYNCHRONOUS -from datetime import timedelta - -# Netentry preconf -WHAT_THE_NETFLOW_PORT = 2055 -WHAT_THE_NETFLOW_IP = "0.0.0.0" - - -# INFLUXDB config - -token = "L56MZGgLNOPcL_nfc4bYujQHPVB0xNGMEHXJj8CEWrE3lioi3PN4eH52NhnEyKdFTfwW1Tlqhl35PAGSva5qXw==" -bucket = "NETFLOW-STAGING" -org = "staging" -url = "http://localhost:8086" -measurement = "testNetFlowPython" -LOCATION_TAG = "YUKIKAZE" -INFLX_SEPARATE_POINTS = 0.1 - -# Initialize InfluxDB client -inflxdb_client = influxdb_client.InfluxDBClient(url=url, token=token, org=org) - -# Other preconf -bigDict = {} - -# Get netentry data ig? -sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -sock.bind((WHAT_THE_NETFLOW_IP, WHAT_THE_NETFLOW_PORT)) -payload, client = sock.recvfrom(4096) # experimental, tested with 1464 bytes -p = netflow.parse_packet(payload) # Test result: -#print(p.entrys) # Test result: 5 - - -for i, entry in enumerate(p.flows, 1): - # prep dict - #tmpEntry = str(entry) - #tmpEntry = tmpEntry[22:-1] - #tmpEntry2 = tmpEntry.replace("'", '"') - - #print(tmpEntry2) - #print(entry - #exit() - #dictEntry = json.loads(tmpEntry2) - #bigDict[i] = (dictEntry) - - - # take data out from netentry - inEntry = entry.data - - # Convert IPs and time duration - # IPs - inEntry["IPV4_SRC_ADDR"] = str(ipaddress.IPv4Address(inEntry["IPV4_SRC_ADDR"])) - inEntry["IPV4_DST_ADDR"] = str(ipaddress.IPv4Address(inEntry["IPV4_DST_ADDR"])) - - # Convert time from ms to HH:MM:SS - first = int(inEntry["FIRST_SWITCHED"]) - last = int(inEntry["LAST_SWITCHED"]) - - inEntry["FIRST_SWITCHED_HR"] = str(timedelta(milliseconds=first)) - inEntry["LAST_SWITCHED_HR"] = str(timedelta(milliseconds=last)) - - - # Prep InfluxDB data - inflxdb_Data_To_Send = ( - influxdb_client.Point(f"{measurement}-script") - .tag("LOCATION", LOCATION_TAG) - .field("dstAddr", inEntry["IPV4_DST_ADDR"]) - .field("srcAddr", inEntry["IPV4_SRC_ADDR"]) - .field("nextHop", inEntry["NEXT_HOP"]) - .field("inptInt", inEntry["INPUT"]) - .field("outptInt", inEntry["OUTPUT"]) - .field("inPackt", inEntry["IN_PACKETS"]) - .field("outPakt", inEntry["IN_OCTETS"]) - .field("frstSwtchd", inEntry["FIRST_SWITCHED"]) - .field("lstSwtchd", inEntry["LAST_SWITCHED"]) - .field("srcPort", inEntry["SRC_PORT"]) - .field("dstPort", inEntry["DST_PORT"]) - .field("tcpFlags", inEntry["TCP_FLAGS"]) - .field("proto", inEntry["PROTO"]) - .field("tos", inEntry["TOS"]) - .field("srcAS", inEntry["SRC_AS"]) - .field("dstAS", inEntry["DST_AS"]) - .field("srcMask", inEntry["SRC_MASK"]) - .field("dstMask", inEntry["DST_MASK"]) - ) - - # idk - write_api = inflxdb_client.write_api(write_options=SYNCHRONOUS) - - # Send data to InfluxDB - write_api.write(bucket=bucket, org="staging", record=inflxdb_Data_To_Send) - time.sleep(INFLX_SEPARATE_POINTS) # separate points by 1 second - - #i+=1 - #type(tmpEntry) - #print(dictEntry) - #print(tmpEntry.lstrip(20)) - - print("----------------") - bigDict[i] = (inEntry) - -#print(bigDict) -print(f"{len(bigDict)} <--- This many entrys") diff --git a/Main.py b/Main.py deleted file mode 100644 index 713abe8..0000000 --- a/Main.py +++ /dev/null @@ -1,17 +0,0 @@ -from scapy.all import * - -while True: - netflow = NetflowHeader()/NetflowHeaderV5(count=1)/NetflowRecordV5(dst="172.20.240.2") - pkt = Ether()/IP()/UDP()/netflow - - print(pkt) - - UDP.payload_guess = [({}, NetflowHeader)] - pkts = sniff(iface="VIP-NET") - - print(netflow) - - - -print(netflow) -print(pkts) diff --git a/INFLUXDB.py b/Python/INFLUXDB.py similarity index 100% rename from INFLUXDB.py rename to Python/INFLUXDB.py diff --git a/INFLUXDBmthrd.py b/Python/INFLUXDBmthrd.py similarity index 100% rename from INFLUXDBmthrd.py rename to Python/INFLUXDBmthrd.py diff --git a/IP2Loc.py b/Python/IP2Loc.py similarity index 100% rename from IP2Loc.py rename to Python/IP2Loc.py diff --git a/proto.py b/Python/proto.py similarity index 100% rename from proto.py rename to Python/proto.py diff --git a/FindGRE.py b/Python/test/FindGRE.py similarity index 100% rename from FindGRE.py rename to Python/test/FindGRE.py diff --git a/mariaDB.py b/Python/test/mariaDB.py similarity index 100% rename from mariaDB.py rename to Python/test/mariaDB.py diff --git a/test.py b/Python/test/test.py similarity index 100% rename from test.py rename to Python/test/test.py diff --git a/test5.py b/Python/test/test5.py similarity index 100% rename from test5.py rename to Python/test/test5.py diff --git a/systemdHowTo/whatDomain.py b/Python/whatDomain.py similarity index 100% rename from systemdHowTo/whatDomain.py rename to Python/whatDomain.py diff --git a/README.md b/README.md index c6b0a44..66fdc97 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,20 @@ -# NetFlow collector and InfluxDB and Grafana +# NetFlowV5 collector and InfluxDB and Grafana + +It works. + +## Python script +Install required modules with +``` +pip install -r requirements.txt +``` + +Then change InfluxDB variables. + +### INFLUXDB.py vs INFLUXDBmthrd.py +First one runs on one thread and should work when there isn't much data on the network. + +Second is when there are a ton of flows that need to be collected. More flows aka more data. + +## sysctl.d +Place it in /etc/sysctl.d/ and apply with ```sysctl -p``` -It works. \ No newline at end of file diff --git a/flow.example b/flow.example deleted file mode 100644 index 88f42fc..0000000 --- a/flow.example +++ /dev/null @@ -1 +0,0 @@ -{1: {'IPV4_SRC_ADDR': '192.168.20.10', 'IPV4_DST_ADDR': '192.168.1.210', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 11, 'IN_PACKETS': 1, 'IN_OCTETS': 288, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 161, 'DST_PORT': 45414, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 28, 'DST_MASK': 24, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 2: {'IPV4_SRC_ADDR': '192.168.20.10', 'IPV4_DST_ADDR': '192.168.1.210', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 11, 'IN_PACKETS': 1, 'IN_OCTETS': 248, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 161, 'DST_PORT': 47003, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 28, 'DST_MASK': 24, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 3: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 71, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 40151, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 4: {'IPV4_SRC_ADDR': '172.19.25.8', 'IPV4_DST_ADDR': '192.168.1.213', 'NEXT_HOP': '83.172.67.1', 'INPUT': 38, 'OUTPUT': 11, 'IN_PACKETS': 2, 'IN_OCTETS': 80, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 54438, 'DST_PORT': 445, 'TCP_FLAGS': 18, 'PROTO': 6, 'TOS': 16, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 24, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 5: {'IPV4_SRC_ADDR': '172.19.25.8', 'IPV4_DST_ADDR': '192.168.1.213', 'NEXT_HOP': '83.172.67.1', 'INPUT': 38, 'OUTPUT': 11, 'IN_PACKETS': 1, 'IN_OCTETS': 40, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 54438, 'DST_PORT': 445, 'TCP_FLAGS': 17, 'PROTO': 6, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 24, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 6: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 74, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 58412, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 7: {'IPV4_SRC_ADDR': '192.168.15.3', 'IPV4_DST_ADDR': '192.168.1.63', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 11, 'IN_PACKETS': 2, 'IN_OCTETS': 221, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 53, 'DST_PORT': 53150, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 28, 'DST_MASK': 24, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 8: {'IPV4_SRC_ADDR': '192.168.15.3', 'IPV4_DST_ADDR': '192.168.1.63', 'NEXT_HOP': '83.172.67.1', 'INPUT': 13, 'OUTPUT': 11, 'IN_PACKETS': 2, 'IN_OCTETS': 221, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 53, 'DST_PORT': 53150, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 28, 'DST_MASK': 24, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 9: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 73, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 41021, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 10: {'IPV4_SRC_ADDR': '192.168.1.1', 'IPV4_DST_ADDR': '192.168.1.160', 'NEXT_HOP': '83.172.67.1', 'INPUT': 0, 'OUTPUT': 11, 'IN_PACKETS': 1, 'IN_OCTETS': 85, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 39422, 'DST_PORT': 53, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 32, 'DST_MASK': 24, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 11: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '77.250.67.8', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 1, 'IN_OCTETS': 40, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 51413, 'DST_PORT': 58107, 'TCP_FLAGS': 4, 'PROTO': 6, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 12: {'IPV4_SRC_ADDR': '192.168.15.3', 'IPV4_DST_ADDR': '192.168.10.60', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 12, 'IN_PACKETS': 1, 'IN_OCTETS': 258, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 53, 'DST_PORT': 51252, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 28, 'DST_MASK': 24, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 13: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '198.54.132.138', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 1, 'IN_OCTETS': 52, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 51413, 'DST_PORT': 40533, 'TCP_FLAGS': 16, 'PROTO': 6, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 14: {'IPV4_SRC_ADDR': '192.168.10.60', 'IPV4_DST_ADDR': '10.1.1.160', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 1, 'IN_OCTETS': 67, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 51252, 'DST_PORT': 53, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 15: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 74, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 44821, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 16: {'IPV4_SRC_ADDR': '192.168.1.63', 'IPV4_DST_ADDR': '192.168.15.3', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 14, 'IN_PACKETS': 2, 'IN_OCTETS': 130, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 53150, 'DST_PORT': 53, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 17: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 71, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 45414, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 18: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '45.67.97.23', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 1, 'IN_OCTETS': 40, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 51413, 'DST_PORT': 63726, 'TCP_FLAGS': 4, 'PROTO': 6, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 19: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 71, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 59516, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 20: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 74, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 57469, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 21: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '185.195.233.179', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 1, 'IN_OCTETS': 40, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 51413, 'DST_PORT': 62176, 'TCP_FLAGS': 16, 'PROTO': 6, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 22: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 71, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 42101, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 23: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 73, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 56654, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 24: {'IPV4_SRC_ADDR': '192.168.1.210', 'IPV4_DST_ADDR': '192.168.20.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 16, 'IN_PACKETS': 1, 'IN_OCTETS': 73, 'FIRST_SWITCHED': 2565250000, 'LAST_SWITCHED': 2565250000, 'SRC_PORT': 52803, 'DST_PORT': 161, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 0, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 24, 'DST_MASK': 28, 'FIRST_SWITCHED_HR': '29 days, 16:34:10', 'LAST_SWITCHED_HR': '29 days, 16:34:10'}, 25: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '91.237.122.178', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 8, 'IN_OCTETS': 1125, 'FIRST_SWITCHED': 2565265000, 'LAST_SWITCHED': 2565266000, 'SRC_PORT': 51413, 'DST_PORT': 54931, 'TCP_FLAGS': 25, 'PROTO': 6, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:25', 'LAST_SWITCHED_HR': '29 days, 16:34:26'}, 26: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '79.239.89.6', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 7, 'IN_OCTETS': 997, 'FIRST_SWITCHED': 2565265000, 'LAST_SWITCHED': 2565267000, 'SRC_PORT': 51413, 'DST_PORT': 63332, 'TCP_FLAGS': 25, 'PROTO': 6, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:25', 'LAST_SWITCHED_HR': '29 days, 16:34:27'}, 27: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '185.156.46.39', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 7, 'IN_OCTETS': 987, 'FIRST_SWITCHED': 2565266000, 'LAST_SWITCHED': 2565267000, 'SRC_PORT': 51413, 'DST_PORT': 54841, 'TCP_FLAGS': 25, 'PROTO': 6, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:26', 'LAST_SWITCHED_HR': '29 days, 16:34:27'}, 28: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '71.135.146.162', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 1, 'IN_OCTETS': 86, 'FIRST_SWITCHED': 2565251000, 'LAST_SWITCHED': 2565251000, 'SRC_PORT': 51413, 'DST_PORT': 35087, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:11', 'LAST_SWITCHED_HR': '29 days, 16:34:11'}, 29: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '67.22.74.10', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 2, 'IN_OCTETS': 172, 'FIRST_SWITCHED': 2565242000, 'LAST_SWITCHED': 2565251000, 'SRC_PORT': 51413, 'DST_PORT': 23799, 'TCP_FLAGS': 0, 'PROTO': 17, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:02', 'LAST_SWITCHED_HR': '29 days, 16:34:11'}, 30: {'IPV4_SRC_ADDR': '192.168.42.6', 'IPV4_DST_ADDR': '138.199.32.158', 'NEXT_HOP': '83.172.67.1', 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 1, 'IN_OCTETS': 40, 'FIRST_SWITCHED': 2565251000, 'LAST_SWITCHED': 2565251000, 'SRC_PORT': 51413, 'DST_PORT': 60032, 'TCP_FLAGS': 16, 'PROTO': 6, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0, 'FIRST_SWITCHED_HR': '29 days, 16:34:11', 'LAST_SWITCHED_HR': '29 days, 16:34:11'}} diff --git a/flow.example.org b/flow.example.org deleted file mode 100644 index 95994f1..0000000 --- a/flow.example.org +++ /dev/null @@ -1,2 +0,0 @@ -Ready -{'IPV4_SRC_ADDR': 3232246278, 'IPV4_DST_ADDR': 100622484, 'NEXT_HOP': 1403798273, 'INPUT': 11, 'OUTPUT': 4, 'IN_PACKETS': 5, 'IN_OCTETS': 468, 'FIRST_SWITCHED': 2565993000, 'LAST_SWITCHED': 2565994000, 'SRC_PORT': 51413, 'DST_PORT': 36839, 'TCP_FLAGS': 25, 'PROTO': 6, 'TOS': 4, 'SRC_AS': 0, 'DST_AS': 0, 'SRC_MASK': 29, 'DST_MASK': 0} diff --git a/main.py b/main.py deleted file mode 100644 index 9f5f0de..0000000 --- a/main.py +++ /dev/null @@ -1,31 +0,0 @@ -import netflow -import socket -import json - -i=1 -bigDict = {} - -sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -sock.bind(("0.0.0.0", 2055)) -payload, client = sock.recvfrom(4096) # experimental, tested with 1464 bytes -p = netflow.parse_packet(payload) # Test result: -#print(p.flows) # Test result: 5 - -for entry in p.flows: - # prep dict - tmpEntry = str(entry) - tmpEntry = tmpEntry[22:-1] - tmpEntry2 = tmpEntry.replace("'", '"') - - #print(tmpEntry2) - - - - dictEntry = json.loads(tmpEntry2) - bigDict[i] = (dictEntry) - i+=1 - #type(tmpEntry) - #print(dictEntry) - #print(tmpEntry.lstrip(20)) - print("----------------") -print(bigDict) diff --git a/main2.py b/main2.py deleted file mode 100644 index 5f63679..0000000 --- a/main2.py +++ /dev/null @@ -1,62 +0,0 @@ -from scapy.all import * -import struct - -# Define NetFlow v5 header and record formats -NETFLOW_V5_HEADER_LEN = 24 -NETFLOW_V5_RECORD_LEN = 48 - -def parse_netflow_v5(packet): - if not packet.haslayer(UDP): - return - - udp_payload = bytes(packet[UDP].payload) - - if len(udp_payload) < NETFLOW_V5_HEADER_LEN: - print("Invalid NetFlow v5 header length") - return - - # Parse header - header = struct.unpack('!HHIIIIBBH', udp_payload[:NETFLOW_V5_HEADER_LEN]) - version, count, uptime, unix_secs, unix_nsecs, flow_seq, engine_type, engine_id, sampling = header - - print(f"\n--- NetFlow v5 Packet ---") - print(f"Version: {version}, Record count: {count}, Sys uptime: {uptime}") - print(f"Unix secs: {unix_secs}, Flow sequence: {flow_seq}") - - # Parse flow records - offset = NETFLOW_V5_HEADER_LEN - for i in range(count): - if offset + NETFLOW_V5_RECORD_LEN > len(udp_payload): - print("Incomplete record") - break - - record = struct.unpack('!IIIHHIIIIHHBBBBHHBBH', udp_payload[offset:offset + NETFLOW_V5_RECORD_LEN]) - src_addr = inet_ntoa(struct.pack('!I', record[0])) - dst_addr = inet_ntoa(struct.pack('!I', record[1])) - next_hop = inet_ntoa(struct.pack('!I', record[2])) - src_port = record[10] - dst_port = record[11] - packets = record[4] - bytes_ = record[5] - - print(f"\nFlow #{i+1}") - print(f"Src IP: {src_addr}:{src_port} → Dst IP: {dst_addr}:{dst_port}") - print(f"Next hop: {next_hop}, Packets: {packets}, Bytes: {bytes_}") - - offset += NETFLOW_V5_RECORD_LEN - -# Sniff on a given interface or from a pcap file -def start_sniff(interface=None, pcap_file=None): - if pcap_file: - packets = rdpcap(pcap_file) - for pkt in packets: - parse_netflow_v5(pkt) - elif interface: - sniff(iface=interface, filter="udp port 2055", prn=parse_netflow_v5) - -# Example usage -if __name__ == "__main__": - # Provide either an interface or a pcap file - start_sniff(pcap_file="netflowv5_sample.pcap") - # start_sniff(interface="eth0") - diff --git a/sysctl.d/netflow.conf b/sysctl.d/netflow.conf new file mode 100644 index 0000000..bf6600b --- /dev/null +++ b/sysctl.d/netflow.conf @@ -0,0 +1,14 @@ +# for less then 75k flows per second +net.core.netdev_max_backlog=4096 +net.core.rmem_default=262144 +net.core.rmem_max=67108864 +net.ipv4.udp_rmem_min=131072 +net.ipv4.udp_mem=2097152 4194304 8388608 + + +# for more than 75k flows per second +#net.core.netdev_max_backlog=8192 +#net.core.rmem_default=262144 +#net.core.rmem_max=134217728 +#net.ipv4.udp_rmem_min=131072 +#net.ipv4.udp_mem=4194304 8388608 16777216 diff --git a/systemdConfig/README.md b/systemdConfig/README.md new file mode 100644 index 0000000..5b34f7f --- /dev/null +++ b/systemdConfig/README.md @@ -0,0 +1,8 @@ +# What you need to prepare + +virtual enviroment in python with installed packages + +python script and InfluxDB + +## Change stuff +In .service file change everything that has and /path/to/dir \ No newline at end of file diff --git a/systemdHowTo/HQNetFlowInflux.service b/systemdConfig/netflowCollector.example.service similarity index 87% rename from systemdHowTo/HQNetFlowInflux.service rename to systemdConfig/netflowCollector.example.service index 4b2dfbe..2015c41 100644 --- a/systemdHowTo/HQNetFlowInflux.service +++ b/systemdConfig/netflowCollector.example.service @@ -1,16 +1,19 @@ [Unit] Description=Netflow to InfluxDB script After=multi-user.target network.target network-online.target +# Place in /etc/systemd/system/ [Service] +User=yuru +Group=yuru Type=simple Restart=on-failure # EnvironmentFile=/etc/NetFlux/netflow.env # User=myuser WorkingDirectory=/etc/NetFlux/HQ/ ExecStart=/etc/NetFlux/HQ/venv/bin/python3 /etc/NetFlux/HQ/HQnetflow.py --serve-in-foreground -# ExecStart=/usr/bin/python3 /usr/bin/NetFlux/INFLUXDBmthrd.py -# StandardInput=tty-force + +#StandardInput=tty-force # Log file will be create if it doesn't exist StandardOutput=append:/var/log/HQNetFlowInflux.log diff --git a/systemdHowTo/HAOSNetFlowInflux.service b/systemdConfig/netflowCollector.service similarity index 56% rename from systemdHowTo/HAOSNetFlowInflux.service rename to systemdConfig/netflowCollector.service index 03d81f6..9598904 100644 --- a/systemdHowTo/HAOSNetFlowInflux.service +++ b/systemdConfig/netflowCollector.service @@ -4,20 +4,19 @@ After=multi-user.target network.target network-online.target # Place in /etc/systemd/system/ [Service] -User=root -Group=root +User= +Group= Type=simple Restart=on-failure # EnvironmentFile=/etc/NetFlux/netflow.env # User=myuser -WorkingDirectory=/etc/NetFlux/HAOS/ -ExecStart=/etc/NetFlux/HAOS/venv/bin/python3 /etc/NetFlux/HAOS/HAOSnetflow.py --serve-in-foreground -# ExecStart=/usr/bin/python3 /usr/bin/NetFlux/INFLUXDBmthrd.py +WorkingDirectory=/dir/to/script/ +ExecStart=/dir/to/script'sVENV/venv/bin/python3 /dir/to/script/NetFlowCollect.py --serve-in-foreground #StandardInput=tty-force # Log file will be create if it doesn't exist -StandardOutput=append:/var/log/HAOSNetFlowInflux.log -StandardError=append:/var/log/HAOSNetFlowInflux.errlog +StandardOutput=append:/var/log/NetFlowCollect.log +StandardError=append:/var/log/NetFlowCollect.errlog # StandardOutput=syslog # StandardError=syslog diff --git a/systemdHowTo/INFLUXDB.py b/systemdHowTo/INFLUXDB.py deleted file mode 100644 index bb102e7..0000000 --- a/systemdHowTo/INFLUXDB.py +++ /dev/null @@ -1,138 +0,0 @@ -import netflow, socket, json, time, os, influxdb_client, ipaddress -from influxdb_client import InfluxDBClient, Point, WritePrecision -from influxdb_client.client.write_api import SYNCHRONOUS, ASYNCHRONOUS, WriteOptions -from datetime import timedelta -from proto import manWhatTheProto -from IP2Loc import ermWhatTheCountry -from whatDomain import ermWhatATheIpFromDomainYaCrazy, ermWhatAAAATheIpFromDomainYaCrazy - -# Netentry preconf -WHAT_THE_NETFLOW_PORT = 2055 -WHAT_THE_NETFLOW_IP = "0.0.0.0" - - -# INFLUXDB config - -token = "apg1gysUeCcxdcRTMmosJTenbEppmUNi9rXlANDB2oNadBdWAu2GVTDc_q_dyo0iyYsckKaOvPRm6ba2NK0y_A==" -bucket = "NETFLOW-7" -org = "staging" -url = "http://localhost:8086" -measurement = "testNetFlowPython" -MACHINE_TAG = "YUKIKAZE" -ROUTER_TAG = "HQ" -INFLX_SEPARATE_POINTS = 0.1 - -# Initialize InfluxDB client and influxdb API -inflxdb_client = influxdb_client.InfluxDBClient(url=url, token=token, org=org) -write_api = inflxdb_client.write_api(write_options=SYNCHRONOUS) - -# Other preconf -bigDict = {} -inflxdb_Datazz_To_Send = [] - -# Bind -sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -sock.bind((WHAT_THE_NETFLOW_IP, WHAT_THE_NETFLOW_PORT)) - -print("Ready") - -while True: - # Get netentry data ig? - payload, client = sock.recvfrom(4096) # experimental, tested with 1464 bytes - p = netflow.parse_packet(payload) # Test result: - #print(p.entrys) # Test result: 5 - - #yesyes = p.flows - #print(yesyes.data) - #exit() - - - - for i, entry in enumerate(p.flows, 1): - # prep dict - #tmpEntry = str(entry) - #tmpEntry = tmpEntry[22:-1] - #tmpEntry2 = tmpEntry.replace("'", '"') - - #print(tmpEntry2) - #print(entry - #exit() - #dictEntry = json.loads(tmpEntry2) - #bigDict[i] = (dictEntry) - - - # take data out from netentry - inEntry = entry.data - - print(inEntry) - exit() - - # Convert IPs and time duration - # IPs - inEntry["IPV4_SRC_ADDR"] = str(ipaddress.IPv4Address(inEntry["IPV4_SRC_ADDR"])) - inEntry["IPV4_DST_ADDR"] = str(ipaddress.IPv4Address(inEntry["IPV4_DST_ADDR"])) - inEntry["NEXT_HOP"] = str(ipaddress.IPv4Address(inEntry["NEXT_HOP"])) - - # Convert time from ms to HH:MM:SS - first = int(inEntry["FIRST_SWITCHED"]) - last = int(inEntry["LAST_SWITCHED"]) - - inEntry["FIRST_SWITCHED_HR"] = str(timedelta(milliseconds=first)) - inEntry["LAST_SWITCHED_HR"] = str(timedelta(milliseconds=last)) - - - # Prep InfluxDB data - inflxdb_Data_To_Send = ( - influxdb_client.Point(f"{measurement}-script") - .tag("MACHINE", MACHINE_TAG) - .tag("ROUTER", ROUTER_TAG) - .field("dstAddr", inEntry["IPV4_DST_ADDR"]) - .field("srcAddr", inEntry["IPV4_SRC_ADDR"]) - .field("nextHop", inEntry["NEXT_HOP"]) - .field("inptInt", inEntry["INPUT"]) - .field("outptInt", inEntry["OUTPUT"]) - .field("inPackt", inEntry["IN_PACKETS"]) - .field("outPakt", inEntry["IN_OCTETS"]) - .field("frstSwtchd", inEntry["FIRST_SWITCHED"]) - .field("lstSwtchd", inEntry["LAST_SWITCHED"]) - .field("srcPort", inEntry["SRC_PORT"]) - .field("dstPort", inEntry["DST_PORT"]) - .field("tcpFlags", inEntry["TCP_FLAGS"]) - .tag("proto", manWhatTheProto(int(inEntry["PROTO"]))) - .field("tos", inEntry["TOS"]) - .field("srcAS", inEntry["SRC_AS"]) - .field("dstAS", inEntry["DST_AS"]) - .field("srcMask", inEntry["SRC_MASK"]) - .field("dstMask", inEntry["DST_MASK"]) - .field("dstCntr", ermWhatTheCountry(str(inEntry["IPV4_DST_ADDR"]))) - .field("srcCntr", ermWhatTheCountry(str(inEntry["IPV4_SRC_ADDR"]))) - ) - - inflxdb_Datazz_To_Send.append(inflxdb_Data_To_Send) - - #i+=1 - #type(tmpEntry) - #print(dictEntry) - #print(tmpEntry.lstrip(20)) - - print("----------------") - bigDict[i] = (inEntry) - - # end while True - - print() - print(bigDict) - exit() - - # Send data to InfluxDB - write_api.write(bucket=bucket, org="staging", record=inflxdb_Data_To_Send) - time.sleep(INFLX_SEPARATE_POINTS) # separate points - - print(f"{len(bigDict)} <--- This many entrys") - - - # Clean up before another loop - bigDict.clear() - inflxdb_Datazz_To_Send.clear() - - #print(bigDict) diff --git a/systemdHowTo/INFLUXDBmthrd.py b/systemdHowTo/INFLUXDBmthrd.py deleted file mode 100644 index 8689f37..0000000 --- a/systemdHowTo/INFLUXDBmthrd.py +++ /dev/null @@ -1,151 +0,0 @@ -import netflow, socket, json, time, os, influxdb_client, ipaddress -from influxdb_client import InfluxDBClient, Point, WritePrecision -from influxdb_client.client.write_api import SYNCHRONOUS, ASYNCHRONOUS, WriteOptions -from datetime import timedelta -from proto import manWhatTheProto -from IP2Loc import ermWhatTheCountry -from whatDomain import ermWhatATheIpFromDomainYaCrazy, ermWhatAAAATheIpFromDomainYaCrazy -from concurrent.futures import ThreadPoolExecutor - -# Netentry preconf -WHAT_THE_NETFLOW_PORT = 2055 -WHAT_THE_NETFLOW_IP = "0.0.0.0" - -# INFLUXDB config -token = "apg1gysUeCcxdcRTMmosJTenbEppmUNi9rXlANDB2oNadBdWAu2GVTDc_q_dyo0iyYsckKaOvPRm6ba2NK0y_A==" -#token = os.getenv("INFLUX_TOKEN") -bucket = "NETFLOW-7" -# bucket = os.getenv("INFLUX_BUCKET") -org = "staging" -# org = os.getenv("INFLUX_ORG") -url = "http://localhost:8086" -# url = os.getenv("INFLUX_URL") -measurement = "testNetFlowPython" -# measurement = os.getenv("INFLUX_MEASUREMENT") -MACHINE_TAG = "YUKIKAZE" -# MACHINE_TAG = os.getenv("INFLUX_MACHINE_TAG") -ROUTER_TAG = "HQ" -# ROUTER_TAG = os.getenv("INFLUX_ROUTER_TAG") -INFLX_SEPARATE_POINTS = 0.05 - -# Initialize InfluxDB client and influxdb API -inflxdb_client = influxdb_client.InfluxDBClient(url=url, token=token, org=org) -#write_api = inflxdb_client.write_api(write_options=SYNCHRONOUS) -write_api = inflxdb_client.write_api(write_options=WriteOptions(batch_size=500, flush_interval=1000)) - -# Threaded flow processor -def process_flow(i, entry): - # prep dict - #tmpEntry = str(entry) - #tmpEntry = tmpEntry[22:-1] - #tmpEntry2 = tmpEntry.replace("'", '"') - - #print(tmpEntry2) - #print(entry) - #exit() - #dictEntry = json.loads(tmpEntry2) - #bigDict[i] = (dictEntry) - - # take data out from netentry - inEntry = entry.data - - # Convert IPs and time duration - # IPs - inEntry["IPV4_SRC_ADDR"] = str(ipaddress.IPv4Address(inEntry["IPV4_SRC_ADDR"])) - inEntry["IPV4_DST_ADDR"] = str(ipaddress.IPv4Address(inEntry["IPV4_DST_ADDR"])) - inEntry["NEXT_HOP"] = str(ipaddress.IPv4Address(inEntry["NEXT_HOP"])) - - # Convert time from ms to HH:MM:SS - first = int(inEntry["FIRST_SWITCHED"]) - last = int(inEntry["LAST_SWITCHED"]) - - inEntry["FIRST_SWITCHED_HR"] = str(timedelta(milliseconds=first)) - inEntry["LAST_SWITCHED_HR"] = str(timedelta(milliseconds=last)) - - # Prep InfluxDB data - inflxdb_Data_To_Send = ( - influxdb_client.Point(f"{measurement}-script") - .tag("MACHINE", MACHINE_TAG) - .tag("ROUTER", ROUTER_TAG) - .field("dstAddr", inEntry["IPV4_DST_ADDR"]) - .field("srcAddr", inEntry["IPV4_SRC_ADDR"]) - .field("nextHop", inEntry["NEXT_HOP"]) - .field("inptInt", inEntry["INPUT"]) - .field("outptInt", inEntry["OUTPUT"]) - .field("inPackt", inEntry["IN_PACKETS"]) - .field("outPakt", inEntry["IN_OCTETS"]) - .field("frstSwtchd", inEntry["FIRST_SWITCHED"]) - .field("lstSwtchd", inEntry["LAST_SWITCHED"]) - .field("srcPort", inEntry["SRC_PORT"]) - .field("dstPort", inEntry["DST_PORT"]) - .field("tcpFlags", inEntry["TCP_FLAGS"]) - .tag("proto", manWhatTheProto(int(inEntry["PROTO"]))) - .field("tos", inEntry["TOS"]) - .field("srcAS", inEntry["SRC_AS"]) - .field("dstAS", inEntry["DST_AS"]) - .field("srcMask", inEntry["SRC_MASK"]) - .field("dstMask", inEntry["DST_MASK"]) - .field("dstCntr", ermWhatTheCountry(str(inEntry["IPV4_DST_ADDR"]))) - .field("srcCntr", ermWhatTheCountry(str(inEntry["IPV4_SRC_ADDR"]))) - ) - - print("----------------") - return (i, inflxdb_Data_To_Send, inEntry) - -# Bind -sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -sock.bind((WHAT_THE_NETFLOW_IP, WHAT_THE_NETFLOW_PORT)) - -print("Ready") - -with ThreadPoolExecutor(max_workers=8) as executor: - # With means that when exiting the "executor" will be cleanly shut down - # as executor is an object that is then used to give jobs to - - while True: - # Get netentry data ig? - sock.settimeout(5) - - - payload, client = sock.recvfrom(4096) # experimental, tested with 1464 bytes - # Tajes UPD packets that are at max 4096 bytes - # payload has the raw netflow data - # client has source IP address as well as port - - p = netflow.parse_packet(payload) # Test result: - #print(p.entrys) # Test result: 5 - - # Submit all entries to thread pool - futures = [executor.submit(process_flow, i, entry) for i, entry in enumerate(p.flows, 1)] - # Big thinkg happen here - # Here I give an executor a job. That job is to run function with arguments i and entry. Then it becomes one thread - # Furthermore for each entry, so flow record, we submit a task to a thread - # In comparasion, without multithreading it only had one for function - # for i, entry in enumerate(p.flows, 1) - # And the results from a job on a thread (executor) are stored in futures ____list____ - - - bigDict = {} - inflxdb_Datazz_To_Send = [] - - for future in futures: - i, point, inEntry = future.result() - # goes through every job done by executor. - # i is being reused from the original enumerate - # point is the InfluxDB-ready data object - # inEntry is what a single flow in the 30 flow dictionary in Netflow - - inflxdb_Datazz_To_Send.append(point) - bigDict[i] = inEntry - - # Send data to InfluxDB - write_api.write(bucket=bucket, org=org, record=inflxdb_Datazz_To_Send) - time.sleep(INFLX_SEPARATE_POINTS) # separate points - - print(f"{len(bigDict)} <--- This many entrys") - - # Clean up before another loop - bigDict.clear() - inflxdb_Datazz_To_Send.clear() - #print(bigDict) - diff --git a/systemdHowTo/IP2LOCATION-LITE-DB9.BIN b/systemdHowTo/IP2LOCATION-LITE-DB9.BIN deleted file mode 100755 index 3819ef6..0000000 Binary files a/systemdHowTo/IP2LOCATION-LITE-DB9.BIN and /dev/null differ diff --git a/systemdHowTo/IP2Loc.py b/systemdHowTo/IP2Loc.py deleted file mode 100644 index a48891e..0000000 --- a/systemdHowTo/IP2Loc.py +++ /dev/null @@ -1,18 +0,0 @@ -import IP2Location -from typing import Optional, Annotated - -# Load database once -ip2loc_db = IP2Location.IP2Location("IP2LOCATION-LITE-DB9.BIN") - -def ermWhatTheCountry(inpIpAddress: Annotated[str, "Some IP address that ya want to get country for"]): - try: - skibidi = ip2loc_db.get_all(inpIpAddress) - - #return rec.country_long # Full country name, e.g. "Sweden" - return skibidi.country_short - - except Exception as errrrrr: - return f"Error: {errrrrr}" - -#print(ermWhatTheCountry("65.109.142.32")) - diff --git a/systemdHowTo/LHNetFlowInflux.service b/systemdHowTo/LHNetFlowInflux.service deleted file mode 100644 index b017a51..0000000 --- a/systemdHowTo/LHNetFlowInflux.service +++ /dev/null @@ -1,24 +0,0 @@ -[Unit] -Description=Netflow to InfluxDB script -After=multi-user.target network.target network-online.target - -[Service] -Type=simple -Restart=on-failure -# EnvironmentFile=/etc/NetFlux/netflow.env -# User=myuser -WorkingDirectory=/etc/NetFlux/LH/ -ExecStart=/etc/NetFlux/LH/venv/bin/python3 /etc/NetFlux/LH/LHnetflow.py --serve-in-foreground -# ExecStart=/usr/bin/python3 /usr/bin/NetFlux/INFLUXDBmthrd.py -# StandardInput=tty-force - -# Log file will be create if it doesn't exist -StandardOutput=append:/var/log/LHNetFlowInflux.log -StandardError=append:/var/log/LHNetFlowInflux.errlog - -# StandardOutput=syslog -# StandardError=syslog -# SyslogIdentifier=NetFlowInflux - -[Install] -WantedBy=multi-user.target diff --git a/systemdHowTo/NetFlowInflux.conf b/systemdHowTo/NetFlowInflux.conf deleted file mode 100644 index fd33c67..0000000 --- a/systemdHowTo/NetFlowInflux.conf +++ /dev/null @@ -1,13 +0,0 @@ -#Put it in /etc/rsyslog.d/ - -if $programname == 'NetFlowInflux' then /var/log/NetFlowInflux.log -& stop - - -#then run -# -#touch /var/log/yapyap -#chown syslog /var/log/yapyap -#ls -l /var/log/yapyap.log -# And then -# systemctl restart rsyslog diff --git a/systemdHowTo/NetFlowInflux.env b/systemdHowTo/NetFlowInflux.env deleted file mode 100644 index 8fb8077..0000000 --- a/systemdHowTo/NetFlowInflux.env +++ /dev/null @@ -1,7 +0,0 @@ -INFLUX_TOKEN="" -INFLUX_BUCKET="" -INFLUX_ORG="" -INFLUX_URL="" -INFLUX_MEASUREMENT="" -INFLUX_MACHINE_TAG="" -INFLUX_ROUTER_TAG="" diff --git a/systemdHowTo/NetFlowInflux.service b/systemdHowTo/NetFlowInflux.service deleted file mode 100644 index 7c5043f..0000000 --- a/systemdHowTo/NetFlowInflux.service +++ /dev/null @@ -1,24 +0,0 @@ -[Unit] -Description=Netflow to InfluxDB script -After=multi-user.target network.target network-online.target - -[Service] -Type=simple -Restart=on-failure -# EnvironmentFile=/etc/NetFlux/netflow.env -# User=myuser -WorkingDirectory=/etc/NetFlux// -ExecStart=/etc/NetFlux//venv/bin/python3 /etc/NetFlux//HAOSnetflow.py --serve-in-foreground -# ExecStart=/usr/bin/python3 /usr/bin/NetFlux/INFLUXDBmthrd.py -StandardInput=tty-force - -# Log file will be create if it doesn't exist -StandardOutput=append:/var/log/NetFlowInflux.log -StandardError=append:/var/log/NetFlowInflux.errlog - -# StandardOutput=syslog -# StandardError=syslog -# SyslogIdentifier=NetFlowInflux - -[Install] -WantedBy=multi-user.target diff --git a/systemdHowTo/proto.py b/systemdHowTo/proto.py deleted file mode 100644 index bb2a536..0000000 --- a/systemdHowTo/proto.py +++ /dev/null @@ -1,178 +0,0 @@ -from typing import Optional, Annotated - -# Source -# https://en.wikipedia.org/wiki/List_of_IP_protocol_numbers -PROTO_MAP = { - 0: "HOPOPT", - 1: "ICMP", - 2: "IGMP", - 3: "GGP", - 4: "IPv4", - 5: "ST", - 6: "TCP", - 7: "CBT", - 8: "EGP", - 9: "IGP", - 10: "BBN-RCC-MON", - 11: "NVP-II", - 12: "PUP", - 13: "ARGUS", - 14: "EMCON", - 15: "XNET", - 16: "CHAOS", - 17: "UDP", - 18: "MUX", - 19: "DCN-MEAS", - 20: "HMP", - 21: "PRM", - 22: "XNS-IDP", - 23: "TRUNK-1", - 24: "TRUNK-2", - 25: "LEAF-1", - 26: "LEAF-2", - 27: "RDP", - 28: "IRTP", - 29: "ISO-TP4", - 30: "NETBLT", - 31: "MFE-NSP", - 32: "MERIT-INP", - 33: "DCCP", - 34: "3PC", - 35: "IDPR", - 36: "XTP", - 37: "DDP", - 38: "IDPR-CMTP", - 39: "TP++", - 40: "IL", - 41: "IPv6", - 42: "SDRP", - 43: "IPv6-Route", - 44: "IPv6-Frag", - 45: "IDRP", - 46: "RSVP", - 47: "GRE", - 48: "DSR", - 49: "BNA", - 50: "ESP", - 51: "AH", - 52: "I-NLSP", - 53: "SWIPE", - 54: "NARP", - 55: "MOBILE", - 56: "TLSP", - 57: "SKIP", - 58: "IPv6-ICMP", - 59: "IPv6-NoNxt", - 60: "IPv6-Opts", - 61: "ANY_HOST_INTERNAL", - 62: "CFTP", - 63: "ANY_LOCAL_NETWORK", - 64: "SAT-EXPAK", - 65: "KRYPTOLAN", - 66: "RVD", - 67: "IPPC", - 68: "ANY_DISTRIBUTED_FS", - 69: "SAT-MON", - 70: "VISA", - 71: "IPCV", - 72: "CPNX", - 73: "CPHB", - 74: "WSN", - 75: "PVP", - 76: "BR-SAT-MON", - 77: "SUN-ND", - 78: "WB-MON", - 79: "WB-EXPAK", - 80: "ISO-IP", - 81: "VMTP", - 82: "SECURE-VMTP", - 83: "VINES", - 84: "TTP", - 85: "NSFNET-IGP", - 86: "DGP", - 87: "TCF", - 88: "EIGRP", - 89: "OSPF", - 90: "Sprite-RPC", - 91: "LARP", - 92: "MTP", - 93: "AX.25", - 94: "IPIP", - 95: "MICP", - 96: "SCC-SP", - 97: "ETHERIP", - 98: "ENCAP", - 99: "ANY_PRIVATE_ENCRYPTION", - 100: "GMTP", - 101: "IFMP", - 102: "PNNI", - 103: "PIM", - 104: "ARIS", - 105: "SCPS", - 106: "QNX", - 107: "A/N", - 108: "IPComp", - 109: "SNP", - 110: "Compaq-Peer", - 111: "IPX-in-IP", - 112: "VRRP", - 113: "PGM", - 114: "ANY_0_HOP", - 115: "L2TP", - 116: "DDX", - 117: "IATP", - 118: "STP", - 119: "SRP", - 120: "UTI", - 121: "SMP", - 122: "SM", - 123: "PTP", - 124: "ISIS over IPv4", - 125: "FIRE", - 126: "CRTP", - 127: "CRUDP", - 128: "SSCOPMCE", - 129: "IPLT", - 130: "SPS", - 131: "PIPE", - 132: "SCTP", - 133: "FC", - 134: "RSVP-E2E-IGNORE", - 135: "Mobility Header", - 136: "UDPLite", - 137: "MPLS-in-IP", - 138: "manet", - 139: "HIP", - 140: "Shim6", - 141: "WESP", - 142: "ROHC", - 143: "Ethernet", - 144: "AGGFRAG", - 145: "NSH" - -} - - -def manWhatTheProto(inpProtoNumbrMaybe: Annotated[int, "Protocol number goes here"]): - - if inpProtoNumbrMaybe <= 145: - return PROTO_MAP.get(inpProtoNumbrMaybe) - elif inpProtoNumbrMaybe >= 146 and inpProtoNumbrMaybe <= 252: - return "Unassigned" - elif inpProtoNumbrMaybe >= 253 and inpProtoNumbrMaybe <= 254: - # Use for experimentation and testing - return "RFC3692" - elif inpProtoNumbrMaybe == 255: - return "Reserved" - elif inpProtoNumbrMaybe not in PROTO_MAP: - return inpProtoNumbrMaybe - else: - return "no" - - #outPotentialProtoNameIfItExistsInInternalList = PROTO_MAP.get(inpProtoNumbrMaybe) - - - - -#print(manWhatTheProto(253)) -#print( PROTO_MAP.get(2)) diff --git a/systemdHowTo/requirements b/systemdHowTo/requirements deleted file mode 100644 index f25a468..0000000 --- a/systemdHowTo/requirements +++ /dev/null @@ -1,8 +0,0 @@ -netflow==0.12.2 -#socket== -#json== -influxdb-client==1.48.0 -ipaddress==1.0.23 -typing==3.7.4.3 -IP2Location==8.10.5 -nslookup==1.8.1 diff --git a/systemdHowTo/updateIP2Lbin.sh b/systemdHowTo/updateIP2Lbin.sh deleted file mode 100755 index 42b3bee..0000000 --- a/systemdHowTo/updateIP2Lbin.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -# ----------------------------- -# IP2Location DB1 Updater -# ----------------------------- - -# Change this URL to your personal download link if needed -DOWNLOAD_URL="https://www.ip2location.com/download/?token=MkyoKFL854ID0FOWoeTWVRsw0SVcbA7ey6tvuzHchsIQ6AMGy7YXIDfwrEEA4Ozn&file=DB9LITEBIN" - -# Define filenames -ZIP_FILE="IP2LOCATION-LITE-DB9.BIN.ZIP" -BIN_FILE="IP2LOCATION-LITE-DB9.BIN" - -echo "Downloading latest IP2Location DB1..." -curl -L -o "$ZIP_FILE" "$DOWNLOAD_URL" - -if [ $? -ne 0 ]; then - echo "Download failed!" - exit 1 -fi - -echo "Unzipping BIN file..." -unzip -o "$ZIP_FILE" - -if [ ! -f "$BIN_FILE" ]; then - echo "Unzip failed or $BIN_FILE not found." - exit 1 -fi - -echo "Cleaning up ZIP..." -rm "$ZIP_FILE" - -echo "Update complete. BIN file ready: $BIN_FILE" - diff --git a/whatDomain.py b/whatDomain.py deleted file mode 100644 index 4b86861..0000000 --- a/whatDomain.py +++ /dev/null @@ -1,78 +0,0 @@ -#from nslookup import Nslookup -from typing import Optional, Annotated -import dns, dns.resolver - -# https://www.codeunderscored.com/nslookup-python/ - -def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]): - #dns_query = Nslookup() - """ - Tells you what IPv4 address/es a domain point to. - Returns: - dict: A dictionary with IP addresses associated with that domain. - - """ - - # i = 0 - outDict = {} - - #result = dns_query.dns_lookup("example.com") - #result = Nslookup.dns_lookup(inpDomainNameOrSomething) - result = dns.resolver.resolve(inpDomainNameOrSomething, 'A') - for i, something in enumerate(result): - outDict[i] = something.to_text() - # i += 1 - - return outDict - -def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]): - #dns_query = Nslookup() - """ - Tells you what IPv6 address/es a domain point to. - Returns: - dict: A dictionary with IP addresses associated with that domain. - - """ - - - # i = 0 - outDict = {} - - #result = dns_query.dns_lookup("example.com") - #result = Nslookup.dns_lookup(inpDomainNameOrSomething) - result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA') - for i, something in enumerate(result): - outDict[i] = something.to_text() - # i += 1 - - return outDict - - -def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]): - #dns_query = Nslookup() - """ - Tells you what IPv6 address/es a domain point to. - Returns: - dict: A dictionary with IP addresses associated with that domain. - - """ - - whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa" - - - # i = 0 - outDict = {} - - #result = dns_query.dns_lookup("example.com") - #result = Nslookup.dns_lookup(inpDomainNameOrSomething) - result = dns.resolver.resolve(whatToCheck, 'PTR') - for i, something in enumerate(result): - outDict[i] = something.to_text() - # i += 1 - - return outDict - - -#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net")) -#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net")) -#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))