diff --git a/opnsense/.gitignore b/opnsense/.gitignore new file mode 100644 index 0000000..169ce1b --- /dev/null +++ b/opnsense/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +test \ No newline at end of file diff --git a/opnsense/main.log b/opnsense/main.log new file mode 100644 index 0000000..fa66501 --- /dev/null +++ b/opnsense/main.log @@ -0,0 +1,187 @@ +[
+

Australia

+

AARNet Pty Ltd

+
,
+

Brazil

+

Cloudfence

+
,
+

China

+

Aivian

+

Peking University open source software mirror

+
,
+

Colombia

+

Ventura Systems

+
,
+

Denmark

+

Aalborg University

+

c0urier.net

+
,
+

Ecuador

+

CEDIA

+

Universidad Estatal de Bolívar

+
,
+

France

+

Vraphim

+
,
+

Germany

+

dns-root.de (Cloudflare CDN)

+

level66.network

+

uvensys GmbH

+

LeaseWeb

+
,
+

Greece

+

University of Crete / Department of Physics

+
,
+

Hungary

+

Quantum Mirror

+
,
+

India

+

Hopbox

+
,
+

Iran

+

Isatis Data Center (isatisidc.ir)

+
,
+

Korea

+

DAXNET, ROKFOSS PROJECT

+

Techlabs, ROKFOSS PROJECT

+

Zzunimirror, ROKFOSS PROJECT

+

HMirror, ROKFOSS PROJECT

+

Keiverse (Gumi, South Korea)

+
,
+

Morocco

+

MARWAN, Rabat

+
,
+

New Zealand

+

catalyst.net Ltd.

+
,
+

Portugal

+

Community (Lisbon)

+
,
+

Spain

+

Raiola Networks (Madrid)

+
,
+

Switzerland

+

Hiho.ch

+

Init7

+

ServerBase AG

+
,
+

Taiwan

+

Nantou County Education Network Center

+
,
+

The Netherlands

+

LeaseWeb

+

Serverion

+

OPNsense

+
,
+

Turkey

+

Verinomi

+
,
+

United Kingdom

+

University of Kent

+
,
+

United States

+

NYC*BUG (East Coast)

+

LeaseWeb (East Coast)

+

LeaseWeb (West Coast)

+

Open Computing Facility at UC Berkeley (Berkeley, California)

+
] +Found the URL: https://opnsense.org +Found the URL: / +Found the URL: # +Found the URL: https://opnsense.org/get-started/ +Found the URL: # +Found the URL: https://opnsense.org/blog/ +Found the URL: https://shop.opnsense.com/ +Found the URL: https://opnsense.org/download/ +Found the URL: https://opnsense.org/opnsense/ +Found the URL: https://opnsense.org/roadmap/ +Found the URL: https://opnsense.org/features/ +Found the URL: https://opnsense.org/partners/ +Found the URL: https://opnsense.org/legal-guidelines/ +Found the URL: https://opnsense.org/contact/ +Found the URL: https://opnsense.org/get-started/ +Found the URL: https://docs.opnsense.org/ +Found the URL: https://opnsense.org/your-invited/ +Found the URL: http://forum.opnsense.org/ +Found the URL: https://docs.opnsense.org/ +Found the URL: https://opnsense.org/donate/ +Found the URL: mailto:project@opnsense.org +Found the URL: / +Found the URL: # +Found the URL: https://opnsense.org/opnsense/ +Found the URL: https://opnsense.org/roadmap/ +Found the URL: https://opnsense.org/features/ +Found the URL: https://opnsense.org/partners/ +Found the URL: https://opnsense.org/legal-guidelines/ +Found the URL: https://opnsense.org/contact/ +Found the URL: https://opnsense.org/get-started/ +Found the URL: https://opnsense.org/get-started/ +Found the URL: https://docs.opnsense.org/ +Found the URL: # +Found the URL: https://opnsense.org/your-invited/ +Found the URL: http://forum.opnsense.org/ +Found the URL: https://docs.opnsense.org/ +Found the URL: https://opnsense.org/donate/ +Found the URL: https://opnsense.org/blog/ +Found the URL: https://shop.opnsense.com/ +Found the URL: https://opnsense.org/download/ +Found the URL: https://opnsense.org/ +Found the URL: https://opnsense.org/donate/ +Found the URL: https://mirror.ams1.nl.leaseweb.net/opnsense/releases/25.1/OPNsense-25.1-dvd-amd64.iso.bz2 +Found the URL: #full-mirror-listing +Found the URL: https://docs.opnsense.org/manual/install.html#download-and-verification +Found the URL: https://shop.opnsense.com/product/opnsense-business-edition/ +Found the URL: https://www.amazon.de/Practical-OPNsense-Building-Enterprise-Firewalls/dp/3757805364/ +Found the URL: https://shop.opnsense.com/product/opnsense-business-edition/ +Found the URL: https://mynetworktraining.com/p/official-opnsense-introduction-course +Found the URL: https://mirror.aarnet.edu.au/pub/opnsense/releases/mirror/ +Found the URL: https://mirror.cloudfence.com.br/opnsense/releases/mirror/ +Found the URL: https://opnsense.aivian.org/releases/mirror/ +Found the URL: https://mirrors.pku.edu.cn/opnsense/releases/mirror/ +Found the URL: https://mirror.venturasystems.tech/opnsense/releases/mirror/ +Found the URL: https://mirrors.dotsrc.org/opnsense/releases/mirror/ +Found the URL: https://opnsense.c0urier.net/releases/mirror/ +Found the URL: https://mirror.cedia.org.ec/opnsense/releases/mirror/ +Found the URL: https://mirror.ueb.edu.ec/opnsense/releases/mirror/ +Found the URL: https://mirror.vraphim.com/opnsense/releases/mirror/ +Found the URL: https://mirror.dns-root.de/opnsense/releases/mirror/ +Found the URL: https://mirror.level66.network/opnsense-dist/releases/mirror/ +Found the URL: https://mirror.uvensys.de/opnsense/releases/mirror/ +Found the URL: https://mirror.fra10.de.leaseweb.net/opnsense/releases/mirror/ +Found the URL: https://ftp.cc.uoc.gr/mirrors/opnsense/releases/mirror/ +Found the URL: https://quantum-mirror.hu/mirrors/pub/opnsense +Found the URL: https://mirrors.hopbox.net/opnsense/releases/mirror/ +Found the URL: https://mirror1.isatisidc.ir/opnsense/releases/mirror/ +Found the URL: https://mirror.amuksa.com/opnsense/releases/mirror/ +Found the URL: https://mirror.techlabs.co.kr/opnsense/releases/mirror/ +Found the URL: https://mirror.zzunipark.com/opnsense/releases/mirror/ +Found the URL: https://mirror.hemino.net/opnsense/releases/mirror/ +Found the URL: https://mirror.keiminem.com/opnsense/releases/mirror/ +Found the URL: https://mirror.marwan.ma/opnsense/releases/mirror/ +Found the URL: https://mirror.catalyst.net.nz/opnsense/releases/mirror/ +Found the URL: https://mirror.leitecastro.com/opnsense/releases/mirror/ +Found the URL: https://mirror.raiolanetworks.com/opnsense/releases/mirror/ +Found the URL: https://opnsense-mirror.hiho.ch/releases/mirror/ +Found the URL: https://mirror.init7.net/opnsense/releases/mirror/ +Found the URL: https://mirror-opnsense.serverbase.ch/releases/mirror/ +Found the URL: https://mirror.ntct.edu.tw/opnsense/releases/mirror/ +Found the URL: https://mirror.ams1.nl.leaseweb.net/opnsense/releases/mirror/ +Found the URL: https://mirror.serverion.com/opnsense/releases/mirror/ +Found the URL: https://pkg.opnsense.org/releases/mirror/ +Found the URL: https://mirror.verinomi.com/opnsense/releases/mirror/ +Found the URL: https://www.mirrorservice.org/sites/opnsense.org/releases/mirror/ +Found the URL: https://mirrors.nycbug.org/pub/opnsense/releases/mirror/ +Found the URL: https://mirror.wdc1.us.leaseweb.net/opnsense/releases/mirror/ +Found the URL: https://mirror.sfo12.us.leaseweb.net/opnsense/releases/mirror/ +Found the URL: https://mirrors.ocf.berkeley.edu/opnsense/ +Found the URL: mailto:project@opnsense.org +Found the URL: mailto:sales@opnsense.com +Found the URL: https://forum.opnsense.org/ +Found the URL: https://web-cdn.bsky.app/profile/did:plc:vpna73dxrqinj6ttwh3wryxv +Found the URL: https://www.linkedin.com/groups/6958160/ +Found the URL: https://www.reddit.com/r/opnsense/?rdt=38653 +Found the URL: https://github.com/opnsense +Found the URL: /donate +Found the URL: https://deciso.com +Found the URL: https://opnsense.org/terms-conditions/ +Found the URL: https://opnsense.org/privacy-en-cookies/ diff --git a/opnsense/main.py b/opnsense/main.py new file mode 100644 index 0000000..1a35b60 --- /dev/null +++ b/opnsense/main.py @@ -0,0 +1,206 @@ +import requests, schedule, time +from bs4 import BeautifulSoup +from whatDomain import * + + + +OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing" +IPv4FILE = "./OPNS_MirrorListV4" +IPv6FILE = "./OPNS_MirrorListV6" + + +# Define EU and American countries as well as Security for security updates +target_countries = set([ + # Europe + "Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark", + "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland", + "Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland", + "Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland", + "United Kingdom", "Moldova", + # America + "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador", + "Mexico", "Peru", "United States", "Uruguay", "Venezuela", + # Others + "Security", "Extras" +]) + + +def sanitizeURL(inpurl: str): + # if not "/" in inpurl[:-1]: + # inpurl += "/" + if "https://" in inpurl: + outurl = inpurl[8:] + elif "http://" in inpurl: + outurl = inpurl[7:] + elif "http://" or "https://" not in url: + outurl = inpurl + else: + return -1 + + # how the fuck does it work? + # I mean I wrote this but I don't know why does it work. + i = 0 + for char in outurl: + i += 1 + if char == "/": + outurl = outurl[:i] + + if char == "/": + outurl = outurl[:-1] + return outurl + + +def getFreshData(): + payload = requests.get(DEBMIRRORURL) + soup = BeautifulSoup(payload.content, "html.parser") + return soup + + +def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup): + + outMirrorDict = {} + current_country = None + + # Iterate through all table rows + for table in SoupInput.find_all("table"): + for row in table.find_all("tr"): + # Check for country name in a full-row header () + strong = row.find("strong") + if strong: + country_name = strong.get_text(strip=True) + if country_name in target_countries: + current_country = country_name + else: + current_country = None + continue # move to next row + + # Check for inline country name in first column + cols = row.find_all("td") + if len(cols) >= 2: + possible_country = cols[0].get_text(strip=True) + link_tag = cols[1].find("a", href=True) + if possible_country in target_countries: + current_country = possible_country + if current_country and link_tag: + url = link_tag['href'] + if current_country not in outMirrorDict: + outMirrorDict[current_country] = [] + outMirrorDict[current_country].append(url) + + outMirrorDict.update({"Security": DEBSECURITYURL}) + outMirrorDict.update({"Extras": EXTRASURL}) + + return outMirrorDict + +def LeJob(): + + print("Starting lookup") + + LeSoup = getFreshData() + + LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup) + + # print(LeMirrorDict) + + with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW: + + for key, urls in LeMirrorDict.items(): + # print(urls) + if key in target_countries: + + for url in urls: + # print(url) + + if url not in fR: + + goodurl = sanitizeURL(url) + # print(goodurl) + + ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl) + + if ip4Dict == -1: + continue + + for key, ip in ip4Dict.items(): + print(ip) + + fW.write(ip + "/32" + "\n") + + + with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: + + for key, urls in LeMirrorDict.items(): + if key in target_countries: + + for url in urls: + + if url not in fR: + + goodurl = sanitizeURL(url) + # print(goodurl) + + ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl) + + if ip6Dict == -1: + continue + + for key, ip in ip6Dict.items(): + # print(ip) + + fW.write(ip + "/128" + "\n") + + +# schedule.every().day.at("12:45").do(LeJob) +# schedule.every().day.at("17:44").do(LeJob) + +# while True: +# schedule.run_pending() +# print("Waiting...") +# time.sleep(30) #Wait one minute +# # LeJob() + +gigalist = [] + +payload = requests.get(OPNSNSMIRRORURL) +soup = BeautifulSoup(payload.content, "html.parser") + +# print(soup) + +# divs = soup.find_all("div", {"class": "download_section"}) + +for data in soup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): + + url = a['href'] + + saniturl = sanitizeURL(url) + + print(saniturl) + IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) + IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) + + if IPv4Dict == -1: + print(f"ERROR RESOLVING {saniturl} IPv4 address") + continue + elif IPv6Dict == -1: + print(f"ERROR RESOLVING {saniturl} IPv6 address") + continue + + for key, ip in IPv4Dict.items(): + print(f"Found the ipv4: {ip}") + + for key, ip in IPv6Dict.items(): + print(f"Found the ipv6: {ip}") + + # print(a.get('href')) #for getting link + # print(a.text) #for getting text between the link + + + +# for a in soup.find_all("div", {"class": "download_section"}, "a", href=True): + + + +# h2zz = divs = soup.find_all("h3") + +# print(f" one link is: {h2zz}") \ No newline at end of file diff --git a/opnsense/requirements.txt b/opnsense/requirements.txt new file mode 100644 index 0000000..a2aa631 --- /dev/null +++ b/opnsense/requirements.txt @@ -0,0 +1,4 @@ +beautifulsoup4==4.13.4 +requests==2.32.3 +schedule==1.2.2 +nslookup==1.8.1 \ No newline at end of file diff --git a/opnsense/whatDomain.py b/opnsense/whatDomain.py new file mode 100644 index 0000000..4661278 --- /dev/null +++ b/opnsense/whatDomain.py @@ -0,0 +1,93 @@ +#from nslookup import Nslookup +from typing import Optional, Annotated +import dns, dns.resolver + +# https://www.codeunderscored.com/nslookup-python/ + +def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]): + #dns_query = Nslookup() + """ + Tells you what IPv4 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(inpDomainNameOrSomething, 'A') + except dns.resolver.NoAnswer: + return -1 + except dns.resolver.NoNameservers or dns.resolver.LifetimeTimeout: + return -1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + +def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]): + #dns_query = Nslookup() + """ + Tells you what IPv6 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA') + except dns.resolver.NoAnswer: + return -1 + except dns.resolver.NoNameservers or dns.resolver.LifetimeTimeout: + return -1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + + +def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]): + #dns_query = Nslookup() + """ + Tells you what IPv6 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa" + + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(whatToCheck, 'PTR') + except dns.resolver.NoAnswer: + return -1 + except dns.resolver.NoNameservers: + return -1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + + +#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net")) +#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net")) +#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))