diff --git a/code/main.py b/code/main.py index 7a98dff..9774e7b 100644 --- a/code/main.py +++ b/code/main.py @@ -123,13 +123,13 @@ def LeJob(): ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl) - if ip4Dict == -1: + try: + for key, ip in ip4Dict.items(): + print(ip) + + fW.write(ip + "/32" + "\n") + except AttributeError: continue - - for key, ip in ip4Dict.items(): - print(ip) - - fW.write(ip + "/32" + "\n") with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: @@ -146,14 +146,14 @@ def LeJob(): ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl) - if ip6Dict == -1: + try: + for key, ip in ip6Dict.items(): + # print(ip) + + fW.write(ip + "/128" + "\n") + except AttributeError: continue - for key, ip in ip6Dict.items(): - # print(ip) - - fW.write(ip + "/128" + "\n") - # schedule.every().day.at("12:45").do(LeJob) schedule.every().day.at("17:44").do(LeJob) diff --git a/code/whatDomain.py b/code/whatDomain.py index d0a0a4d..70cb81e 100644 --- a/code/whatDomain.py +++ b/code/whatDomain.py @@ -21,7 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom try: result = dns.resolver.resolve(inpDomainNameOrSomething, 'A') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1 @@ -46,7 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, " try: result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1 @@ -74,7 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP try: result = dns.resolver.resolve(whatToCheck, 'PTR') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1 diff --git a/docker/Dockerfile b/docker/Dockerfile index 3c92766..755777b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,6 +4,9 @@ FROM alpine:latest ENV DEBMIRRORURL="https://www.debian.org/mirror/list" ENV EXTRAREPOS=True ENV SECURITYREPOS=True +ENV GITURLPROTO=https +ENV GITURL=tea.shupogaki.org +ENV GITREPOPATH=YuruC3/Debain-repos.git RUN apk update && \ apk add python3 py3-pip git @@ -12,7 +15,7 @@ RUN apk update && \ RUN mkdir /etc/debmirror WORKDIR /etc/debmirror/ -RUN git clone https://tea.shupogaki.org/YuruC3/Debain-repos && \ +RUN git clone {{GITURLPROTO}}://{{GITURL}}/{{GITREPOPATH}} && \ ln -s /etc/debmirror/Debain-repos/MirrorListV4 /etc/debmirror/MirrorListV4 && \ ln -s /etc/debmirror/Debain-repos/MirrorListV6 /etc/debmirror/MirrorListV6 # RUN touch /etc/debmirror/MirrorListV6 && \ diff --git a/docker/mainDocker.py b/docker/mainDocker.py index 0cb0ed8..6196537 100644 --- a/docker/mainDocker.py +++ b/docker/mainDocker.py @@ -129,13 +129,13 @@ def LeJob(): ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl) - if ip4Dict == -1: + try: + for key, ip in ip4Dict.items(): + print(ip) + + fW.write(ip + "/32" + "\n") + except AttributeError: continue - - for key, ip in ip4Dict.items(): - # print(ip) - - fW.write(ip + "/32" + "\n") with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: @@ -152,14 +152,14 @@ def LeJob(): ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl) - if ip6Dict == -1: + try: + for key, ip in ip6Dict.items(): + # print(ip) + + fW.write(ip + "/128" + "\n") + except AttributeError: continue - for key, ip in ip6Dict.items(): - # print(ip) - - fW.write(ip + "/128" + "\n") - LeJob() diff --git a/docker/whatDomain.py b/docker/whatDomain.py index d0a0a4d..70cb81e 100644 --- a/docker/whatDomain.py +++ b/docker/whatDomain.py @@ -21,7 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom try: result = dns.resolver.resolve(inpDomainNameOrSomething, 'A') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1 @@ -46,7 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, " try: result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1 @@ -74,7 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP try: result = dns.resolver.resolve(whatToCheck, 'PTR') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1 diff --git a/opnsense/OPNS_MirrorListV4 b/opnsense/OPNS_MirrorListV4 new file mode 100644 index 0000000..98fbf3f --- /dev/null +++ b/opnsense/OPNS_MirrorListV4 @@ -0,0 +1,52 @@ +202.158.214.106/32 +200.201.197.203/32 +178.157.61.30/32 +216.24.182.173/32 +162.105.120.100/32 +104.21.112.1/32 +104.21.16.1/32 +104.21.32.1/32 +104.21.48.1/32 +104.21.64.1/32 +104.21.80.1/32 +104.21.96.1/32 +130.225.254.116/32 +46.32.144.253/32 +201.159.221.67/32 +201.159.221.67/32 +172.67.206.93/32 +104.21.22.179/32 +141.98.136.67/32 +37.208.104.98/32 +37.58.58.140/32 +147.52.159.50/32 +180.151.236.94/32 +78.39.136.13/32 +121.141.119.252/32 +121.141.34.226/32 +121.171.109.2/32 +220.118.162.155/32 +220.118.162.15/32 +211.205.104.177/32 +112.222.241.182/32 +112.164.39.61/32 +221.138.103.74/32 +221.142.142.68/32 +121.159.44.64/32 +196.200.160.70/32 +150.242.40.7/32 +23.147.168.189/32 +91.132.103.246/32 +46.235.145.153/32 +109.202.202.202/32 +212.237.209.20/32 +163.22.168.137/32 +5.79.108.33/32 +185.28.39.100/32 +89.149.222.99/32 +193.162.43.250/32 +212.219.56.184/32 +66.111.2.15/32 +207.244.94.80/32 +209.58.135.187/32 +169.229.200.70/32 diff --git a/opnsense/OPNS_MirrorListV6 b/opnsense/OPNS_MirrorListV6 new file mode 100644 index 0000000..e904a88 --- /dev/null +++ b/opnsense/OPNS_MirrorListV6 @@ -0,0 +1,31 @@ +2001:388:30bc:cafe::beef/128 +2001:da8:201:1120::a269:7864/128 +2606:4700:3030::6815:7001/128 +2606:4700:3030::6815:1001/128 +2606:4700:3030::6815:2001/128 +2606:4700:3030::6815:3001/128 +2606:4700:3030::6815:4001/128 +2606:4700:3030::6815:5001/128 +2606:4700:3030::6815:6001/128 +2001:878:346::116/128 +2001:470:dd11:1337:d0cd:bfff:feaa:fb37/128 +2800:68:0:bebe::3/128 +2606:4700:3036::ac43:ce5d/128 +2606:4700:3034::6815:16b3/128 +2a09:11c0:1200:210:ffff:ffff:8d62:8843/128 +2a00:d0c0:d::100/128 +2a00:c98:2030:a034::21/128 +2001:648:2c00:6c08::2/128 +2001:4310:f1::70/128 +2602:faa5:189::/128 +2602:faa5:190::/128 +2a12:d282:102:f6::1/128 +2a00:d08:1:c::153/128 +2001:1620::1620/128 +2a03:da40:2::20/128 +2001:1af8:4700:b210::33/128 +2001:1af8:5300:a010:1::1/128 +2001:630:341:12::184/128 +2604:9a00:2010:a0b8::5/128 +2605:fe80:2100:b001::5187/128 +2607:f140:0:32::70/128 diff --git a/opnsense/main.py b/opnsense/main.py index 8387031..0c7dc76 100644 --- a/opnsense/main.py +++ b/opnsense/main.py @@ -35,104 +35,63 @@ def sanitizeURL(inpurl: str): def getFreshData(): - payload = requests.get(DEBMIRRORURL) + payload = requests.get(OPNSNSMIRRORURL) soup = BeautifulSoup(payload.content, "html.parser") return soup - -def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup): - - outMirrorDict = {} - current_country = None - - # Iterate through all table rows - for table in SoupInput.find_all("table"): - for row in table.find_all("tr"): - # Check for country name in a full-row header () - strong = row.find("strong") - if strong: - country_name = strong.get_text(strip=True) - if country_name in target_countries: - current_country = country_name - else: - current_country = None - continue # move to next row - - # Check for inline country name in first column - cols = row.find_all("td") - if len(cols) >= 2: - possible_country = cols[0].get_text(strip=True) - link_tag = cols[1].find("a", href=True) - if possible_country in target_countries: - current_country = possible_country - if current_country and link_tag: - url = link_tag['href'] - if current_country not in outMirrorDict: - outMirrorDict[current_country] = [] - outMirrorDict[current_country].append(url) - - outMirrorDict.update({"Security": DEBSECURITYURL}) - outMirrorDict.update({"Extras": EXTRASURL}) - - return outMirrorDict - def LeJob(): print("Starting lookup") LeSoup = getFreshData() - LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup) - # print(LeMirrorDict) with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW: - for key, urls in LeMirrorDict.items(): - # print(urls) - if key in target_countries: - - for url in urls: - # print(url) - - if url not in fR: + for data in LeSoup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): - goodurl = sanitizeURL(url) - # print(goodurl) + url = a['href'] - ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl) + saniturl = sanitizeURL(url) - if ip4Dict == -1: - continue + # print(saniturl) + IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) - for key, ip in ip4Dict.items(): - print(ip) + # print(IPv4Dict) + try: + for key, ip in IPv4Dict.items(): + print(f"Found the ipv4: {ip}") + fW.write(ip + "/32" + "\n") - fW.write(ip + "/32" + "\n") - + # If int returned from WhatDomain then it is error. + # Error type is described via prints from whatdomain functions + except AttributeError: + continue with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: - for key, urls in LeMirrorDict.items(): - if key in target_countries: + for data in LeSoup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): - for url in urls: - - if url not in fR: + url = a['href'] - goodurl = sanitizeURL(url) - # print(goodurl) + saniturl = sanitizeURL(url) - ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl) + # print(saniturl) + IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) - if ip6Dict == -1: - continue - - for key, ip in ip6Dict.items(): - # print(ip) - - fW.write(ip + "/128" + "\n") + # print(IPv6Dict) + try: + for key, ip in IPv6Dict.items(): + print(f"Found the ipv6: {ip}") + fW.write(ip + "/128" + "\n") + # If int returned from WhatDomain then it is error. + # Error type is described via prints from whatdomain functions + except AttributeError: + continue # schedule.every().day.at("12:45").do(LeJob) # schedule.every().day.at("17:44").do(LeJob) @@ -143,68 +102,34 @@ def LeJob(): # time.sleep(30) #Wait one minute # # LeJob() +LeJob() -def DNSerrHandling(inpErrorCode: int): +# gigalist = [] - match inpErrorCode: - case -1: - return "No answer from dns server." - case -2: - # return "All nameservers failed to answer the query." - print("All nameservers failed to answer the query.\n Fix your DNS servers.") - exit(-1) - case -3: - return "The DNS query name does not exist." - case -4: - return "The DNS querry got timed out." - case _: - return "Not implemented" +# payload = requests.get(OPNSNSMIRRORURL) +# soup = BeautifulSoup(payload.content, "html.parser") +# for data in soup.find_all('div', class_='download_section'): +# for a in data.find_all('a', href=True): -gigalist = [] +# url = a['href'] -payload = requests.get(OPNSNSMIRRORURL) -soup = BeautifulSoup(payload.content, "html.parser") +# saniturl = sanitizeURL(url) -# print(soup) +# # print(saniturl) +# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) +# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) -# divs = soup.find_all("div", {"class": "download_section"}) +# # print(IPv4Dict) +# try: +# for key, ip in IPv4Dict.items(): +# print(f"Found the ipv4: {ip}") -for data in soup.find_all('div', class_='download_section'): - for a in data.find_all('a', href=True): +# for key, ip in IPv6Dict.items(): +# print(f"Found the ipv6: {ip}") +# # If int returned from WhatDomain then it is error. +# # Error type is described via prints from whatdomain functions +# except AttributeError: +# continue - url = a['href'] - - saniturl = sanitizeURL(url) - - print(saniturl) - IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) - IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) - - - if type(IPv4Dict) == int: - print(f"{DNSerrHandling(IPv4Dict)}") - continue - elif type(IPv6Dict) == int: - print(f"ERROR RESOLVING {saniturl} IPv6 address with error code {IPv6Dict}") - continue - - for key, ip in IPv4Dict.items(): - print(f"Found the ipv4: {ip}") - - for key, ip in IPv6Dict.items(): - print(f"Found the ipv6: {ip}") - - # print(a.get('href')) #for getting link - # print(a.text) #for getting text between the link - - - -# for a in soup.find_all("div", {"class": "download_section"}, "a", href=True): - - - -# h2zz = divs = soup.find_all("h3") - -# print(f" one link is: {h2zz}") \ No newline at end of file diff --git a/opnsense/whatDomain.py b/opnsense/whatDomain.py index 1b22887..70cb81e 100644 --- a/opnsense/whatDomain.py +++ b/opnsense/whatDomain.py @@ -21,13 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom try: result = dns.resolver.resolve(inpDomainNameOrSomething, 'A') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 except dns.resolver.NoNameservers: - return -2 + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 except dns.resolver.NXDOMAIN: - return -3 + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 except dns.resolver.LifetimeTimeout: - return -4 + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1 @@ -52,13 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, " try: result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 except dns.resolver.NoNameservers: - return -2 + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 except dns.resolver.NXDOMAIN: - return -3 + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 except dns.resolver.LifetimeTimeout: - return -4 + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1 @@ -86,13 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP try: result = dns.resolver.resolve(whatToCheck, 'PTR') except dns.resolver.NoAnswer: - return -1 + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 except dns.resolver.NoNameservers: - return -2 + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 except dns.resolver.NXDOMAIN: - return -3 + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 except dns.resolver.LifetimeTimeout: - return -4 + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 for i, something in enumerate(result): outDict[i] = something.to_text() # i += 1