Working OPNsense thing.

Now to integrate it into main

code/main.py

@@ -123,13 +123,13 @@ def LeJob():
 
                         ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
 
-                        if ip4Dict == -1:
+                        try:
+                            for key, ip in ip4Dict.items():
+                                print(ip)
+
+                                fW.write(ip + "/32" +  "\n")
+                        except AttributeError: 
                             continue
-
-                        for key, ip in ip4Dict.items():
-                            print(ip)
-
-                            fW.write(ip + "/32" +  "\n")
                         
 
     with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
@@ -146,14 +146,14 @@ def LeJob():
 
                         ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
 
-                        if ip6Dict == -1:
+                        try:
+                            for key, ip in ip6Dict.items():
+                                # print(ip)
+
+                                fW.write(ip + "/128" + "\n")
+                        except AttributeError:
                             continue
 
-                        for key, ip in ip6Dict.items():
-                            # print(ip)
-
-                            fW.write(ip + "/128" + "\n")
-
 
 # schedule.every().day.at("12:45").do(LeJob)
 schedule.every().day.at("17:44").do(LeJob)

code/whatDomain.py

@@ -21,7 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
     try:
         result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1
@@ -46,7 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "
     try:
         result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1
@@ -74,7 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
     try:
         result = dns.resolver.resolve(whatToCheck, 'PTR')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1

docker/Dockerfile

@@ -4,6 +4,9 @@ FROM alpine:latest
 ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
 ENV EXTRAREPOS=True
 ENV SECURITYREPOS=True
+ENV GITURLPROTO=https
+ENV GITURL=tea.shupogaki.org
+ENV GITREPOPATH=YuruC3/Debain-repos.git
 
 RUN apk update && \
     apk add python3 py3-pip git  
@@ -12,7 +15,7 @@ RUN apk update && \
 RUN mkdir /etc/debmirror
 WORKDIR /etc/debmirror/
 
-RUN git clone https://tea.shupogaki.org/YuruC3/Debain-repos && \
+RUN git clone {{GITURLPROTO}}://{{GITURL}}/{{GITREPOPATH}} && \
     ln -s /etc/debmirror/Debain-repos/MirrorListV4 /etc/debmirror/MirrorListV4 && \
     ln -s /etc/debmirror/Debain-repos/MirrorListV6 /etc/debmirror/MirrorListV6
 # RUN touch /etc/debmirror/MirrorListV6 && \

docker/mainDocker.py

@@ -129,13 +129,13 @@ def LeJob():
 
                         ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
 
-                        if ip4Dict == -1:
+                        try:
+                            for key, ip in ip4Dict.items():
+                                print(ip)
+
+                                fW.write(ip + "/32" +  "\n")
+                        except AttributeError: 
                             continue
-
-                        for key, ip in ip4Dict.items():
-                            # print(ip)
-
-                            fW.write(ip + "/32" +  "\n")
                         
 
     with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
@@ -152,14 +152,14 @@ def LeJob():
 
                         ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
 
-                        if ip6Dict == -1:
+                        try:
+                            for key, ip in ip6Dict.items():
+                                # print(ip)
+
+                                fW.write(ip + "/128" + "\n")
+                        except AttributeError:
                             continue
 
-                        for key, ip in ip6Dict.items():
-                            # print(ip)
-
-                            fW.write(ip + "/128" + "\n")
-
 
 
 LeJob()

docker/whatDomain.py

@@ -21,7 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
     try:
         result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1
@@ -46,7 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "
     try:
         result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1
@@ -74,7 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
     try:
         result = dns.resolver.resolve(whatToCheck, 'PTR')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1

opnsense/OPNS_MirrorListV4

@@ -0,0 +1,52 @@
+202.158.214.106/32
+200.201.197.203/32
+178.157.61.30/32
+216.24.182.173/32
+162.105.120.100/32
+104.21.112.1/32
+104.21.16.1/32
+104.21.32.1/32
+104.21.48.1/32
+104.21.64.1/32
+104.21.80.1/32
+104.21.96.1/32
+130.225.254.116/32
+46.32.144.253/32
+201.159.221.67/32
+201.159.221.67/32
+172.67.206.93/32
+104.21.22.179/32
+141.98.136.67/32
+37.208.104.98/32
+37.58.58.140/32
+147.52.159.50/32
+180.151.236.94/32
+78.39.136.13/32
+121.141.119.252/32
+121.141.34.226/32
+121.171.109.2/32
+220.118.162.155/32
+220.118.162.15/32
+211.205.104.177/32
+112.222.241.182/32
+112.164.39.61/32
+221.138.103.74/32
+221.142.142.68/32
+121.159.44.64/32
+196.200.160.70/32
+150.242.40.7/32
+23.147.168.189/32
+91.132.103.246/32
+46.235.145.153/32
+109.202.202.202/32
+212.237.209.20/32
+163.22.168.137/32
+5.79.108.33/32
+185.28.39.100/32
+89.149.222.99/32
+193.162.43.250/32
+212.219.56.184/32
+66.111.2.15/32
+207.244.94.80/32
+209.58.135.187/32
+169.229.200.70/32

opnsense/OPNS_MirrorListV6

@@ -0,0 +1,31 @@
+2001:388:30bc:cafe::beef/128
+2001:da8:201:1120::a269:7864/128
+2606:4700:3030::6815:7001/128
+2606:4700:3030::6815:1001/128
+2606:4700:3030::6815:2001/128
+2606:4700:3030::6815:3001/128
+2606:4700:3030::6815:4001/128
+2606:4700:3030::6815:5001/128
+2606:4700:3030::6815:6001/128
+2001:878:346::116/128
+2001:470:dd11:1337:d0cd:bfff:feaa:fb37/128
+2800:68:0:bebe::3/128
+2606:4700:3036::ac43:ce5d/128
+2606:4700:3034::6815:16b3/128
+2a09:11c0:1200:210:ffff:ffff:8d62:8843/128
+2a00:d0c0:d::100/128
+2a00:c98:2030:a034::21/128
+2001:648:2c00:6c08::2/128
+2001:4310:f1::70/128
+2602:faa5:189::/128
+2602:faa5:190::/128
+2a12:d282:102:f6::1/128
+2a00:d08:1:c::153/128
+2001:1620::1620/128
+2a03:da40:2::20/128
+2001:1af8:4700:b210::33/128
+2001:1af8:5300:a010:1::1/128
+2001:630:341:12::184/128
+2604:9a00:2010:a0b8::5/128
+2605:fe80:2100:b001::5187/128
+2607:f140:0:32::70/128

opnsense/main.py

@@ -35,104 +35,63 @@ def sanitizeURL(inpurl: str):
 
 
 def getFreshData():
-    payload = requests.get(DEBMIRRORURL)
+    payload = requests.get(OPNSNSMIRRORURL)
     soup = BeautifulSoup(payload.content, "html.parser")
     return soup
 
-
-def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
-
-    outMirrorDict = {}
-    current_country = None
-
-    # Iterate through all table rows
-    for table in SoupInput.find_all("table"):
-        for row in table.find_all("tr"):
-            # Check for country name in a full-row header (<strong><big>)
-            strong = row.find("strong")
-            if strong:
-                country_name = strong.get_text(strip=True)
-                if country_name in target_countries:
-                    current_country = country_name
-                else:
-                    current_country = None
-                continue  # move to next row
-
-            # Check for inline country name in first column
-            cols = row.find_all("td")
-            if len(cols) >= 2:
-                possible_country = cols[0].get_text(strip=True)
-                link_tag = cols[1].find("a", href=True)
-                if possible_country in target_countries:
-                    current_country = possible_country
-                if current_country and link_tag:
-                    url = link_tag['href']
-                    if current_country not in outMirrorDict:
-                        outMirrorDict[current_country] = []
-                    outMirrorDict[current_country].append(url)
-
-    outMirrorDict.update({"Security": DEBSECURITYURL})
-    outMirrorDict.update({"Extras": EXTRASURL})
-
-    return outMirrorDict
-
 def LeJob():
 
     print("Starting lookup")
 
     LeSoup = getFreshData()
 
-    LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)
-
     # print(LeMirrorDict)
 
     with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
 
-        for key, urls in LeMirrorDict.items():
+        for data in LeSoup.find_all('div', class_='download_section'):
-            # print(urls)
+            for a in data.find_all('a', href=True):
-            if key in target_countries:
-                
-                for url in urls:
-                    # print(url)
-                    
-                    if url not in fR:
 
-                        goodurl = sanitizeURL(url)
+                url = a['href']
-                        # print(goodurl)
 
-                        ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
+                saniturl = sanitizeURL(url)
 
-                        if ip4Dict == -1:
+                # print(saniturl)
-                            continue
+                IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
 
-                        for key, ip in ip4Dict.items():
+                # print(IPv4Dict)
-                            print(ip)
+                try:
+                    for key, ip in IPv4Dict.items():
+                        print(f"Found the ipv4: {ip}")
+                        fW.write(ip + "/32" +  "\n")
 
-                            fW.write(ip + "/32" +  "\n")
+                # If int returned from WhatDomain then it is error.
-                        
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue 
 
     with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
 
-        for key, urls in LeMirrorDict.items():
+        for data in LeSoup.find_all('div', class_='download_section'):
-            if key in target_countries:
+            for a in data.find_all('a', href=True):
                 
-                for url in urls:
+                url = a['href']
-                    
-                    if url not in fR:
 
-                        goodurl = sanitizeURL(url)
+                saniturl = sanitizeURL(url)
-                        # print(goodurl)
 
-                        ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
+                # print(saniturl)
+                IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
 
-                        if ip6Dict == -1:
+                # print(IPv6Dict)
-                            continue
+                try:
-
+                    for key, ip in IPv6Dict.items():
-                        for key, ip in ip6Dict.items():
+                        print(f"Found the ipv6: {ip}")
-                            # print(ip)
+                        fW.write(ip + "/128" + "\n")
-
-                            fW.write(ip + "/128" + "\n")
 
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue
 
 # schedule.every().day.at("12:45").do(LeJob)
 # schedule.every().day.at("17:44").do(LeJob)
@@ -143,68 +102,34 @@ def LeJob():
 #     time.sleep(30)  #Wait one minute
 #     # LeJob()
 
+LeJob()
 
-def DNSerrHandling(inpErrorCode: int):
+# gigalist = []
 
-    match inpErrorCode:
+# payload = requests.get(OPNSNSMIRRORURL)
-        case -1:
+# soup = BeautifulSoup(payload.content, "html.parser")
-            return "No answer from dns server."
-        case -2:
-            # return "All nameservers failed to answer the query."
-            print("All nameservers failed to answer the query.\n Fix your DNS servers.")
-            exit(-1)
-        case -3:
-            return "The DNS query name does not exist."
-        case -4:
-            return "The DNS querry got timed out."
-        case _:
-            return "Not implemented"
 
 
+# for data in soup.find_all('div', class_='download_section'):
+#     for a in data.find_all('a', href=True):
 
-gigalist = []
+#         url = a['href']
 
-payload = requests.get(OPNSNSMIRRORURL)
+#         saniturl = sanitizeURL(url)
-soup = BeautifulSoup(payload.content, "html.parser")
 
-# print(soup)
+#         # print(saniturl)
+#         IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
+#         IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
 
-# divs = soup.find_all("div", {"class": "download_section"})
+#         # print(IPv4Dict)
+#         try:
+#             for key, ip in IPv4Dict.items():
+#                 print(f"Found the ipv4: {ip}")
 
-for data in soup.find_all('div', class_='download_section'):
+#             for key, ip in IPv6Dict.items():
-    for a in data.find_all('a', href=True):
+#                 print(f"Found the ipv6: {ip}")
+#         # If int returned from WhatDomain then it is error.
+#         # Error type is described via prints from whatdomain functions
+#         except AttributeError:
+#             continue
 
-        url = a['href']
-
-        saniturl = sanitizeURL(url)
-
-        print(saniturl)
-        IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
-        IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
-
-
-        if type(IPv4Dict) == int:
-            print(f"{DNSerrHandling(IPv4Dict)}")
-            continue
-        elif type(IPv6Dict) == int:
-            print(f"ERROR RESOLVING {saniturl} IPv6 address with error code {IPv6Dict}")
-            continue
-        
-        for key, ip in IPv4Dict.items():
-            print(f"Found the ipv4: {ip}")
-
-        for key, ip in IPv6Dict.items():
-            print(f"Found the ipv6: {ip}")
-
-        # print(a.get('href')) #for getting link
-        # print(a.text) #for getting text between the link
-
-
-
-# for a in soup.find_all("div", {"class": "download_section"}, "a", href=True):
-
-
-
-# h2zz = divs = soup.find_all("h3")
-
-# print(f" one link is: {h2zz}")

opnsense/whatDomain.py

@@ -21,13 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
     try:
         result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
     except dns.resolver.NoNameservers:
-        return -2
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
     except dns.resolver.NXDOMAIN:
-        return -3
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
     except dns.resolver.LifetimeTimeout:
-        return -4
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1
@@ -52,13 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "
     try:
         result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
     except dns.resolver.NoNameservers:
-        return -2
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
     except dns.resolver.NXDOMAIN:
-        return -3
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
     except dns.resolver.LifetimeTimeout:
-        return -4
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1
@@ -86,13 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
     try:
         result = dns.resolver.resolve(whatToCheck, 'PTR')
     except dns.resolver.NoAnswer:
-        return -1
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
     except dns.resolver.NoNameservers:
-        return -2
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
     except dns.resolver.NXDOMAIN:
-        return -3
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
     except dns.resolver.LifetimeTimeout:
-        return -4
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
         # i += 1