YuruC3/Debain-repos
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects Releases Wiki Activity

Working OPNsense thing.

Browse Source 
Now to integrate it into main
This commit is contained in:
YuruC3 2025-05-14 09:57:06 +02:00
parent 6beadaba5f
commit 17423793f1
9 changed files with 289 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
code/main.py
View File

@ -123,13 +123,13 @@ def LeJob():
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
if ip4Dict == -1:
try:
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/32" + "\n")
except AttributeError:
continue
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/32" + "\n")
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
@ -146,14 +146,14 @@ def LeJob():
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
if ip6Dict == -1:
try:
for key, ip in ip6Dict.items():
# print(ip)
fW.write(ip + "/128" + "\n")
except AttributeError:
continue
for key, ip in ip6Dict.items():
# print(ip)
fW.write(ip + "/128" + "\n")
# schedule.every().day.at("12:45").do(LeJob)
schedule.every().day.at("17:44").do(LeJob)

48
code/whatDomain.py
View File

@ -21,7 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
try:
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
@ -46,7 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "
try:
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
@ -74,7 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
try:
result = dns.resolver.resolve(whatToCheck, 'PTR')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1

5
docker/Dockerfile
View File

@ -4,6 +4,9 @@ FROM alpine:latest
ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
ENV EXTRAREPOS=True
ENV SECURITYREPOS=True
ENV GITURLPROTO=https
ENV GITURL=tea.shupogaki.org
ENV GITREPOPATH=YuruC3/Debain-repos.git
RUN apk update && \
apk add python3 py3-pip git
@ -12,7 +15,7 @@ RUN apk update && \
RUN mkdir /etc/debmirror
WORKDIR /etc/debmirror/
RUN git clone https://tea.shupogaki.org/YuruC3/Debain-repos && \
RUN git clone {{GITURLPROTO}}://{{GITURL}}/{{GITREPOPATH}} && \
ln -s /etc/debmirror/Debain-repos/MirrorListV4 /etc/debmirror/MirrorListV4 && \
ln -s /etc/debmirror/Debain-repos/MirrorListV6 /etc/debmirror/MirrorListV6
# RUN touch /etc/debmirror/MirrorListV6 && \

24
docker/mainDocker.py
View File

@ -129,13 +129,13 @@ def LeJob():
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
if ip4Dict == -1:
try:
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/32" + "\n")
except AttributeError:
continue
for key, ip in ip4Dict.items():
# print(ip)
fW.write(ip + "/32" + "\n")
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
@ -152,14 +152,14 @@ def LeJob():
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
if ip6Dict == -1:
try:
for key, ip in ip6Dict.items():
# print(ip)
fW.write(ip + "/128" + "\n")
except AttributeError:
continue
for key, ip in ip6Dict.items():
# print(ip)
fW.write(ip + "/128" + "\n")
LeJob()

48
docker/whatDomain.py
View File

@ -21,7 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
try:
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
@ -46,7 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "
try:
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
@ -74,7 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
try:
result = dns.resolver.resolve(whatToCheck, 'PTR')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1

52
opnsense/OPNS_MirrorListV4 Normal file
View File

@ -0,0 +1,52 @@
202.158.214.106/32
200.201.197.203/32
178.157.61.30/32
216.24.182.173/32
162.105.120.100/32
104.21.112.1/32
104.21.16.1/32
104.21.32.1/32
104.21.48.1/32
104.21.64.1/32
104.21.80.1/32
104.21.96.1/32
130.225.254.116/32
46.32.144.253/32
201.159.221.67/32
201.159.221.67/32
172.67.206.93/32
104.21.22.179/32
141.98.136.67/32
37.208.104.98/32
37.58.58.140/32
147.52.159.50/32
180.151.236.94/32
78.39.136.13/32
121.141.119.252/32
121.141.34.226/32
121.171.109.2/32
220.118.162.155/32
220.118.162.15/32
211.205.104.177/32
112.222.241.182/32
112.164.39.61/32
221.138.103.74/32
221.142.142.68/32
121.159.44.64/32
196.200.160.70/32
150.242.40.7/32
23.147.168.189/32
91.132.103.246/32
46.235.145.153/32
109.202.202.202/32
212.237.209.20/32
163.22.168.137/32
5.79.108.33/32
185.28.39.100/32
89.149.222.99/32
193.162.43.250/32
212.219.56.184/32
66.111.2.15/32
207.244.94.80/32
209.58.135.187/32
169.229.200.70/32

31
opnsense/OPNS_MirrorListV6 Normal file
View File

@ -0,0 +1,31 @@
2001:388:30bc:cafe::beef/128
2001:da8:201:1120::a269:7864/128
2606:4700:3030::6815:7001/128
2606:4700:3030::6815:1001/128
2606:4700:3030::6815:2001/128
2606:4700:3030::6815:3001/128
2606:4700:3030::6815:4001/128
2606:4700:3030::6815:5001/128
2606:4700:3030::6815:6001/128
2001:878:346::116/128
2001:470:dd11:1337:d0cd:bfff:feaa:fb37/128
2800:68:0:bebe::3/128
2606:4700:3036::ac43:ce5d/128
2606:4700:3034::6815:16b3/128
2a09:11c0:1200:210:ffff:ffff:8d62:8843/128
2a00:d0c0:d::100/128
2a00:c98:2030:a034::21/128
2001:648:2c00:6c08::2/128
2001:4310:f1::70/128
2602:faa5:189::/128
2602:faa5:190::/128
2a12:d282:102:f6::1/128
2a00:d08:1:c::153/128
2001:1620::1620/128
2a03:da40:2::20/128
2001:1af8:4700:b210::33/128
2001:1af8:5300:a010:1::1/128
2001:630:341:12::184/128
2604:9a00:2010:a0b8::5/128
2605:fe80:2100:b001::5187/128
2607:f140:0:32::70/128

179
opnsense/main.py
View File

@ -35,104 +35,63 @@ def sanitizeURL(inpurl: str):
def getFreshData():
payload = requests.get(DEBMIRRORURL)
payload = requests.get(OPNSNSMIRRORURL)
soup = BeautifulSoup(payload.content, "html.parser")
return soup
def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
outMirrorDict = {}
current_country = None
# Iterate through all table rows
for table in SoupInput.find_all("table"):
for row in table.find_all("tr"):
# Check for country name in a full-row header (<strong><big>)
strong = row.find("strong")
if strong:
country_name = strong.get_text(strip=True)
if country_name in target_countries:
current_country = country_name
else:
current_country = None
continue # move to next row
# Check for inline country name in first column
cols = row.find_all("td")
if len(cols) >= 2:
possible_country = cols[0].get_text(strip=True)
link_tag = cols[1].find("a", href=True)
if possible_country in target_countries:
current_country = possible_country
if current_country and link_tag:
url = link_tag['href']
if current_country not in outMirrorDict:
outMirrorDict[current_country] = []
outMirrorDict[current_country].append(url)
outMirrorDict.update({"Security": DEBSECURITYURL})
outMirrorDict.update({"Extras": EXTRASURL})
return outMirrorDict
def LeJob():
print("Starting lookup")
LeSoup = getFreshData()
LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)
# print(LeMirrorDict)
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
for key, urls in LeMirrorDict.items():
# print(urls)
if key in target_countries:
for url in urls:
# print(url)
if url not in fR:
for data in LeSoup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
goodurl = sanitizeURL(url)
# print(goodurl)
url = a['href']
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
saniturl = sanitizeURL(url)
if ip4Dict == -1:
continue
# print(saniturl)
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
for key, ip in ip4Dict.items():
print(ip)
# print(IPv4Dict)
try:
for key, ip in IPv4Dict.items():
print(f"Found the ipv4: {ip}")
fW.write(ip + "/32" + "\n")
fW.write(ip + "/32" + "\n")
# If int returned from WhatDomain then it is error.
# Error type is described via prints from whatdomain functions
except AttributeError:
continue
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
for key, urls in LeMirrorDict.items():
if key in target_countries:
for data in LeSoup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
for url in urls:
if url not in fR:
url = a['href']
goodurl = sanitizeURL(url)
# print(goodurl)
saniturl = sanitizeURL(url)
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
# print(saniturl)
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
if ip6Dict == -1:
continue
for key, ip in ip6Dict.items():
# print(ip)
fW.write(ip + "/128" + "\n")
# print(IPv6Dict)
try:
for key, ip in IPv6Dict.items():
print(f"Found the ipv6: {ip}")
fW.write(ip + "/128" + "\n")
# If int returned from WhatDomain then it is error.
# Error type is described via prints from whatdomain functions
except AttributeError:
continue
# schedule.every().day.at("12:45").do(LeJob)
# schedule.every().day.at("17:44").do(LeJob)
@ -143,68 +102,34 @@ def LeJob():
# time.sleep(30) #Wait one minute
# # LeJob()
LeJob()
def DNSerrHandling(inpErrorCode: int):
# gigalist = []
match inpErrorCode:
case -1:
return "No answer from dns server."
case -2:
# return "All nameservers failed to answer the query."
print("All nameservers failed to answer the query.\n Fix your DNS servers.")
exit(-1)
case -3:
return "The DNS query name does not exist."
case -4:
return "The DNS querry got timed out."
case _:
return "Not implemented"
# payload = requests.get(OPNSNSMIRRORURL)
# soup = BeautifulSoup(payload.content, "html.parser")
# for data in soup.find_all('div', class_='download_section'):
# for a in data.find_all('a', href=True):
gigalist = []
# url = a['href']
payload = requests.get(OPNSNSMIRRORURL)
soup = BeautifulSoup(payload.content, "html.parser")
# saniturl = sanitizeURL(url)
# print(soup)
# # print(saniturl)
# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
# divs = soup.find_all("div", {"class": "download_section"})
# # print(IPv4Dict)
# try:
# for key, ip in IPv4Dict.items():
# print(f"Found the ipv4: {ip}")
for data in soup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
# for key, ip in IPv6Dict.items():
# print(f"Found the ipv6: {ip}")
# # If int returned from WhatDomain then it is error.
# # Error type is described via prints from whatdomain functions
# except AttributeError:
# continue
url = a['href']
saniturl = sanitizeURL(url)
print(saniturl)
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
if type(IPv4Dict) == int:
print(f"{DNSerrHandling(IPv4Dict)}")
continue
elif type(IPv6Dict) == int:
print(f"ERROR RESOLVING {saniturl} IPv6 address with error code {IPv6Dict}")
continue
for key, ip in IPv4Dict.items():
print(f"Found the ipv4: {ip}")
for key, ip in IPv6Dict.items():
print(f"Found the ipv6: {ip}")
# print(a.get('href')) #for getting link
# print(a.text) #for getting text between the link
# for a in soup.find_all("div", {"class": "download_section"}, "a", href=True):
# h2zz = divs = soup.find_all("h3")
# print(f" one link is: {h2zz}")

48
opnsense/whatDomain.py
View File

@ -21,13 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
try:
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
return -2
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
return -3
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
return -4
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
@ -52,13 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "
try:
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
return -2
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
return -3
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
return -4
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
@ -86,13 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
try:
result = dns.resolver.resolve(whatToCheck, 'PTR')
except dns.resolver.NoAnswer:
return -1
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
return -2
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
return -3
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
return -4
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1