Trying to integrate all functions into one container

This commit is contained in:
2026-01-13 19:13:22 +01:00
parent 880da3b79d
commit d13a36ee23
11 changed files with 928 additions and 0 deletions

View File

@@ -0,0 +1,96 @@
import requests, schedule, time, os
# from bs4 import BeautifulSoup
from whatDomain import *
EXTRAURL = []
repoListFile = open(os.getenv("REPOFILE", "customRepoList.list"), 'r')
i = 0
for repoUrl in repoListFile:
i += 1
print(repoUrl.strip())
EXTRAURL.append(repoUrl.strip())
repoListFile.close()
CUSTOMIPV4FILENAME = str(os.getenv("CUSTOMIPV4FILENAME", "CustomMirrorListV4"))
CUSTOMIPV6FILENAME = str(os.getenv("CUSTOMIPV6FILENAME", "CustomMirrorListV6"))
# EXTRAURL = list(os.getenv("EXTRAURL", "https://mdu.se/"))
IPv4FILE = f"/etc/debmirror/{CUSTOMIPV4FILENAME}"
IPv6FILE = f"/etc/debmirror/{CUSTOMIPV6FILENAME}"
def sanitizeURL(inpurl: str):
if "https://" in inpurl:
outurl = inpurl[8:]
elif "http://" in inpurl:
outurl = inpurl[7:]
elif "" in inpurl:
return 7
elif "http://" or "https://" not in url:
outurl = inpurl
else:
return inpurl
i = 0
for char in outurl:
i += 1
if char == "/":
outurl = outurl[:i]
if char == "/":
outurl = outurl[:-1]
return outurl
def LeJob():
print("Starting lookup")
# print(LeMirrorDict)
with open(IPv4FILE, "w",) as fW:
for url in EXTRAURL:
goodurl = sanitizeURL(url)
# print(goodurl)
if goodurl == 7:
continue
# ip4Dict = ermWhatATheIpFromDomainYaCrazy(url)
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
try:
for key, ip in ip4Dict.items():
print(ip + "/32")
fW.write(ip + "/32" + "\n")
except AttributeError:
continue
with open(IPv6FILE, "w",) as fW:
for url in EXTRAURL:
goodurl = sanitizeURL(url)
if goodurl == 7:
continue
# print(goodurl)
# ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url)
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
try:
for key, ip in ip6Dict.items():
print(ip + "/128")
fW.write(ip + "/128" + "\n")
except AttributeError:
continue
LeJob()
print("Done")

View File

@@ -0,0 +1,165 @@
import requests, schedule, time, os
from bs4 import BeautifulSoup
from whatDomain import *
DEBMIRRORURL = str(os.getenv("DEBMIRRORURL", "https://www.debian.org/mirror/list"))
DEBSECURITYURL = str(os.getenv("DEBSECURITYURL", "https://security.debian.org/debian-security/"))
EXTRASURL = ["https://download.docker.com/linux/debian/",
# Double just to be sure. Even though they point to the same IP
"http://download.proxmox.com/debian/",
"https://enterprise.proxmox.com/debian/pve/",
# That's for nvidia docker toolkit something something
# stuff that makes containers use nvenc and shit
"https://nvidia.github.io/libnvidia-container/stable/deb/",
"https://nvidia.github.io/libnvidia-container/experimental/deb/"]
IPv4FILE = "/etc/debmirror/MirrorListV4"
IPv6FILE = "/etc/debmirror/MirrorListV6"
# Define EU and American countries as well as Security for security updates
target_countries = set([
# Europe
"Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark",
"Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland",
"Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland",
"Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland",
"United Kingdom", "Moldova",
# America
"Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
"Mexico", "Peru", "United States", "Uruguay", "Venezuela",
# Others
"Security", "Extras"
])
def sanitizeURL(inpurl: str):
if "https://" in inpurl:
outurl = inpurl[8:]
elif "http://" in inpurl:
outurl = inpurl[7:]
elif "" in inpurl:
return 7
elif "http://" or "https://" not in url:
outurl = inpurl
else:
return inpurl
i = 0
for char in outurl:
i += 1
if char == "/":
outurl = outurl[:i]
if char == "/":
outurl = outurl[:-1]
return outurl
def getFreshData():
payload = requests.get(DEBMIRRORURL)
soup = BeautifulSoup(payload.content, "html.parser")
return soup
def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
outMirrorDict = {}
current_country = None
# Iterate through all table rows
for table in SoupInput.find_all("table"):
for row in table.find_all("tr"):
# Check for country name in a full-row header (<strong><big>)
strong = row.find("strong")
if strong:
country_name = strong.get_text(strip=True)
if country_name in target_countries:
current_country = country_name
else:
current_country = None
continue # move to next row
# Check for inline country name in first column
cols = row.find_all("td")
if len(cols) >= 2:
possible_country = cols[0].get_text(strip=True)
link_tag = cols[1].find("a", href=True)
if possible_country in target_countries:
current_country = possible_country
if current_country and link_tag:
url = link_tag['href']
if current_country not in outMirrorDict:
outMirrorDict[current_country] = []
outMirrorDict[current_country].append(url)
if os.environ["SECURITYREPOS"]:
outMirrorDict.update({"Security": DEBSECURITYURL})
if os.environ["EXTRAREPOS"]:
outMirrorDict.update({"Extras": EXTRASURL})
return outMirrorDict
def LeJob():
print("Starting lookup")
LeSoup = getFreshData()
LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)
# print(LeMirrorDict)
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
for key, urls in LeMirrorDict.items():
# print(urls)
if key in target_countries:
for url in urls:
# print(url)
if url not in fR:
goodurl = sanitizeURL(url)
# print(goodurl)
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
try:
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/32" + "\n")
except AttributeError:
continue
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
for key, urls in LeMirrorDict.items():
if key in target_countries:
for url in urls:
if url not in fR:
goodurl = sanitizeURL(url)
# print(goodurl)
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
try:
for key, ip in ip6Dict.items():
# print(ip)
fW.write(ip + "/128" + "\n")
except AttributeError:
continue
LeJob()
print("Done")

View File

@@ -0,0 +1,135 @@
import requests, schedule, time
from bs4 import BeautifulSoup
from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
OPNSNSMIRRORURL = str(os.getenv("OPNSNSMIRRORURL", "https://opnsense.org/download/#full-mirror-listing"))
IPv4FILE = str(os.getenv("IPv4FILE", "/etc/debmirror/OPNS_MirrorListV4"))
IPv6FILE = str(os.getenv("IPv6FILE", "/etc/debmirror/OPNS_MirrorListV6"))
def sanitizeURL(inpurl: str):
if not "/" in inpurl[:-1]:
inpurl += "/"
if "https://" in inpurl:
outurl = inpurl[8:]
elif "http://" in inpurl:
outurl = inpurl[7:]
elif "http://" or "https://" not in url:
outurl = inpurl
else:
return -1
# how the fuck does it work?
# I mean I wrote this but I don't know why does it work.
i = 0
for char in outurl:
i += 1
if char == "/":
outurl = outurl[:i]
if char == "/":
outurl = outurl[:-1]
return outurl
def getFreshData():
payload = requests.get(OPNSNSMIRRORURL)
soup = BeautifulSoup(payload.content, "html.parser")
return soup
def LeJob():
print("Starting lookup")
LeSoup = getFreshData()
# print(LeMirrorDict)
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
for data in LeSoup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
url = a['href']
saniturl = sanitizeURL(url)
# print(saniturl)
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
# print(IPv4Dict)
try:
for key, ip in IPv4Dict.items():
print(f"Found the ipv4: {ip}")
fW.write(ip + "/32" + "\n")
# If int returned from WhatDomain then it is error.
# Error type is described via prints from whatdomain functions
except AttributeError:
continue
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
for data in LeSoup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
url = a['href']
saniturl = sanitizeURL(url)
# print(saniturl)
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
# print(IPv6Dict)
try:
for key, ip in IPv6Dict.items():
print(f"Found the ipv6: {ip}")
fW.write(ip + "/128" + "\n")
# If int returned from WhatDomain then it is error.
# Error type is described via prints from whatdomain functions
except AttributeError:
continue
# schedule.every().day.at("12:45").do(LeJob)
# schedule.every().day.at("17:44").do(LeJob)
# while True:
# schedule.run_pending()
# print("Waiting...")
# time.sleep(30) #Wait one minute
# # LeJob()
LeJob()
# gigalist = []
# payload = requests.get(OPNSNSMIRRORURL)
# soup = BeautifulSoup(payload.content, "html.parser")
# for data in soup.find_all('div', class_='download_section'):
# for a in data.find_all('a', href=True):
# url = a['href']
# saniturl = sanitizeURL(url)
# # print(saniturl)
# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
# # print(IPv4Dict)
# try:
# for key, ip in IPv4Dict.items():
# print(f"Found the ipv4: {ip}")
# for key, ip in IPv6Dict.items():
# print(f"Found the ipv6: {ip}")
# # If int returned from WhatDomain then it is error.
# # Error type is described via prints from whatdomain functions
# except AttributeError:
# continue

View File

@@ -0,0 +1,135 @@
import requests, schedule, time
from bs4 import BeautifulSoup
from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing"
IPv4FILE = "./OPNS_MirrorListV4"
IPv6FILE = "./OPNS_MirrorListV6"
def sanitizeURL(inpurl: str):
if not "/" in inpurl[:-1]:
inpurl += "/"
if "https://" in inpurl:
outurl = inpurl[8:]
elif "http://" in inpurl:
outurl = inpurl[7:]
elif "http://" or "https://" not in url:
outurl = inpurl
else:
return -1
# how the fuck does it work?
# I mean I wrote this but I don't know why does it work.
i = 0
for char in outurl:
i += 1
if char == "/":
outurl = outurl[:i]
if char == "/":
outurl = outurl[:-1]
return outurl
def getFreshData():
payload = requests.get(OPNSNSMIRRORURL)
soup = BeautifulSoup(payload.content, "html.parser")
return soup
def LeJob():
print("Starting lookup")
LeSoup = getFreshData()
# print(LeMirrorDict)
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
for data in LeSoup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
url = a['href']
saniturl = sanitizeURL(url)
# print(saniturl)
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
# print(IPv4Dict)
try:
for key, ip in IPv4Dict.items():
print(f"Found the ipv4: {ip}")
fW.write(ip + "/32" + "\n")
# If int returned from WhatDomain then it is error.
# Error type is described via prints from whatdomain functions
except AttributeError:
continue
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
for data in LeSoup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
url = a['href']
saniturl = sanitizeURL(url)
# print(saniturl)
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
# print(IPv6Dict)
try:
for key, ip in IPv6Dict.items():
print(f"Found the ipv6: {ip}")
fW.write(ip + "/128" + "\n")
# If int returned from WhatDomain then it is error.
# Error type is described via prints from whatdomain functions
except AttributeError:
continue
# schedule.every().day.at("12:45").do(LeJob)
# schedule.every().day.at("17:44").do(LeJob)
# while True:
# schedule.run_pending()
# print("Waiting...")
# time.sleep(30) #Wait one minute
# # LeJob()
LeJob()
# gigalist = []
# payload = requests.get(OPNSNSMIRRORURL)
# soup = BeautifulSoup(payload.content, "html.parser")
# for data in soup.find_all('div', class_='download_section'):
# for a in data.find_all('a', href=True):
# url = a['href']
# saniturl = sanitizeURL(url)
# # print(saniturl)
# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
# # print(IPv4Dict)
# try:
# for key, ip in IPv4Dict.items():
# print(f"Found the ipv4: {ip}")
# for key, ip in IPv6Dict.items():
# print(f"Found the ipv6: {ip}")
# # If int returned from WhatDomain then it is error.
# # Error type is described via prints from whatdomain functions
# except AttributeError:
# continue

View File

@@ -0,0 +1,129 @@
#from nslookup import Nslookup
from typing import Optional, Annotated
import dns, dns.resolver
# https://www.codeunderscored.com/nslookup-python/
def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
#dns_query = Nslookup()
"""
Tells you what IPv4 address/es a domain point to.
Returns:
dict: A dictionary with IP addresses associated with that domain.
"""
# i = 0
outDict = {}
#result = dns_query.dns_lookup("example.com")
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
try:
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
except dns.resolver.NoAnswer:
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
return outDict
def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
#dns_query = Nslookup()
"""
Tells you what IPv6 address/es a domain point to.
Returns:
dict: A dictionary with IP addresses associated with that domain.
"""
# i = 0
outDict = {}
#result = dns_query.dns_lookup("example.com")
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
try:
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
except dns.resolver.NoAnswer:
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
return outDict
def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]):
#dns_query = Nslookup()
"""
Tells you what IPv6 address/es a domain point to.
Returns:
dict: A dictionary with IP addresses associated with that domain.
"""
whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa"
# i = 0
outDict = {}
#result = dns_query.dns_lookup("example.com")
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
try:
result = dns.resolver.resolve(whatToCheck, 'PTR')
except dns.resolver.NoAnswer:
print("\nDNS ERROR")
print("No answer from dns server.\n")
return 1
except dns.resolver.NoNameservers:
print("\nDNS ERROR")
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
return 1
except dns.resolver.NXDOMAIN:
print("\nDNS ERROR")
print("The DNS query name does not exist.\n")
return 1
except dns.resolver.LifetimeTimeout:
print("\nDNS ERROR")
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
return 1
for i, something in enumerate(result):
outDict[i] = something.to_text()
# i += 1
return outDict
#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))