Working OPNsense thing.
Now to integrate it into main
This commit is contained in:
YuruC3
179
opnsense/main.py
179
opnsense/main.py
@@ -35,104 +35,63 @@ def sanitizeURL(inpurl: str):
|
||||
|
||||
|
||||
def getFreshData():
|
||||
payload = requests.get(DEBMIRRORURL)
|
||||
payload = requests.get(OPNSNSMIRRORURL)
|
||||
soup = BeautifulSoup(payload.content, "html.parser")
|
||||
return soup
|
||||
|
||||
|
||||
def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
|
||||
|
||||
outMirrorDict = {}
|
||||
current_country = None
|
||||
|
||||
# Iterate through all table rows
|
||||
for table in SoupInput.find_all("table"):
|
||||
for row in table.find_all("tr"):
|
||||
# Check for country name in a full-row header (<strong><big>)
|
||||
strong = row.find("strong")
|
||||
if strong:
|
||||
country_name = strong.get_text(strip=True)
|
||||
if country_name in target_countries:
|
||||
current_country = country_name
|
||||
else:
|
||||
current_country = None
|
||||
continue # move to next row
|
||||
|
||||
# Check for inline country name in first column
|
||||
cols = row.find_all("td")
|
||||
if len(cols) >= 2:
|
||||
possible_country = cols[0].get_text(strip=True)
|
||||
link_tag = cols[1].find("a", href=True)
|
||||
if possible_country in target_countries:
|
||||
current_country = possible_country
|
||||
if current_country and link_tag:
|
||||
url = link_tag['href']
|
||||
if current_country not in outMirrorDict:
|
||||
outMirrorDict[current_country] = []
|
||||
outMirrorDict[current_country].append(url)
|
||||
|
||||
outMirrorDict.update({"Security": DEBSECURITYURL})
|
||||
outMirrorDict.update({"Extras": EXTRASURL})
|
||||
|
||||
return outMirrorDict
|
||||
|
||||
def LeJob():
|
||||
|
||||
print("Starting lookup")
|
||||
|
||||
LeSoup = getFreshData()
|
||||
|
||||
LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)
|
||||
|
||||
# print(LeMirrorDict)
|
||||
|
||||
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
|
||||
|
||||
for key, urls in LeMirrorDict.items():
|
||||
# print(urls)
|
||||
if key in target_countries:
|
||||
|
||||
for url in urls:
|
||||
# print(url)
|
||||
|
||||
if url not in fR:
|
||||
for data in LeSoup.find_all('div', class_='download_section'):
|
||||
for a in data.find_all('a', href=True):
|
||||
|
||||
goodurl = sanitizeURL(url)
|
||||
# print(goodurl)
|
||||
url = a['href']
|
||||
|
||||
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
|
||||
saniturl = sanitizeURL(url)
|
||||
|
||||
if ip4Dict == -1:
|
||||
continue
|
||||
# print(saniturl)
|
||||
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
for key, ip in ip4Dict.items():
|
||||
print(ip)
|
||||
# print(IPv4Dict)
|
||||
try:
|
||||
for key, ip in IPv4Dict.items():
|
||||
print(f"Found the ipv4: {ip}")
|
||||
fW.write(ip + "/32" + "\n")
|
||||
|
||||
fW.write(ip + "/32" + "\n")
|
||||
|
||||
# If int returned from WhatDomain then it is error.
|
||||
# Error type is described via prints from whatdomain functions
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
|
||||
|
||||
for key, urls in LeMirrorDict.items():
|
||||
if key in target_countries:
|
||||
for data in LeSoup.find_all('div', class_='download_section'):
|
||||
for a in data.find_all('a', href=True):
|
||||
|
||||
for url in urls:
|
||||
|
||||
if url not in fR:
|
||||
url = a['href']
|
||||
|
||||
goodurl = sanitizeURL(url)
|
||||
# print(goodurl)
|
||||
saniturl = sanitizeURL(url)
|
||||
|
||||
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
|
||||
# print(saniturl)
|
||||
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
if ip6Dict == -1:
|
||||
continue
|
||||
|
||||
for key, ip in ip6Dict.items():
|
||||
# print(ip)
|
||||
|
||||
fW.write(ip + "/128" + "\n")
|
||||
# print(IPv6Dict)
|
||||
try:
|
||||
for key, ip in IPv6Dict.items():
|
||||
print(f"Found the ipv6: {ip}")
|
||||
fW.write(ip + "/128" + "\n")
|
||||
|
||||
# If int returned from WhatDomain then it is error.
|
||||
# Error type is described via prints from whatdomain functions
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
# schedule.every().day.at("12:45").do(LeJob)
|
||||
# schedule.every().day.at("17:44").do(LeJob)
|
||||
@@ -143,68 +102,34 @@ def LeJob():
|
||||
# time.sleep(30) #Wait one minute
|
||||
# # LeJob()
|
||||
|
||||
LeJob()
|
||||
|
||||
def DNSerrHandling(inpErrorCode: int):
|
||||
# gigalist = []
|
||||
|
||||
match inpErrorCode:
|
||||
case -1:
|
||||
return "No answer from dns server."
|
||||
case -2:
|
||||
# return "All nameservers failed to answer the query."
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.")
|
||||
exit(-1)
|
||||
case -3:
|
||||
return "The DNS query name does not exist."
|
||||
case -4:
|
||||
return "The DNS querry got timed out."
|
||||
case _:
|
||||
return "Not implemented"
|
||||
# payload = requests.get(OPNSNSMIRRORURL)
|
||||
# soup = BeautifulSoup(payload.content, "html.parser")
|
||||
|
||||
|
||||
# for data in soup.find_all('div', class_='download_section'):
|
||||
# for a in data.find_all('a', href=True):
|
||||
|
||||
gigalist = []
|
||||
# url = a['href']
|
||||
|
||||
payload = requests.get(OPNSNSMIRRORURL)
|
||||
soup = BeautifulSoup(payload.content, "html.parser")
|
||||
# saniturl = sanitizeURL(url)
|
||||
|
||||
# print(soup)
|
||||
# # print(saniturl)
|
||||
# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||
# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
# divs = soup.find_all("div", {"class": "download_section"})
|
||||
# # print(IPv4Dict)
|
||||
# try:
|
||||
# for key, ip in IPv4Dict.items():
|
||||
# print(f"Found the ipv4: {ip}")
|
||||
|
||||
for data in soup.find_all('div', class_='download_section'):
|
||||
for a in data.find_all('a', href=True):
|
||||
# for key, ip in IPv6Dict.items():
|
||||
# print(f"Found the ipv6: {ip}")
|
||||
# # If int returned from WhatDomain then it is error.
|
||||
# # Error type is described via prints from whatdomain functions
|
||||
# except AttributeError:
|
||||
# continue
|
||||
|
||||
url = a['href']
|
||||
|
||||
saniturl = sanitizeURL(url)
|
||||
|
||||
print(saniturl)
|
||||
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
|
||||
if type(IPv4Dict) == int:
|
||||
print(f"{DNSerrHandling(IPv4Dict)}")
|
||||
continue
|
||||
elif type(IPv6Dict) == int:
|
||||
print(f"ERROR RESOLVING {saniturl} IPv6 address with error code {IPv6Dict}")
|
||||
continue
|
||||
|
||||
for key, ip in IPv4Dict.items():
|
||||
print(f"Found the ipv4: {ip}")
|
||||
|
||||
for key, ip in IPv6Dict.items():
|
||||
print(f"Found the ipv6: {ip}")
|
||||
|
||||
# print(a.get('href')) #for getting link
|
||||
# print(a.text) #for getting text between the link
|
||||
|
||||
|
||||
|
||||
# for a in soup.find_all("div", {"class": "download_section"}, "a", href=True):
|
||||
|
||||
|
||||
|
||||
# h2zz = divs = soup.find_all("h3")
|
||||
|
||||
# print(f" one link is: {h2zz}")
|
||||
Reference in New Issue
Block a user