136 lines
3.8 KiB
Python
136 lines
3.8 KiB
Python
import requests, schedule, time
|
|
from bs4 import BeautifulSoup
|
|
from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
|
|
|
|
|
|
|
|
OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing"
|
|
IPv4FILE = "/etc/debmirror/OPNS_MirrorListV4"
|
|
IPv6FILE = "/etc/debmirror/OPNS_MirrorListV6"
|
|
|
|
|
|
def sanitizeURL(inpurl: str):
|
|
if not "/" in inpurl[:-1]:
|
|
inpurl += "/"
|
|
if "https://" in inpurl:
|
|
outurl = inpurl[8:]
|
|
elif "http://" in inpurl:
|
|
outurl = inpurl[7:]
|
|
elif "http://" or "https://" not in url:
|
|
outurl = inpurl
|
|
else:
|
|
return -1
|
|
|
|
# how the fuck does it work?
|
|
# I mean I wrote this but I don't know why does it work.
|
|
i = 0
|
|
for char in outurl:
|
|
i += 1
|
|
if char == "/":
|
|
outurl = outurl[:i]
|
|
|
|
if char == "/":
|
|
outurl = outurl[:-1]
|
|
return outurl
|
|
|
|
|
|
def getFreshData():
|
|
payload = requests.get(OPNSNSMIRRORURL)
|
|
soup = BeautifulSoup(payload.content, "html.parser")
|
|
return soup
|
|
|
|
def LeJob():
|
|
|
|
print("Starting lookup")
|
|
|
|
LeSoup = getFreshData()
|
|
|
|
# print(LeMirrorDict)
|
|
|
|
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
|
|
|
|
for data in LeSoup.find_all('div', class_='download_section'):
|
|
for a in data.find_all('a', href=True):
|
|
|
|
url = a['href']
|
|
|
|
saniturl = sanitizeURL(url)
|
|
|
|
# print(saniturl)
|
|
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
|
|
|
# print(IPv4Dict)
|
|
try:
|
|
for key, ip in IPv4Dict.items():
|
|
print(f"Found the ipv4: {ip}")
|
|
fW.write(ip + "/32" + "\n")
|
|
|
|
# If int returned from WhatDomain then it is error.
|
|
# Error type is described via prints from whatdomain functions
|
|
except AttributeError:
|
|
continue
|
|
|
|
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
|
|
|
|
for data in LeSoup.find_all('div', class_='download_section'):
|
|
for a in data.find_all('a', href=True):
|
|
|
|
url = a['href']
|
|
|
|
saniturl = sanitizeURL(url)
|
|
|
|
# print(saniturl)
|
|
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
|
|
|
# print(IPv6Dict)
|
|
try:
|
|
for key, ip in IPv6Dict.items():
|
|
print(f"Found the ipv6: {ip}")
|
|
fW.write(ip + "/128" + "\n")
|
|
|
|
# If int returned from WhatDomain then it is error.
|
|
# Error type is described via prints from whatdomain functions
|
|
except AttributeError:
|
|
continue
|
|
|
|
# schedule.every().day.at("12:45").do(LeJob)
|
|
# schedule.every().day.at("17:44").do(LeJob)
|
|
|
|
# while True:
|
|
# schedule.run_pending()
|
|
# print("Waiting...")
|
|
# time.sleep(30) #Wait one minute
|
|
# # LeJob()
|
|
|
|
LeJob()
|
|
|
|
# gigalist = []
|
|
|
|
# payload = requests.get(OPNSNSMIRRORURL)
|
|
# soup = BeautifulSoup(payload.content, "html.parser")
|
|
|
|
|
|
# for data in soup.find_all('div', class_='download_section'):
|
|
# for a in data.find_all('a', href=True):
|
|
|
|
# url = a['href']
|
|
|
|
# saniturl = sanitizeURL(url)
|
|
|
|
# # print(saniturl)
|
|
# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
|
# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
|
|
|
# # print(IPv4Dict)
|
|
# try:
|
|
# for key, ip in IPv4Dict.items():
|
|
# print(f"Found the ipv4: {ip}")
|
|
|
|
# for key, ip in IPv6Dict.items():
|
|
# print(f"Found the ipv6: {ip}")
|
|
# # If int returned from WhatDomain then it is error.
|
|
# # Error type is described via prints from whatdomain functions
|
|
# except AttributeError:
|
|
# continue
|
|
|