diff --git a/docker/Dockerfile b/docker/Dockerfile index 1b1f48e..5db9211 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,11 +14,15 @@ WORKDIR /etc/debmirror/ RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \ ln -s /etc/debmirror/Repo-IP-lists/MirrorListV4 /etc/debmirror/MirrorListV4 && \ - ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6 + ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6 && \ + ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV4 /etc/debmirror/OPNS_MirrorListV4 && \ + ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV6 /etc/debmirror/OPNS_MirrorListV6 + # RUN touch /etc/debmirror/MirrorListV6 && \ # touch /etc/debmirror/MirrorListV4 COPY mainDocker.py /etc/debmirror/ +COPY mainOPNsense.py /etc/debmirror/ COPY whatDomain.py /etc/debmirror/ COPY requirements.txt /etc/debmirror/ diff --git a/docker/cron-jobs b/docker/cron-jobs index 08ce8dc..ec48208 100644 --- a/docker/cron-jobs +++ b/docker/cron-jobs @@ -1,7 +1,8 @@ # python # 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py -*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py # git push # 30 19 * * * /bin/sh /etc/debmirror/gitPush.sh -*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh \ No newline at end of file +#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh \ No newline at end of file diff --git a/docker/gitPush.sh b/docker/gitPush.sh index 585041c..f79280c 100644 --- a/docker/gitPush.sh +++ b/docker/gitPush.sh @@ -20,8 +20,11 @@ git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}" git config user.name "UpdateBot" git config user.email "UpdateBot@localhost.local" +# stage files +git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6 + # If anything to commit locally, commit it now -if ! git diff --quiet; then +if ! git diff --quiet --cached; then echo "[$(date)] Committing local changes before pulling" git commit -m "Auto-commit before pull on $(date -Iseconds)" fi @@ -29,9 +32,6 @@ fi # Now pull the latest git pull --rebase --autostash -# stage files -git add MirrorListV4 MirrorListV6 - # Commit and push only if there's anything new staged if git diff --quiet; then echo "[$(date)] No changes to commit." diff --git a/docker/mainOPNsense.py b/docker/mainOPNsense.py new file mode 100644 index 0000000..f0ab872 --- /dev/null +++ b/docker/mainOPNsense.py @@ -0,0 +1,135 @@ +import requests, schedule, time +from bs4 import BeautifulSoup +from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy + + + +OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing" +IPv4FILE = "/etc/debmirror/OPNS_MirrorListV4" +IPv6FILE = "/etc/debmirror/OPNS_MirrorListV6" + + +def sanitizeURL(inpurl: str): + if not "/" in inpurl[:-1]: + inpurl += "/" + if "https://" in inpurl: + outurl = inpurl[8:] + elif "http://" in inpurl: + outurl = inpurl[7:] + elif "http://" or "https://" not in url: + outurl = inpurl + else: + return -1 + + # how the fuck does it work? + # I mean I wrote this but I don't know why does it work. + i = 0 + for char in outurl: + i += 1 + if char == "/": + outurl = outurl[:i] + + if char == "/": + outurl = outurl[:-1] + return outurl + + +def getFreshData(): + payload = requests.get(OPNSNSMIRRORURL) + soup = BeautifulSoup(payload.content, "html.parser") + return soup + +def LeJob(): + + print("Starting lookup") + + LeSoup = getFreshData() + + # print(LeMirrorDict) + + with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW: + + for data in LeSoup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): + + url = a['href'] + + saniturl = sanitizeURL(url) + + # print(saniturl) + IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) + + # print(IPv4Dict) + try: + for key, ip in IPv4Dict.items(): + print(f"Found the ipv4: {ip}") + fW.write(ip + "/32" + "\n") + + # If int returned from WhatDomain then it is error. + # Error type is described via prints from whatdomain functions + except AttributeError: + continue + + with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: + + for data in LeSoup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): + + url = a['href'] + + saniturl = sanitizeURL(url) + + # print(saniturl) + IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) + + # print(IPv6Dict) + try: + for key, ip in IPv6Dict.items(): + print(f"Found the ipv6: {ip}") + fW.write(ip + "/128" + "\n") + + # If int returned from WhatDomain then it is error. + # Error type is described via prints from whatdomain functions + except AttributeError: + continue + +# schedule.every().day.at("12:45").do(LeJob) +# schedule.every().day.at("17:44").do(LeJob) + +# while True: +# schedule.run_pending() +# print("Waiting...") +# time.sleep(30) #Wait one minute +# # LeJob() + +LeJob() + +# gigalist = [] + +# payload = requests.get(OPNSNSMIRRORURL) +# soup = BeautifulSoup(payload.content, "html.parser") + + +# for data in soup.find_all('div', class_='download_section'): +# for a in data.find_all('a', href=True): + +# url = a['href'] + +# saniturl = sanitizeURL(url) + +# # print(saniturl) +# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) +# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) + +# # print(IPv4Dict) +# try: +# for key, ip in IPv4Dict.items(): +# print(f"Found the ipv4: {ip}") + +# for key, ip in IPv6Dict.items(): +# print(f"Found the ipv6: {ip}") +# # If int returned from WhatDomain then it is error. +# # Error type is described via prints from whatdomain functions +# except AttributeError: +# continue +