From d13a36ee234b12423939377ba30aed1eea7791ed Mon Sep 17 00:00:00 2001 From: YuruC3 Date: Tue, 13 Jan 2026 19:13:22 +0100 Subject: [PATCH] Trying to integrate all functions into one container --- AiO_Container/Dockerfile | 51 +++++++++ AiO_Container/code/mainCustom.py | 96 +++++++++++++++++ AiO_Container/code/mainDocker.py | 165 +++++++++++++++++++++++++++++ AiO_Container/code/mainOPNsense.py | 135 +++++++++++++++++++++++ AiO_Container/code/opnsense.py | 135 +++++++++++++++++++++++ AiO_Container/code/whatDomain.py | 129 ++++++++++++++++++++++ AiO_Container/cron-jobs | 11 ++ AiO_Container/entrypoint.sh | 79 ++++++++++++++ AiO_Container/gitPush.sh | 100 +++++++++++++++++ AiO_Container/init.sh | 23 ++++ AiO_Container/requirements.txt | 4 + 11 files changed, 928 insertions(+) create mode 100644 AiO_Container/Dockerfile create mode 100644 AiO_Container/code/mainCustom.py create mode 100644 AiO_Container/code/mainDocker.py create mode 100644 AiO_Container/code/mainOPNsense.py create mode 100644 AiO_Container/code/opnsense.py create mode 100644 AiO_Container/code/whatDomain.py create mode 100644 AiO_Container/cron-jobs create mode 100644 AiO_Container/entrypoint.sh create mode 100644 AiO_Container/gitPush.sh create mode 100644 AiO_Container/init.sh create mode 100644 AiO_Container/requirements.txt diff --git a/AiO_Container/Dockerfile b/AiO_Container/Dockerfile new file mode 100644 index 0000000..3d4a204 --- /dev/null +++ b/AiO_Container/Dockerfile @@ -0,0 +1,51 @@ +FROM alpine:latest + +# https://docs.docker.com/reference/dockerfile/#environment-replacement +ENV DEBMIRRORURL="https://www.debian.org/mirror/list" + +# By default these are scraped +ENV EXTRA_REPOS=False +ENV SECURITY_REPOS=True +ENV DEBIAN_REPOS=True +ENV OPNSENSE_REPOS=True + +RUN apk update && \ + apk add python3 py3-pip git su-exec curl wget unzip && \ + mkdir -p /etc/debmirror + +WORKDIR /etc/debmirror/ + +# This will be done in Init script based on what ENV's are configured +# RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \ +# ln -s /etc/debmirror/Repo-IP-lists/MirrorListV4 /etc/debmirror/MirrorListV4 && \ +# ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6 && \ +# ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV4 /etc/debmirror/OPNS_MirrorListV4 && \ +# ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV6 /etc/debmirror/OPNS_MirrorListV6 +# RUN touch /etc/debmirror/MirrorListV6 && \ +# touch /etc/debmirror/MirrorListV4 + +# Copy code +COPY ./code/* /etc/debmirror/ + +# Copy important files +# COPY ./init.sh /etc/debmirror/ + +# Install python modules +COPY ./requirements.txt /etc/debmirror/ +RUN python3 -m venv venv && \ + venv/bin/python3 -m pip install --upgrade pip && \ + venv/bin/pip3 install -r requirements.txt + +# Setup cronjobs +# COPY cron-jobs /etc/crontabs/ +# RUN chmod 0644 /etc/crontabs/cron-jobs && \ +# crontab /etc/crontabs/cron-jobs + +# Copy and configure git copying script +COPY ./gitPush.sh /etc/debmirror/ +RUN chmod +x /etc/debmirror/gitPush.sh + +ENTRYPOINT [ "/entrypoint.sh" ] + +# Start cron +CMD ["/usr/sbin/crond", "-f"] diff --git a/AiO_Container/code/mainCustom.py b/AiO_Container/code/mainCustom.py new file mode 100644 index 0000000..20576ba --- /dev/null +++ b/AiO_Container/code/mainCustom.py @@ -0,0 +1,96 @@ +import requests, schedule, time, os +# from bs4 import BeautifulSoup +from whatDomain import * + +EXTRAURL = [] + +repoListFile = open(os.getenv("REPOFILE", "customRepoList.list"), 'r') +i = 0 +for repoUrl in repoListFile: + i += 1 + print(repoUrl.strip()) + EXTRAURL.append(repoUrl.strip()) +repoListFile.close() + +CUSTOMIPV4FILENAME = str(os.getenv("CUSTOMIPV4FILENAME", "CustomMirrorListV4")) +CUSTOMIPV6FILENAME = str(os.getenv("CUSTOMIPV6FILENAME", "CustomMirrorListV6")) + +# EXTRAURL = list(os.getenv("EXTRAURL", "https://mdu.se/")) +IPv4FILE = f"/etc/debmirror/{CUSTOMIPV4FILENAME}" +IPv6FILE = f"/etc/debmirror/{CUSTOMIPV6FILENAME}" + +def sanitizeURL(inpurl: str): + if "https://" in inpurl: + outurl = inpurl[8:] + elif "http://" in inpurl: + outurl = inpurl[7:] + elif "" in inpurl: + return 7 + elif "http://" or "https://" not in url: + outurl = inpurl + else: + return inpurl + + i = 0 + for char in outurl: + i += 1 + if char == "/": + outurl = outurl[:i] + + if char == "/": + outurl = outurl[:-1] + return outurl + +def LeJob(): + + print("Starting lookup") + + # print(LeMirrorDict) + + with open(IPv4FILE, "w",) as fW: + + for url in EXTRAURL: + goodurl = sanitizeURL(url) + # print(goodurl) + + if goodurl == 7: + continue + + # ip4Dict = ermWhatATheIpFromDomainYaCrazy(url) + ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl) + + try: + for key, ip in ip4Dict.items(): + print(ip + "/32") + + fW.write(ip + "/32" + "\n") + except AttributeError: + continue + + + with open(IPv6FILE, "w",) as fW: + + for url in EXTRAURL: + goodurl = sanitizeURL(url) + + if goodurl == 7: + continue + # print(goodurl) + + # ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url) + ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl) + + try: + for key, ip in ip6Dict.items(): + print(ip + "/128") + + fW.write(ip + "/128" + "\n") + except AttributeError: + continue + + + + +LeJob() + +print("Done") \ No newline at end of file diff --git a/AiO_Container/code/mainDocker.py b/AiO_Container/code/mainDocker.py new file mode 100644 index 0000000..4277530 --- /dev/null +++ b/AiO_Container/code/mainDocker.py @@ -0,0 +1,165 @@ +import requests, schedule, time, os +from bs4 import BeautifulSoup +from whatDomain import * + +DEBMIRRORURL = str(os.getenv("DEBMIRRORURL", "https://www.debian.org/mirror/list")) +DEBSECURITYURL = str(os.getenv("DEBSECURITYURL", "https://security.debian.org/debian-security/")) + +EXTRASURL = ["https://download.docker.com/linux/debian/", + # Double just to be sure. Even though they point to the same IP + "http://download.proxmox.com/debian/", + "https://enterprise.proxmox.com/debian/pve/", + # That's for nvidia docker toolkit something something + # stuff that makes containers use nvenc and shit + "https://nvidia.github.io/libnvidia-container/stable/deb/", + "https://nvidia.github.io/libnvidia-container/experimental/deb/"] + +IPv4FILE = "/etc/debmirror/MirrorListV4" +IPv6FILE = "/etc/debmirror/MirrorListV6" + + +# Define EU and American countries as well as Security for security updates +target_countries = set([ + # Europe + "Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark", + "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland", + "Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland", + "Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland", + "United Kingdom", "Moldova", + # America + "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador", + "Mexico", "Peru", "United States", "Uruguay", "Venezuela", + # Others + "Security", "Extras" +]) + + +def sanitizeURL(inpurl: str): + if "https://" in inpurl: + outurl = inpurl[8:] + elif "http://" in inpurl: + outurl = inpurl[7:] + elif "" in inpurl: + return 7 + elif "http://" or "https://" not in url: + outurl = inpurl + else: + return inpurl + + i = 0 + for char in outurl: + i += 1 + if char == "/": + outurl = outurl[:i] + + if char == "/": + outurl = outurl[:-1] + return outurl + + +def getFreshData(): + payload = requests.get(DEBMIRRORURL) + soup = BeautifulSoup(payload.content, "html.parser") + return soup + + +def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup): + + outMirrorDict = {} + current_country = None + + # Iterate through all table rows + for table in SoupInput.find_all("table"): + for row in table.find_all("tr"): + # Check for country name in a full-row header () + strong = row.find("strong") + if strong: + country_name = strong.get_text(strip=True) + if country_name in target_countries: + current_country = country_name + else: + current_country = None + continue # move to next row + + # Check for inline country name in first column + cols = row.find_all("td") + if len(cols) >= 2: + possible_country = cols[0].get_text(strip=True) + link_tag = cols[1].find("a", href=True) + if possible_country in target_countries: + current_country = possible_country + if current_country and link_tag: + url = link_tag['href'] + if current_country not in outMirrorDict: + outMirrorDict[current_country] = [] + outMirrorDict[current_country].append(url) + + if os.environ["SECURITYREPOS"]: + outMirrorDict.update({"Security": DEBSECURITYURL}) + if os.environ["EXTRAREPOS"]: + outMirrorDict.update({"Extras": EXTRASURL}) + + return outMirrorDict + +def LeJob(): + + print("Starting lookup") + + LeSoup = getFreshData() + + LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup) + + # print(LeMirrorDict) + + with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW: + + for key, urls in LeMirrorDict.items(): + # print(urls) + if key in target_countries: + + for url in urls: + # print(url) + + if url not in fR: + + goodurl = sanitizeURL(url) + # print(goodurl) + + ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl) + + try: + for key, ip in ip4Dict.items(): + print(ip) + + fW.write(ip + "/32" + "\n") + except AttributeError: + continue + + + with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: + + for key, urls in LeMirrorDict.items(): + if key in target_countries: + + for url in urls: + + if url not in fR: + + goodurl = sanitizeURL(url) + # print(goodurl) + + ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl) + + try: + for key, ip in ip6Dict.items(): + # print(ip) + + fW.write(ip + "/128" + "\n") + except AttributeError: + continue + + + +LeJob() + +print("Done") \ No newline at end of file diff --git a/AiO_Container/code/mainOPNsense.py b/AiO_Container/code/mainOPNsense.py new file mode 100644 index 0000000..ca75831 --- /dev/null +++ b/AiO_Container/code/mainOPNsense.py @@ -0,0 +1,135 @@ +import requests, schedule, time +from bs4 import BeautifulSoup +from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy + + +OPNSNSMIRRORURL = str(os.getenv("OPNSNSMIRRORURL", "https://opnsense.org/download/#full-mirror-listing")) + +IPv4FILE = str(os.getenv("IPv4FILE", "/etc/debmirror/OPNS_MirrorListV4")) +IPv6FILE = str(os.getenv("IPv6FILE", "/etc/debmirror/OPNS_MirrorListV6")) + + +def sanitizeURL(inpurl: str): + if not "/" in inpurl[:-1]: + inpurl += "/" + if "https://" in inpurl: + outurl = inpurl[8:] + elif "http://" in inpurl: + outurl = inpurl[7:] + elif "http://" or "https://" not in url: + outurl = inpurl + else: + return -1 + + # how the fuck does it work? + # I mean I wrote this but I don't know why does it work. + i = 0 + for char in outurl: + i += 1 + if char == "/": + outurl = outurl[:i] + + if char == "/": + outurl = outurl[:-1] + return outurl + + +def getFreshData(): + payload = requests.get(OPNSNSMIRRORURL) + soup = BeautifulSoup(payload.content, "html.parser") + return soup + +def LeJob(): + + print("Starting lookup") + + LeSoup = getFreshData() + + # print(LeMirrorDict) + + with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW: + + for data in LeSoup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): + + url = a['href'] + + saniturl = sanitizeURL(url) + + # print(saniturl) + IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) + + # print(IPv4Dict) + try: + for key, ip in IPv4Dict.items(): + print(f"Found the ipv4: {ip}") + fW.write(ip + "/32" + "\n") + + # If int returned from WhatDomain then it is error. + # Error type is described via prints from whatdomain functions + except AttributeError: + continue + + with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: + + for data in LeSoup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): + + url = a['href'] + + saniturl = sanitizeURL(url) + + # print(saniturl) + IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) + + # print(IPv6Dict) + try: + for key, ip in IPv6Dict.items(): + print(f"Found the ipv6: {ip}") + fW.write(ip + "/128" + "\n") + + # If int returned from WhatDomain then it is error. + # Error type is described via prints from whatdomain functions + except AttributeError: + continue + +# schedule.every().day.at("12:45").do(LeJob) +# schedule.every().day.at("17:44").do(LeJob) + +# while True: +# schedule.run_pending() +# print("Waiting...") +# time.sleep(30) #Wait one minute +# # LeJob() + +LeJob() + +# gigalist = [] + +# payload = requests.get(OPNSNSMIRRORURL) +# soup = BeautifulSoup(payload.content, "html.parser") + + +# for data in soup.find_all('div', class_='download_section'): +# for a in data.find_all('a', href=True): + +# url = a['href'] + +# saniturl = sanitizeURL(url) + +# # print(saniturl) +# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) +# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) + +# # print(IPv4Dict) +# try: +# for key, ip in IPv4Dict.items(): +# print(f"Found the ipv4: {ip}") + +# for key, ip in IPv6Dict.items(): +# print(f"Found the ipv6: {ip}") +# # If int returned from WhatDomain then it is error. +# # Error type is described via prints from whatdomain functions +# except AttributeError: +# continue + diff --git a/AiO_Container/code/opnsense.py b/AiO_Container/code/opnsense.py new file mode 100644 index 0000000..0c7dc76 --- /dev/null +++ b/AiO_Container/code/opnsense.py @@ -0,0 +1,135 @@ +import requests, schedule, time +from bs4 import BeautifulSoup +from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy + + + +OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing" +IPv4FILE = "./OPNS_MirrorListV4" +IPv6FILE = "./OPNS_MirrorListV6" + + +def sanitizeURL(inpurl: str): + if not "/" in inpurl[:-1]: + inpurl += "/" + if "https://" in inpurl: + outurl = inpurl[8:] + elif "http://" in inpurl: + outurl = inpurl[7:] + elif "http://" or "https://" not in url: + outurl = inpurl + else: + return -1 + + # how the fuck does it work? + # I mean I wrote this but I don't know why does it work. + i = 0 + for char in outurl: + i += 1 + if char == "/": + outurl = outurl[:i] + + if char == "/": + outurl = outurl[:-1] + return outurl + + +def getFreshData(): + payload = requests.get(OPNSNSMIRRORURL) + soup = BeautifulSoup(payload.content, "html.parser") + return soup + +def LeJob(): + + print("Starting lookup") + + LeSoup = getFreshData() + + # print(LeMirrorDict) + + with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW: + + for data in LeSoup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): + + url = a['href'] + + saniturl = sanitizeURL(url) + + # print(saniturl) + IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) + + # print(IPv4Dict) + try: + for key, ip in IPv4Dict.items(): + print(f"Found the ipv4: {ip}") + fW.write(ip + "/32" + "\n") + + # If int returned from WhatDomain then it is error. + # Error type is described via prints from whatdomain functions + except AttributeError: + continue + + with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: + + for data in LeSoup.find_all('div', class_='download_section'): + for a in data.find_all('a', href=True): + + url = a['href'] + + saniturl = sanitizeURL(url) + + # print(saniturl) + IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) + + # print(IPv6Dict) + try: + for key, ip in IPv6Dict.items(): + print(f"Found the ipv6: {ip}") + fW.write(ip + "/128" + "\n") + + # If int returned from WhatDomain then it is error. + # Error type is described via prints from whatdomain functions + except AttributeError: + continue + +# schedule.every().day.at("12:45").do(LeJob) +# schedule.every().day.at("17:44").do(LeJob) + +# while True: +# schedule.run_pending() +# print("Waiting...") +# time.sleep(30) #Wait one minute +# # LeJob() + +LeJob() + +# gigalist = [] + +# payload = requests.get(OPNSNSMIRRORURL) +# soup = BeautifulSoup(payload.content, "html.parser") + + +# for data in soup.find_all('div', class_='download_section'): +# for a in data.find_all('a', href=True): + +# url = a['href'] + +# saniturl = sanitizeURL(url) + +# # print(saniturl) +# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl) +# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl) + +# # print(IPv4Dict) +# try: +# for key, ip in IPv4Dict.items(): +# print(f"Found the ipv4: {ip}") + +# for key, ip in IPv6Dict.items(): +# print(f"Found the ipv6: {ip}") +# # If int returned from WhatDomain then it is error. +# # Error type is described via prints from whatdomain functions +# except AttributeError: +# continue + diff --git a/AiO_Container/code/whatDomain.py b/AiO_Container/code/whatDomain.py new file mode 100644 index 0000000..70cb81e --- /dev/null +++ b/AiO_Container/code/whatDomain.py @@ -0,0 +1,129 @@ +#from nslookup import Nslookup +from typing import Optional, Annotated +import dns, dns.resolver + +# https://www.codeunderscored.com/nslookup-python/ + +def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]): + #dns_query = Nslookup() + """ + Tells you what IPv4 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(inpDomainNameOrSomething, 'A') + except dns.resolver.NoAnswer: + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + +def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]): + #dns_query = Nslookup() + """ + Tells you what IPv6 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA') + except dns.resolver.NoAnswer: + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + + +def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]): + #dns_query = Nslookup() + """ + Tells you what IPv6 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa" + + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(whatToCheck, 'PTR') + except dns.resolver.NoAnswer: + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + + +#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net")) +#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net")) +#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226")) diff --git a/AiO_Container/cron-jobs b/AiO_Container/cron-jobs new file mode 100644 index 0000000..cc20a27 --- /dev/null +++ b/AiO_Container/cron-jobs @@ -0,0 +1,11 @@ +# python +20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py +25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py +# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py + +# git push +30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh +#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh \ No newline at end of file diff --git a/AiO_Container/entrypoint.sh b/AiO_Container/entrypoint.sh new file mode 100644 index 0000000..3bba1e6 --- /dev/null +++ b/AiO_Container/entrypoint.sh @@ -0,0 +1,79 @@ +#!/bin/sh +set -e + +# Default to 1600 if not provided +PUID="${PUID:-1600}" +PGID="${PGID:-1600}" +USER="${USER:-muyu}" +GROUP="${USER:-muyu}" +CHOWNPATH="/etc/debmirror" +EXTRA_REPOS="${EXTRA_REPOS:-False}" +SECURITY_REPOS="${SECURITY_REPOS:-True}" +DEBIAN_REPOS="${DEBIAN_REPOS:-True}" +OPNSENSE_REPOS="${OPNSENSE_REPOS:-True}" + +# Create group if missing +if ! getent group "$GROUP" >/dev/null 2>&1; then + addgroup -g "$PGID" "$GROUP" +fi + +# Create user if missing +if ! id -u "$USER" >/dev/null 2>&1; then + adduser -D -u "$PUID" -G "$GROUP" "$USER" +fi + +# Prepare cron file +touch /etc/crontabs/cron-jobs + +# Create symlinks based on what is passed +# Also add cronjobs when applicable +git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists +if [[ "$EXTRA_REPOS" ]]; then + ln -s /etc/debmirror/Repo-IP-lists/CustomMirrorListV4 /etc/debmirror/CustomMirrorListV4 + ln -s /etc/debmirror/Repo-IP-lists/CustomMirrorListV6 /etc/debmirror/CustomMirrorListV6 + + echo "00 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainCustom.py" >> /etc/crontabs/cron-jobs +fi +if [[ "$SECURITY_REPOS" ]]; then + ln -s /etc/debmirror/Repo-IP-lists/SecurityMirrorListV4 /etc/debmirror/SecurityMirrorListV4 + ln -s /etc/debmirror/Repo-IP-lists/SecurityMirrorListV6 /etc/debmirror/SecurityMirrorListV6 + + echo "05 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainSec.py" >> /etc/crontabs/cron-jobs +fi +if [[ "$DEBIAN_REPOS" ]]; then + ln -s /etc/debmirror/Repo-IP-lists/DebianMirrorListV4 /etc/debmirror/DebianMirrorListV4 + ln -s /etc/debmirror/Repo-IP-lists/DebianMirrorListV6 /etc/debmirror/DebianMirrorListV6 + + echo "10 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py" >> /etc/crontabs/cron-jobs +fi +if [[ "$OPNSENSE_REPOS" ]]; then + ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMIrrorListV4 /etc/debmirror/OPNsenseMIrrorListV4 + ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMIrrorListV6 /etc/debmirror/OPNsenseMIrrorListV6 + + echo "15 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py" >> /etc/crontabs/cron-jobs +fi + +echo "30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh" >> /etc/crontabs/cron-jobs + +# Configure the crontab to work +chmod 0644 /etc/crontabs/cron-jobs +crontab /etc/crontabs/cron-jobs + +# Fix permissions +chown -R "$PUID:$PGID" "$CHOWNPATH" + +# Drop privileges & run command +exec su-exec "$PUID:$PGID" "$@" + + +# # python +# 20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py +# 25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +# # 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py +# # 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +# #*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +# #*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py + +# # git push +# 30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh +# #*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh \ No newline at end of file diff --git a/AiO_Container/gitPush.sh b/AiO_Container/gitPush.sh new file mode 100644 index 0000000..fa7e983 --- /dev/null +++ b/AiO_Container/gitPush.sh @@ -0,0 +1,100 @@ +#!/bin/sh +set -e + +# Check if these are passed and terminate if they are not +GITEA_TOKEN="${GITEA_TOKEN:-6767}" +GITURL="${GITURL:-6767}" +GITREPOPATH="${GITREPOPATH:-6767}" +GITURLPROTO="${GITURLPROTO:-http}" + +# if [[ "$GITEA_TOKEN" == "6767" || "$GITREPOPATH" == "6767" || "$GITURL" == "6767" ]]; then +if [[ "$GITEA_TOKEN" == "6767" ]]; then + exit 1 +fi + +WORKPTH="${WORKPTH:-/etc/debmirror/}" +REPO_DIR="${REPO_DIR:-$WORKPTH/Repo-IP-lists}" +REPO_URL="${REPO_URL:-GITURLPROTO://GITURL/GITREPOPATH}" + + +# Clone repo if not exists +if [[ ! -d "$REPO_DIR/.git" ]]; then + echo "[$(date)] Cloning repository..." + git clone "$REPO_URL" "$REPO_DIR" +fi + +cd "$REPO_DIR" + +# Abort previous rebase/cherry-pick if stuck +git rebase --abort 2>/dev/null || true +git cherry-pick --abort 2>/dev/null || true + +# Make sure we're on a clean 'main' +git checkout main || git checkout -b main origin/main +git pull --rebase --autostash + +# git remote set-url origin "${GITURLPROTO}://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}" +git remote set-url origin "https://${GITEA_TOKEN}@YuruC3/Repo-IP-lists" + + +git config user.name "UpdateBot" +git config user.email "UpdateBot@localhost.local" + +# Stage the files +git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6 + +# Only proceed if there are staged changes +if [[ ! git diff --quiet --cached ]]; then + echo "[$(date)] Committing and pushing changes..." + git commit -m "Auto-update mirror list on $(date -Iseconds)" + # git pull --rebase --autostash + git push --quiet + echo "[$(date)] Changes pushed." +else + echo "[$(date)] No changes to commit or push." +fi + + +# #!/bin/sh +# set -e + +# WORKPTH="/etc/debmirror/" +# REPO_DIRd="$WORKPTH/Repo-IP-lists" +# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}" + + +# # Clone repo only if it doesn't already exist +# if [ ! -d "$REPO_DIR/.git" ]; then +# echo "[$(date)] Cloning repository..." +# git clone "$REPO_URL" "$REPO_DIR" +# fi + + +# cd "$REPO_DIR" + +# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}" + +# git config user.name "UpdateBot" +# git config user.email "UpdateBot@localhost.local" + +# # stage files +# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6 + +# # If anything to commit locally, commit it now +# if ! git diff --quiet --cached; then +# echo "[$(date)] Committing local changes before pulling" +# git commit -m "Auto-commit before pull on $(date -Iseconds)" +# fi + +# # Now pull the latest +# git pull --rebase --autostash + +# # Commit and push only if there's anything new staged +# if git diff --quiet; then +# echo "[$(date)] No changes to commit." +# else +# git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet +# git push --quiet +# echo "[$(date)] Changes pushed successfully." +# fi + diff --git a/AiO_Container/init.sh b/AiO_Container/init.sh new file mode 100644 index 0000000..d049807 --- /dev/null +++ b/AiO_Container/init.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -e + +echo "EXTRAURL: $EXTRAURL" + +if [[ -f "$REPOFILE" ]]; then + echo "URL file exists" +else + touch $REPOFILE + echo $EXTRAURL | tee $REPOFILE +fi + +echo "nameserver $DNSSRV" > /etc/resolv.conf +echo "search local" >> /etc/resolv.conf + +git clone https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH} +ln -s /etc/customMirrors/Repo-IP-lists/$IPV4FILENAME /etc/customMirrors/$IPV4FILENAME +ln -s /etc/customMirrors/Repo-IP-lists/$IPV6FILENAME /etc/customMirrors/$IPV6FILENAME + +echo -n "$CRONTABSET /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py" > /etc/crontabs/customListCron + +exec /usr/sbin/crond -f \ No newline at end of file diff --git a/AiO_Container/requirements.txt b/AiO_Container/requirements.txt new file mode 100644 index 0000000..a2aa631 --- /dev/null +++ b/AiO_Container/requirements.txt @@ -0,0 +1,4 @@ +beautifulsoup4==4.13.4 +requests==2.32.3 +schedule==1.2.2 +nslookup==1.8.1 \ No newline at end of file