Trying to integrate all functions into one container

AiO_Container/Dockerfile

@@ -0,0 +1,51 @@
+FROM alpine:latest
+
+# https://docs.docker.com/reference/dockerfile/#environment-replacement
+ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
+
+# By default these are scraped
+ENV EXTRA_REPOS=False
+ENV SECURITY_REPOS=True
+ENV DEBIAN_REPOS=True
+ENV OPNSENSE_REPOS=True
+
+RUN apk update && \
+    apk add python3 py3-pip git su-exec curl wget unzip && \
+    mkdir -p /etc/debmirror   
+
+WORKDIR /etc/debmirror/
+
+# This will be done in Init script based on what ENV's are configured
+# RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \
+#     ln -s /etc/debmirror/Repo-IP-lists/MirrorListV4 /etc/debmirror/MirrorListV4 && \
+#     ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6 && \
+#     ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV4 /etc/debmirror/OPNS_MirrorListV4 && \
+#     ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV6 /etc/debmirror/OPNS_MirrorListV6
+# RUN touch /etc/debmirror/MirrorListV6 && \
+#     touch /etc/debmirror/MirrorListV4
+
+# Copy code
+COPY ./code/* /etc/debmirror/
+
+# Copy important files
+# COPY ./init.sh /etc/debmirror/
+
+# Install python modules
+COPY ./requirements.txt /etc/debmirror/
+RUN python3 -m venv venv && \
+    venv/bin/python3 -m pip install --upgrade pip && \
+    venv/bin/pip3 install -r requirements.txt
+
+# Setup cronjobs
+# COPY cron-jobs /etc/crontabs/
+# RUN chmod 0644 /etc/crontabs/cron-jobs && \
+#     crontab /etc/crontabs/cron-jobs
+
+# Copy and configure git copying script
+COPY ./gitPush.sh /etc/debmirror/
+RUN chmod +x /etc/debmirror/gitPush.sh
+
+ENTRYPOINT [ "/entrypoint.sh" ]
+
+# Start cron
+CMD ["/usr/sbin/crond", "-f"]

AiO_Container/code/mainCustom.py

@@ -0,0 +1,96 @@
+import requests, schedule, time, os
+# from bs4 import BeautifulSoup
+from whatDomain import *
+
+EXTRAURL = []
+
+repoListFile = open(os.getenv("REPOFILE", "customRepoList.list"), 'r')
+i = 0
+for repoUrl in repoListFile:
+    i += 1
+    print(repoUrl.strip())
+    EXTRAURL.append(repoUrl.strip())
+repoListFile.close()
+
+CUSTOMIPV4FILENAME = str(os.getenv("CUSTOMIPV4FILENAME", "CustomMirrorListV4"))
+CUSTOMIPV6FILENAME = str(os.getenv("CUSTOMIPV6FILENAME", "CustomMirrorListV6"))
+
+# EXTRAURL = list(os.getenv("EXTRAURL", "https://mdu.se/"))
+IPv4FILE = f"/etc/debmirror/{CUSTOMIPV4FILENAME}"
+IPv6FILE = f"/etc/debmirror/{CUSTOMIPV6FILENAME}"
+
+def sanitizeURL(inpurl: str):
+    if "https://" in inpurl:
+        outurl = inpurl[8:]
+    elif "http://" in inpurl:
+        outurl = inpurl[7:]
+    elif "" in inpurl:
+        return 7
+    elif "http://" or "https://" not in url:
+        outurl = inpurl
+    else:
+        return inpurl
+
+    i = 0
+    for char in outurl:
+        i += 1
+        if char == "/":
+            outurl = outurl[:i]
+
+    if char == "/":
+        outurl = outurl[:-1]
+    return outurl
+
+def LeJob():
+
+    print("Starting lookup")
+
+    # print(LeMirrorDict)
+
+    with open(IPv4FILE, "w",) as fW:
+
+        for url in EXTRAURL:
+            goodurl = sanitizeURL(url)
+            # print(goodurl)
+
+            if goodurl == 7:
+                continue
+
+            # ip4Dict = ermWhatATheIpFromDomainYaCrazy(url)
+            ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
+
+            try:
+                for key, ip in ip4Dict.items():
+                    print(ip + "/32")
+
+                    fW.write(ip + "/32" +  "\n")
+            except AttributeError: 
+                continue
+                        
+
+    with open(IPv6FILE, "w",) as fW:
+
+        for url in EXTRAURL:
+            goodurl = sanitizeURL(url)
+
+            if goodurl == 7:
+                continue
+            # print(goodurl)
+
+            # ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url)
+            ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
+
+            try:
+                for key, ip in ip6Dict.items():
+                    print(ip + "/128")
+
+                    fW.write(ip + "/128" + "\n")
+            except AttributeError: 
+                continue
+
+
+
+
+LeJob()
+
+print("Done")

AiO_Container/code/mainDocker.py

@@ -0,0 +1,165 @@
+import requests, schedule, time, os
+from bs4 import BeautifulSoup
+from whatDomain import *
+
+DEBMIRRORURL = str(os.getenv("DEBMIRRORURL", "https://www.debian.org/mirror/list"))
+DEBSECURITYURL = str(os.getenv("DEBSECURITYURL", "https://security.debian.org/debian-security/"))
+
+EXTRASURL = ["https://download.docker.com/linux/debian/", 
+            # Double just to be sure. Even though they point to the same IP
+            "http://download.proxmox.com/debian/", 
+            "https://enterprise.proxmox.com/debian/pve/",
+            # That's for nvidia docker toolkit something something
+            # stuff that makes containers use nvenc and shit
+            "https://nvidia.github.io/libnvidia-container/stable/deb/",
+            "https://nvidia.github.io/libnvidia-container/experimental/deb/"]
+
+IPv4FILE = "/etc/debmirror/MirrorListV4"
+IPv6FILE = "/etc/debmirror/MirrorListV6"
+
+
+# Define EU and American countries as well as Security for security updates
+target_countries = set([
+    # Europe
+    "Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark",
+    "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland",
+    "Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland",
+    "Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland",
+    "United Kingdom", "Moldova",
+    # America
+    "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
+    "Mexico", "Peru", "United States", "Uruguay", "Venezuela",
+    # Others
+    "Security", "Extras"
+])
+
+
+def sanitizeURL(inpurl: str):
+    if "https://" in inpurl:
+        outurl = inpurl[8:]
+    elif "http://" in inpurl:
+        outurl = inpurl[7:]
+    elif "" in inpurl:
+        return 7
+    elif "http://" or "https://" not in url:
+        outurl = inpurl
+    else:
+        return inpurl
+
+    i = 0
+    for char in outurl:
+        i += 1
+        if char == "/":
+            outurl = outurl[:i]
+
+    if char == "/":
+        outurl = outurl[:-1]
+    return outurl
+
+
+def getFreshData():
+    payload = requests.get(DEBMIRRORURL)
+    soup = BeautifulSoup(payload.content, "html.parser")
+    return soup
+
+
+def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
+
+    outMirrorDict = {}
+    current_country = None
+
+    # Iterate through all table rows
+    for table in SoupInput.find_all("table"):
+        for row in table.find_all("tr"):
+            # Check for country name in a full-row header (<strong><big>)
+            strong = row.find("strong")
+            if strong:
+                country_name = strong.get_text(strip=True)
+                if country_name in target_countries:
+                    current_country = country_name
+                else:
+                    current_country = None
+                continue  # move to next row
+
+            # Check for inline country name in first column
+            cols = row.find_all("td")
+            if len(cols) >= 2:
+                possible_country = cols[0].get_text(strip=True)
+                link_tag = cols[1].find("a", href=True)
+                if possible_country in target_countries:
+                    current_country = possible_country
+                if current_country and link_tag:
+                    url = link_tag['href']
+                    if current_country not in outMirrorDict:
+                        outMirrorDict[current_country] = []
+                    outMirrorDict[current_country].append(url)
+
+    if os.environ["SECURITYREPOS"]:
+        outMirrorDict.update({"Security": DEBSECURITYURL})
+    if os.environ["EXTRAREPOS"]:
+        outMirrorDict.update({"Extras": EXTRASURL})
+
+    return outMirrorDict
+
+def LeJob():
+
+    print("Starting lookup")
+
+    LeSoup = getFreshData()
+
+    LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)
+
+    # print(LeMirrorDict)
+
+    with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
+
+        for key, urls in LeMirrorDict.items():
+            # print(urls)
+            if key in target_countries:
+                
+                for url in urls:
+                    # print(url)
+                    
+                    if url not in fR:
+
+                        goodurl = sanitizeURL(url)
+                        # print(goodurl)
+
+                        ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
+
+                        try:
+                            for key, ip in ip4Dict.items():
+                                print(ip)
+
+                                fW.write(ip + "/32" +  "\n")
+                        except AttributeError: 
+                            continue
+                        
+
+    with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
+
+        for key, urls in LeMirrorDict.items():
+            if key in target_countries:
+                
+                for url in urls:
+                    
+                    if url not in fR:
+
+                        goodurl = sanitizeURL(url)
+                        # print(goodurl)
+
+                        ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
+
+                        try:
+                            for key, ip in ip6Dict.items():
+                                # print(ip)
+
+                                fW.write(ip + "/128" + "\n")
+                        except AttributeError:
+                            continue
+
+
+
+LeJob()
+
+print("Done")

AiO_Container/code/mainOPNsense.py

@@ -0,0 +1,135 @@
+import requests, schedule, time
+from bs4 import BeautifulSoup
+from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
+
+
+OPNSNSMIRRORURL = str(os.getenv("OPNSNSMIRRORURL", "https://opnsense.org/download/#full-mirror-listing"))
+ 
+IPv4FILE = str(os.getenv("IPv4FILE", "/etc/debmirror/OPNS_MirrorListV4"))
+IPv6FILE = str(os.getenv("IPv6FILE", "/etc/debmirror/OPNS_MirrorListV6"))
+
+
+def sanitizeURL(inpurl: str):
+    if not "/" in inpurl[:-1]:
+        inpurl += "/" 
+    if "https://" in inpurl:
+        outurl = inpurl[8:]
+    elif "http://" in inpurl:
+        outurl = inpurl[7:]
+    elif "http://" or "https://" not in url:
+        outurl = inpurl
+    else:
+        return -1
+
+    # how the fuck does it work?
+    # I mean I wrote this but I don't know why does it work.
+    i = 0
+    for char in outurl:
+        i += 1
+        if char == "/":
+            outurl = outurl[:i]
+
+    if char == "/":
+        outurl = outurl[:-1]
+    return outurl
+
+
+def getFreshData():
+    payload = requests.get(OPNSNSMIRRORURL)
+    soup = BeautifulSoup(payload.content, "html.parser")
+    return soup
+
+def LeJob():
+
+    print("Starting lookup")
+
+    LeSoup = getFreshData()
+
+    # print(LeMirrorDict)
+
+    with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
+
+        for data in LeSoup.find_all('div', class_='download_section'):
+            for a in data.find_all('a', href=True):
+
+                url = a['href']
+
+                saniturl = sanitizeURL(url)
+
+                # print(saniturl)
+                IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
+
+                # print(IPv4Dict)
+                try:
+                    for key, ip in IPv4Dict.items():
+                        print(f"Found the ipv4: {ip}")
+                        fW.write(ip + "/32" +  "\n")
+
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue 
+
+    with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
+
+        for data in LeSoup.find_all('div', class_='download_section'):
+            for a in data.find_all('a', href=True):
+                
+                url = a['href']
+
+                saniturl = sanitizeURL(url)
+
+                # print(saniturl)
+                IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
+
+                # print(IPv6Dict)
+                try:
+                    for key, ip in IPv6Dict.items():
+                        print(f"Found the ipv6: {ip}")
+                        fW.write(ip + "/128" + "\n")
+
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue
+
+# schedule.every().day.at("12:45").do(LeJob)
+# schedule.every().day.at("17:44").do(LeJob)
+
+# while True:
+#     schedule.run_pending()
+#     print("Waiting...")
+#     time.sleep(30)  #Wait one minute
+#     # LeJob()
+
+LeJob()
+
+# gigalist = []
+
+# payload = requests.get(OPNSNSMIRRORURL)
+# soup = BeautifulSoup(payload.content, "html.parser")
+
+
+# for data in soup.find_all('div', class_='download_section'):
+#     for a in data.find_all('a', href=True):
+
+#         url = a['href']
+
+#         saniturl = sanitizeURL(url)
+
+#         # print(saniturl)
+#         IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
+#         IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
+
+#         # print(IPv4Dict)
+#         try:
+#             for key, ip in IPv4Dict.items():
+#                 print(f"Found the ipv4: {ip}")
+
+#             for key, ip in IPv6Dict.items():
+#                 print(f"Found the ipv6: {ip}")
+#         # If int returned from WhatDomain then it is error.
+#         # Error type is described via prints from whatdomain functions
+#         except AttributeError:
+#             continue
+

AiO_Container/code/opnsense.py

@@ -0,0 +1,135 @@
+import requests, schedule, time
+from bs4 import BeautifulSoup
+from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
+
+
+
+OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing"
+IPv4FILE = "./OPNS_MirrorListV4"
+IPv6FILE = "./OPNS_MirrorListV6"
+
+
+def sanitizeURL(inpurl: str):
+    if not "/" in inpurl[:-1]:
+        inpurl += "/" 
+    if "https://" in inpurl:
+        outurl = inpurl[8:]
+    elif "http://" in inpurl:
+        outurl = inpurl[7:]
+    elif "http://" or "https://" not in url:
+        outurl = inpurl
+    else:
+        return -1
+
+    # how the fuck does it work?
+    # I mean I wrote this but I don't know why does it work.
+    i = 0
+    for char in outurl:
+        i += 1
+        if char == "/":
+            outurl = outurl[:i]
+
+    if char == "/":
+        outurl = outurl[:-1]
+    return outurl
+
+
+def getFreshData():
+    payload = requests.get(OPNSNSMIRRORURL)
+    soup = BeautifulSoup(payload.content, "html.parser")
+    return soup
+
+def LeJob():
+
+    print("Starting lookup")
+
+    LeSoup = getFreshData()
+
+    # print(LeMirrorDict)
+
+    with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
+
+        for data in LeSoup.find_all('div', class_='download_section'):
+            for a in data.find_all('a', href=True):
+
+                url = a['href']
+
+                saniturl = sanitizeURL(url)
+
+                # print(saniturl)
+                IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
+
+                # print(IPv4Dict)
+                try:
+                    for key, ip in IPv4Dict.items():
+                        print(f"Found the ipv4: {ip}")
+                        fW.write(ip + "/32" +  "\n")
+
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue 
+
+    with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
+
+        for data in LeSoup.find_all('div', class_='download_section'):
+            for a in data.find_all('a', href=True):
+                
+                url = a['href']
+
+                saniturl = sanitizeURL(url)
+
+                # print(saniturl)
+                IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
+
+                # print(IPv6Dict)
+                try:
+                    for key, ip in IPv6Dict.items():
+                        print(f"Found the ipv6: {ip}")
+                        fW.write(ip + "/128" + "\n")
+
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue
+
+# schedule.every().day.at("12:45").do(LeJob)
+# schedule.every().day.at("17:44").do(LeJob)
+
+# while True:
+#     schedule.run_pending()
+#     print("Waiting...")
+#     time.sleep(30)  #Wait one minute
+#     # LeJob()
+
+LeJob()
+
+# gigalist = []
+
+# payload = requests.get(OPNSNSMIRRORURL)
+# soup = BeautifulSoup(payload.content, "html.parser")
+
+
+# for data in soup.find_all('div', class_='download_section'):
+#     for a in data.find_all('a', href=True):
+
+#         url = a['href']
+
+#         saniturl = sanitizeURL(url)
+
+#         # print(saniturl)
+#         IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
+#         IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
+
+#         # print(IPv4Dict)
+#         try:
+#             for key, ip in IPv4Dict.items():
+#                 print(f"Found the ipv4: {ip}")
+
+#             for key, ip in IPv6Dict.items():
+#                 print(f"Found the ipv6: {ip}")
+#         # If int returned from WhatDomain then it is error.
+#         # Error type is described via prints from whatdomain functions
+#         except AttributeError:
+#             continue
+

AiO_Container/code/whatDomain.py

@@ -0,0 +1,129 @@
+#from nslookup import Nslookup
+from typing import Optional, Annotated
+import dns, dns.resolver
+
+# https://www.codeunderscored.com/nslookup-python/
+
+def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
+    #dns_query = Nslookup()
+    """
+    Tells you what IPv4 address/es a domain point to.
+    Returns:
+        dict: A dictionary with IP addresses associated with that domain.
+
+    """
+
+    # i = 0
+    outDict = {} 
+
+    #result = dns_query.dns_lookup("example.com")
+    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
+    try:
+        result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
+    except dns.resolver.NoAnswer:
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
+    for i, something in enumerate(result):
+        outDict[i] = something.to_text()
+        # i += 1
+
+    return outDict
+
+def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
+    #dns_query = Nslookup()
+    """
+    Tells you what IPv6 address/es a domain point to.
+    Returns:
+        dict: A dictionary with IP addresses associated with that domain.
+
+    """
+
+
+    # i = 0
+    outDict = {} 
+
+    #result = dns_query.dns_lookup("example.com")
+    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
+    try:
+        result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
+    except dns.resolver.NoAnswer:
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
+    for i, something in enumerate(result):
+        outDict[i] = something.to_text()
+        # i += 1
+
+    return outDict
+
+
+def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]):
+    #dns_query = Nslookup()
+    """
+    Tells you what IPv6 address/es a domain point to.
+    Returns:
+        dict: A dictionary with IP addresses associated with that domain.
+
+    """
+
+    whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa"
+
+
+    # i = 0
+    outDict = {}
+
+    #result = dns_query.dns_lookup("example.com")
+    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
+    try:
+        result = dns.resolver.resolve(whatToCheck, 'PTR')
+    except dns.resolver.NoAnswer:
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
+    for i, something in enumerate(result):
+        outDict[i] = something.to_text()
+        # i += 1
+
+    return outDict
+
+
+#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
+#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
+#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))

AiO_Container/cron-jobs

@@ -0,0 +1,11 @@
+# python
+20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+
+# git push 
+30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh
+#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh

AiO_Container/entrypoint.sh

@@ -0,0 +1,79 @@
+#!/bin/sh
+set -e
+
+# Default to 1600 if not provided
+PUID="${PUID:-1600}"
+PGID="${PGID:-1600}"
+USER="${USER:-muyu}"
+GROUP="${USER:-muyu}"
+CHOWNPATH="/etc/debmirror"
+EXTRA_REPOS="${EXTRA_REPOS:-False}"
+SECURITY_REPOS="${SECURITY_REPOS:-True}"
+DEBIAN_REPOS="${DEBIAN_REPOS:-True}"
+OPNSENSE_REPOS="${OPNSENSE_REPOS:-True}"
+
+# Create group if missing
+if ! getent group "$GROUP" >/dev/null 2>&1; then
+    addgroup -g "$PGID" "$GROUP"
+fi
+
+# Create user if missing
+if ! id -u "$USER" >/dev/null 2>&1; then
+    adduser -D -u "$PUID" -G "$GROUP" "$USER"
+fi
+
+# Prepare cron file 
+touch /etc/crontabs/cron-jobs
+
+# Create symlinks based on what is passed
+# Also add cronjobs when applicable 
+git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists
+if [[ "$EXTRA_REPOS" ]]; then
+    ln -s /etc/debmirror/Repo-IP-lists/CustomMirrorListV4 /etc/debmirror/CustomMirrorListV4
+    ln -s /etc/debmirror/Repo-IP-lists/CustomMirrorListV6 /etc/debmirror/CustomMirrorListV6
+
+    echo "00 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainCustom.py"  >> /etc/crontabs/cron-jobs
+fi
+if [[ "$SECURITY_REPOS" ]]; then
+    ln -s /etc/debmirror/Repo-IP-lists/SecurityMirrorListV4 /etc/debmirror/SecurityMirrorListV4
+    ln -s /etc/debmirror/Repo-IP-lists/SecurityMirrorListV6 /etc/debmirror/SecurityMirrorListV6
+
+    echo "05 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainSec.py"  >> /etc/crontabs/cron-jobs
+fi
+if [[ "$DEBIAN_REPOS" ]]; then
+    ln -s /etc/debmirror/Repo-IP-lists/DebianMirrorListV4 /etc/debmirror/DebianMirrorListV4 
+    ln -s /etc/debmirror/Repo-IP-lists/DebianMirrorListV6 /etc/debmirror/DebianMirrorListV6 
+
+    echo "10 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py"  >> /etc/crontabs/cron-jobs
+fi
+if [[ "$OPNSENSE_REPOS" ]]; then
+    ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMIrrorListV4 /etc/debmirror/OPNsenseMIrrorListV4 
+    ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMIrrorListV6 /etc/debmirror/OPNsenseMIrrorListV6
+    
+    echo "15 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py"  >> /etc/crontabs/cron-jobs
+fi
+
+echo "30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh"  >> /etc/crontabs/cron-jobs
+
+# Configure the crontab to work 
+chmod 0644 /etc/crontabs/cron-jobs
+crontab /etc/crontabs/cron-jobs
+
+# Fix permissions
+chown -R "$PUID:$PGID" "$CHOWNPATH"
+
+# Drop privileges & run command
+exec su-exec "$PUID:$PGID" "$@"
+
+
+# # python
+# 20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+# 25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+# # 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+# # 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+# #*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+# #*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+
+# # git push 
+# 30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh
+# #*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh

AiO_Container/gitPush.sh

@@ -0,0 +1,100 @@
+#!/bin/sh
+set -e
+
+# Check if these are passed and terminate if they are not 
+GITEA_TOKEN="${GITEA_TOKEN:-6767}"
+GITURL="${GITURL:-6767}"
+GITREPOPATH="${GITREPOPATH:-6767}"
+GITURLPROTO="${GITURLPROTO:-http}"
+
+# if [[ "$GITEA_TOKEN" == "6767" || "$GITREPOPATH" == "6767" || "$GITURL" == "6767" ]]; then
+if [[ "$GITEA_TOKEN" == "6767" ]]; then
+    exit 1
+fi
+
+WORKPTH="${WORKPTH:-/etc/debmirror/}"
+REPO_DIR="${REPO_DIR:-$WORKPTH/Repo-IP-lists}"
+REPO_URL="${REPO_URL:-GITURLPROTO://GITURL/GITREPOPATH}"
+
+
+# Clone repo if not exists
+if [[ ! -d "$REPO_DIR/.git" ]]; then
+    echo "[$(date)] Cloning repository..."
+    git clone "$REPO_URL" "$REPO_DIR"
+fi
+
+cd "$REPO_DIR"
+
+# Abort previous rebase/cherry-pick if stuck
+git rebase --abort 2>/dev/null || true
+git cherry-pick --abort 2>/dev/null || true
+
+# Make sure we're on a clean 'main'
+git checkout main || git checkout -b main origin/main
+git pull --rebase --autostash
+
+# git remote set-url origin "${GITURLPROTO}://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
+git remote set-url origin "https://${GITEA_TOKEN}@YuruC3/Repo-IP-lists"
+
+
+git config user.name "UpdateBot"
+git config user.email "UpdateBot@localhost.local"
+
+# Stage the files
+git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
+
+# Only proceed if there are staged changes
+if [[ ! git diff --quiet --cached ]]; then
+    echo "[$(date)] Committing and pushing changes..."
+    git commit -m "Auto-update mirror list on $(date -Iseconds)"
+    # git pull --rebase --autostash
+    git push --quiet
+    echo "[$(date)] Changes pushed."
+else
+    echo "[$(date)] No changes to commit or push."
+fi
+
+
+# #!/bin/sh
+# set -e
+
+# WORKPTH="/etc/debmirror/"
+# REPO_DIRd="$WORKPTH/Repo-IP-lists"
+# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
+
+
+# # Clone repo only if it doesn't already exist
+# if [ ! -d "$REPO_DIR/.git" ]; then
+#     echo "[$(date)] Cloning repository..."
+#     git clone "$REPO_URL" "$REPO_DIR"
+# fi
+
+
+# cd "$REPO_DIR"
+
+# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
+
+# git config user.name "UpdateBot"
+# git config user.email "UpdateBot@localhost.local"
+
+# # stage files
+# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
+
+# # If anything to commit locally, commit it now
+# if ! git diff --quiet --cached; then
+#     echo "[$(date)] Committing local changes before pulling"
+#     git commit -m "Auto-commit before pull on $(date -Iseconds)"
+# fi
+
+# # Now pull the latest
+# git pull --rebase --autostash
+
+# # Commit and push only if there's anything new staged
+# if git diff --quiet; then
+#     echo "[$(date)] No changes to commit."
+# else
+#     git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet
+#     git push --quiet
+#     echo "[$(date)] Changes pushed successfully."
+# fi
+

AiO_Container/init.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+echo "EXTRAURL: $EXTRAURL"
+
+if [[ -f "$REPOFILE" ]]; then
+    echo "URL file exists"
+else
+    touch $REPOFILE
+    echo $EXTRAURL | tee $REPOFILE
+fi
+
+echo "nameserver $DNSSRV" > /etc/resolv.conf 
+echo "search local" >> /etc/resolv.conf
+
+git clone https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}
+ln -s /etc/customMirrors/Repo-IP-lists/$IPV4FILENAME /etc/customMirrors/$IPV4FILENAME
+ln -s /etc/customMirrors/Repo-IP-lists/$IPV6FILENAME /etc/customMirrors/$IPV6FILENAME
+
+echo -n "$CRONTABSET /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py" > /etc/crontabs/customListCron
+
+exec /usr/sbin/crond -f

AiO_Container/requirements.txt

@@ -0,0 +1,4 @@
+beautifulsoup4==4.13.4
+requests==2.32.3
+schedule==1.2.2
+nslookup==1.8.1