InProgress Revamp of domainLookup

AiO_Container/Dockerfile

@@ -1,11 +1,12 @@
 FROM alpine:latest
 
 # https://docs.docker.com/reference/dockerfile/#environment-replacement
-ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
+# ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
 
 # By default these are scraped
-ENV EXTRA_REPOS=False
-ENV SECURITY_REPOS=True
+ENV EXTRA_REPOS=True
+ENV PROXMOX_REPOS=True
+ENV DOCKER_REPOS=True
 ENV DEBIAN_REPOS=True
 ENV OPNSENSE_REPOS=True
 
@@ -25,7 +26,12 @@ WORKDIR /etc/debmirror/
 #     touch /etc/debmirror/MirrorListV4
 
 # Copy code
-COPY ./code/* /etc/debmirror/
+COPY ./code/main.py /etc/debmirror/
+COPY ./code/whatDomain.py /etc/debmirror/
+
+
+COPY ./entrypoint.sh /
+RUN chmod +x /entrypoint.sh
 
 # Copy important files
 # COPY ./init.sh /etc/debmirror/

AiO_Container/code/main.py

@@ -0,0 +1,408 @@
+import requests, schedule, time, os
+from bs4 import BeautifulSoup
+from whatDomain import *
+# import whatDomain
+
+# Debian
+DEBMIRRORURL = str(os.getenv("DEBMIRRORURL", "https://www.debian.org/mirror/list"))
+DEBSECURITYURL = str(os.getenv("DEBSECURITYURL", "https://security.debian.org/debian-security/"))
+DEBIANMIRRORLISTV4 = "/etc/debmirror/Repo-IP-lists/DebianMirrorListV4"
+DEBIANMIRRORLISTV6 = "/etc/debmirror/Repo-IP-lists/DebianMirrorListV6"
+
+target_countries = set([
+    # Europe
+    "Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark",
+    "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland",
+    "Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland",
+    "Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland",
+    "United Kingdom", "Moldova",
+    # America
+    "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
+    "Mexico", "Peru", "United States", "Uruguay", "Venezuela"  #,
+    # Others
+    # "Security", "Extras"
+])
+
+
+# Proxmox
+PROXURL = ["http://download.proxmox.com/debian/", "https://enterprise.proxmox.com/debian/pve/",]
+PROXMOXMIRRORLISTV4 = "/etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV4"
+PROXMOXMIRRORLISTV6 = "/etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV6"
+
+
+# Docker
+DOCKERURLS = ["https://download.docker.com/linux/debian/", 
+            "https://nvidia.github.io/libnvidia-container/stable/deb/",
+            "https://nvidia.github.io/libnvidia-container/experimental/deb/"]
+DOCKERMIRRORLISTV4 = "/etc/debmirror/Repo-IP-lists/DockerMirrorListV4"
+DOCKERMIRRORLISTV6 = "/etc/debmirror/Repo-IP-lists/DockerMirrorListV6"
+
+
+# OPNsense
+OPNSNSMIRRORURL = str(os.getenv("OPNSNSMIRRORURL", "https://opnsense.org/download/#full-mirror-listing"))
+OPNSENSEMIRRORLISTV4 =  "/etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV4"
+OPNSENSEMIRRORLISTV6 = "/etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV6"
+
+
+# Custom
+EXTRAURL = []
+if os.getenv("EXTRA_REPOS", True):
+    try:
+        with open(os.getenv("REPOFILE", "/etc/debmirror/customRepoList.list"), 'r') as repoListFile:
+            for repoUrl in repoListFile:
+                # print(repoUrl.strip())
+                EXTRAURL.append(repoUrl.strip())
+    except FileNotFoundError:
+        print("File not passed")
+        exit(2)
+
+CUSTOMMIRRORLISTV4 = "/etc/debmirror/Repo-IP-lists/CustomMirrorListV4"
+CUSTOMMIRRORLISTV6 = "/etc/debmirror/Repo-IP-lists/CustomMirrorListV6"
+
+
+# Helper
+def sanitizeURL(inpurl: str):
+    if "https://" in inpurl:
+        outurl = inpurl[8:]
+    elif "http://" in inpurl:
+        outurl = inpurl[7:]
+    # elif " " in inpurl:
+    #     return 7
+    elif "http://" or "https://" not in inpurl:
+        outurl = inpurl
+    else:
+        return inpurl
+
+    i = 0
+    for char in outurl:
+        i += 1
+        if char == "/":
+            outurl = outurl[:i]
+
+    if char == "/":
+        outurl = outurl[:-1]
+    return outurl
+
+
+
+# Debian
+def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
+
+    outMirrorDict = {}
+    current_country = None
+
+    # Iterate through all table rows
+    for table in SoupInput.find_all("table"):
+        for row in table.find_all("tr"):
+            # Check for country name in a full-row header (<strong><big>)
+            strong = row.find("strong")
+            if strong:
+                country_name = strong.get_text(strip=True)
+                if country_name in target_countries:
+                    current_country = country_name
+                else:
+                    current_country = None
+                continue  # move to next row
+
+            # Check for inline country name in first column
+            cols = row.find_all("td")
+            if len(cols) >= 2:
+                possible_country = cols[0].get_text(strip=True)
+                link_tag = cols[1].find("a", href=True)
+                if possible_country in target_countries:
+                    current_country = possible_country
+                if current_country and link_tag:
+                    url = link_tag['href']
+                    if current_country not in outMirrorDict:
+                        outMirrorDict[current_country] = []
+                    outMirrorDict[current_country].append(url)
+
+    # if os.environ["SECURITYREPOS"]:
+    outMirrorDict.update({"Security": DEBSECURITYURL})
+
+    return outMirrorDict
+
+def debianJob():
+
+    print("\nStarting lookup for Debian")
+
+    payload = requests.get(DEBMIRRORURL)
+    LeSoup = BeautifulSoup(payload.content, "html.parser")
+
+    LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)
+
+    # print(LeMirrorDict)
+
+    with open(DEBIANMIRRORLISTV4, "r",) as fR, open(DEBIANMIRRORLISTV4, "w",) as fW:
+
+        for key, urls in LeMirrorDict.items():
+            # print(urls)
+            if key in target_countries:
+                
+                for url in urls:
+                    # print(url)
+                    
+                    if url not in fR:
+
+                        goodurl = sanitizeURL(url)
+                
+                        if goodurl == 7:
+                            continue
+                        # print(goodurl)
+
+                        ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
+
+                        try:
+                            for key, ip in ip4Dict.items():
+                                print(ip)
+
+                                fW.write(ip + "/32" +  "\n")
+                        except AttributeError: 
+                            continue
+                        
+
+    with open(DEBIANMIRRORLISTV6, "r",) as fR, open(DEBIANMIRRORLISTV6, "w",) as fW:
+
+        for key, urls in LeMirrorDict.items():
+            if key in target_countries:
+                
+                for url in urls:
+                    
+                    if url not in fR:
+
+                        goodurl = sanitizeURL(url)
+                
+                        if goodurl == 7:
+                            continue
+                        # print(goodurl)
+
+                        ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
+
+                        try:
+                            for key, ip in ip6Dict.items():
+                                # print(ip)
+
+                                fW.write(ip + "/128" + "\n")
+                        except AttributeError:
+                            continue
+
+
+
+# Proxmox
+def proxmoxJob():
+
+    print("\nStarting lookup for Proxmox")
+
+    with open(PROXMOXMIRRORLISTV4, "r",) as fR, open(PROXMOXMIRRORLISTV4, "w",) as fW:
+
+        for url in PROXURL:
+            if url not in fR:
+                sanitizedURL = sanitizeURL(url)
+                
+                if sanitizedURL == 7:
+                    continue
+
+                ip4Dict = ermWhatATheIpFromDomainYaCrazy(sanitizedURL)
+
+                try:
+                    for key, ip in ip4Dict.items():
+                        print(ip)
+                        fW.write(ip + "/32" +  "\n")
+                except AttributeError: 
+                    continue
+
+    with open(PROXMOXMIRRORLISTV6, "r",) as fR, open(PROXMOXMIRRORLISTV6, "w",) as fW:
+
+        for url in PROXURL:
+            if url not in fR:
+                sanitizedURL = sanitizeURL(url)
+                
+                if sanitizedURL == 7:
+                    continue
+
+                ip4Dict = ermWhatAAAATheIpFromDomainYaCrazy(sanitizedURL)
+
+                try:
+                    for key, ip in ip4Dict.items():
+                        print(ip)
+                        fW.write(ip + "/128" +  "\n")
+                except AttributeError: 
+                    continue
+
+
+# OPNsense
+def opnsenseJob():
+
+    print("\nStarting lookup for OPNSense")
+
+    payload = requests.get(OPNSNSMIRRORURL)
+    LeOPNSoup = BeautifulSoup(payload.content, "html.parser")
+
+    # print(LeMirrorDict)
+
+    with open(OPNSENSEMIRRORLISTV4, "r",) as fR, open(OPNSENSEMIRRORLISTV4, "w",) as fW:
+
+        for data in LeOPNSoup.find_all('div', class_='download_section'):
+            for a in data.find_all('a', href=True):
+
+                url = a['href']
+
+                saniturl = sanitizeURL(url)
+                
+                if saniturl == 7:
+                    continue
+
+                # print(saniturl)
+                IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
+
+                # print(IPv4Dict)
+                try:
+                    for key, ip in IPv4Dict.items():
+                        print(f"Found the ipv4: {ip}")
+                        fW.write(ip + "/32" +  "\n")
+
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue 
+
+    with open(OPNSENSEMIRRORLISTV6, "r",) as fR, open(OPNSENSEMIRRORLISTV6, "w",) as fW:
+
+        for data in LeOPNSoup.find_all('div', class_='download_section'):
+            for a in data.find_all('a', href=True):
+                
+                url = a['href']
+
+                saniturl = sanitizeURL(url)
+                
+                if saniturl == 7:
+                    continue
+
+                # print(saniturl)
+                IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
+
+                # print(IPv6Dict)
+                try:
+                    for key, ip in IPv6Dict.items():
+                        print(f"Found the ipv6: {ip}")
+                        fW.write(ip + "/128" + "\n")
+
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue
+
+# Docker
+def dockerJob():
+
+    print("\nStarting lookup for Docker")
+
+    with open(DOCKERMIRRORLISTV4, "r",) as fR, open(DOCKERMIRRORLISTV4, "w",) as fW:
+
+        for url in DOCKERURLS:
+            if url not in fR:
+                sanitizedURL = sanitizeURL(url)
+                
+                if sanitizedURL == 7:
+                    continue
+
+                ip4Dict = ermWhatATheIpFromDomainYaCrazy(sanitizedURL)
+
+                try:
+                    for key, ip in ip4Dict.items():
+                        print(ip)
+                        fW.write(ip + "/32" +  "\n")
+                except AttributeError: 
+                    continue
+    with open(DOCKERMIRRORLISTV6, "r",) as fR, open(DOCKERMIRRORLISTV6, "w",) as fW:
+
+        for url in DOCKERURLS:
+            if url not in fR:
+                sanitizedURL = sanitizeURL(url)
+
+                if sanitizedURL == 7:
+                    continue
+
+                ip4Dict = ermWhatAAAATheIpFromDomainYaCrazy(sanitizedURL)
+
+                try:
+                    for key, ip in ip4Dict.items():
+                        print(ip)
+                        fW.write(ip + "/128" +  "\n")
+                except AttributeError: 
+                    continue
+
+# Custom
+def customLinksJob():
+
+    print("\nStarting lookup for Custom domains")
+
+    with open(CUSTOMMIRRORLISTV4, "w",) as fW, open(CUSTOMMIRRORLISTV4, "r",) as fR:
+
+        for url in EXTRAURL:
+            # print(url)
+            goodurl = sanitizeURL(url)
+            # print(goodurl)
+            if url not in fR:
+                if goodurl == 7:
+                    continue
+                # print(goodurl)
+
+                # ip4Dict = ermWhatATheIpFromDomainYaCrazy(url)
+                ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
+
+                try:
+                    for key, ip in ip4Dict.items():
+                        print(ip + "/32")
+
+                        fW.write(ip + "/32" +  "\n")
+                except AttributeError: 
+                    continue
+                        
+
+    with open(CUSTOMMIRRORLISTV6, "w",) as fW, open(CUSTOMMIRRORLISTV6, "r",) as fR:
+
+        for url in EXTRAURL:
+            goodurl = sanitizeURL(url)
+            # print(goodurl)
+            if url not in fR:
+
+                if goodurl == 7:
+                    continue
+                # print(goodurl)
+
+                # ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url)
+                ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
+
+                try:
+                    for key, ip in ip6Dict.items():
+                        print(ip + "/128")
+
+                        fW.write(ip + "/128" + "\n")
+                except AttributeError: 
+                    continue
+
+
+
+
+
+if __name__ == "__main__":
+
+    if os.getenv("EXTRA_REPOS", True):
+        customLinksJob()
+
+    
+    if os.getenv("PROXMOX_REPOS", True):
+        proxmoxJob()
+
+    
+    if os.getenv("DOCKER_REPOS", True):
+        dockerJob()
+
+    
+    if os.getenv("DEBIAN_REPOS", True):
+        debianJob()
+
+    
+    if os.getenv("OPNSENSE_REPOS", True):
+        opnsenseJob()

AiO_Container/code/mainCustom.py

@@ -1,6 +1,7 @@
 import requests, schedule, time, os
 # from bs4 import BeautifulSoup
-from whatDomain import *
+# from whatDomain import *
+import whatDomain
 
 EXTRAURL = []
 
@@ -90,7 +91,7 @@ def LeJob():
 
 
 
+if __name__ == "__main__":
+    LeJob()
 
-LeJob()
-
-print("Done")
+    print("Done")

AiO_Container/code/mainDocker.py

@@ -10,7 +10,7 @@ EXTRASURL = ["https://download.docker.com/linux/debian/",
             "http://download.proxmox.com/debian/", 
             "https://enterprise.proxmox.com/debian/pve/",
             # That's for nvidia docker toolkit something something
-            # stuff that makes containers use nvenc and shit
+            # stuff that makes containers use nvenc
             "https://nvidia.github.io/libnvidia-container/stable/deb/",
             "https://nvidia.github.io/libnvidia-container/experimental/deb/"]
 

AiO_Container/code/momoid.py

@@ -0,0 +1,77 @@
+# https://dnspython.readthedocs.io/en/latest/
+import subprocess
+from typing import Optional, Annotated
+# import dns, dns.resolver
+
+
+class Momoid:
+
+    def __init__(
+        self,
+        resolverIPaddr: Annotated[int, "IP Address of a DNS server"] = "1.1.1.1",
+        DoH: Annotated[bool, "If DNS over HTTPS should be used"] = True
+        ): 
+
+        if DoH:
+            self.__MOMOID_CMD = ["dig", "+short", "+https", f"@{resolverIPaddr}"]
+        else:
+            self.__MOMOID_CMD = ["dig", "+short", f"@{resolverIPaddr}"]
+        
+
+    def recordA(self, domain: Annotated[str, "Domain to resolve to a A record"]) -> list:
+        if not domain:
+            return 0
+
+        outpt = subprocess.run([*self.__MOMOID_CMD, domain, "A"], capture_output=True, text=True)
+        
+        if outpt.stdout.strip() == "":
+            return 0
+        else:
+            outList = outpt.stdout.strip().split("\n")
+            return outList
+
+    def recordAAAA(self, domain: Annotated[str, "Domain to resolve to a AAAA record"]) -> list:
+        if not domain:
+            return 0
+
+        outpt = subprocess.run([*self.__MOMOID_CMD, domain, "AAAA"], capture_output=True, text=True)
+        
+        if outpt.stdout.strip() == "":
+            return 0
+        else:
+            outList = outpt.stdout.strip().split("\n")
+            return outList
+            
+    def recordPTR(self, domain: Annotated[str, "Domain to resolve to a PTR record"]) -> list:
+        # 1.2.3.4 -> 4.3.2.1.in-addr.arpa
+        if not domain:
+            return 0
+
+        outpt = subprocess.run([*self.__MOMOID_CMD, domain, "PTR"], capture_output=True, text=True)
+        
+        if outpt.stdout.strip() == "":
+            return 0
+        else:
+            outList = outpt.stdout.strip().split("\n")
+            return outList
+        
+        
+    def __ipv4ToPTR(ipString: Annotated[str, "a string with an IPv4 address"]):
+        arpa = "in-addr.arpa"
+        ipList = ipa.split(".")
+        ipList.reverse()
+        outstr = ""
+
+        for thing in ipList:
+            outstr += thing
+            outstr += "."
+
+        return outstr + arpa
+   
+thing = Momoid("1.1.1.1", False)
+
+x = thing.recordA("shupogaki.org")
+
+print(x)
+
+

AiO_Container/code/whatDomain.py

@@ -4,7 +4,8 @@ import dns, dns.resolver
 
 # https://www.codeunderscored.com/nslookup-python/
 
-def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
+def ermWhatATheIpFromDomainYaCrazy(
+    inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
     #dns_query = Nslookup()
     """
     Tells you what IPv4 address/es a domain point to.
@@ -22,19 +23,19 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
         result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
     except dns.resolver.NoAnswer:
         print("\nDNS ERROR")
-        print("No answer from dns server.\n")
+        print(f"No answer from dns server when working on {inpDomainNameOrSomething}.\n")
         return 1
     except dns.resolver.NoNameservers:
         print("\nDNS ERROR")
-        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        print(f"All nameservers failed to answer the query when working on {inpDomainNameOrSomething}.\n Fix your DNS servers.\n")
         return 1
     except dns.resolver.NXDOMAIN:
         print("\nDNS ERROR")
-        print("The DNS query name does not exist.\n")
+        print(f"The DNS query name does not exist for domain {inpDomainNameOrSomething}.\n")
         return 1
     except dns.resolver.LifetimeTimeout:
         print("\nDNS ERROR")
-        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        print(f"The DNS querry got timed out.\nVerify that your Firewall of DNS AdBlock isn't blocking requests for that domain.\nFailed resolving {inpDomainNameOrSomething}\n")
         return 1
     for i, something in enumerate(result):
         outDict[i] = something.to_text()
@@ -124,6 +125,8 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
     return outDict
 
 
-#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
-#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
-#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))
+
+if __name__ == "__main__":
+    print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
+    print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
+    print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))

AiO_Container/docker-compose.yml

@@ -0,0 +1,21 @@
+---
+
+services:
+  debmirup:
+    container_name: Debian_Mirrors_Updater
+    # build: ./
+    image: tea.shupogaki.org/yuruc3/debianrepolist:v0.4.1
+    volumes:
+      - ./customRepoList.list:/etc/debmirror/customRepoList.list
+    environment:
+      - GITURLPROTO=https
+      - GITURL=tea.shupogaki.org
+      - GITREPOPATH=YuruC3/Repo-IP-lists.git
+      - GITEA_TOKEN=6767
+
+      - EXTRA_REPOS=True
+      - PROXMOX_REPOS=True
+      - DOCKER_REPOS=True
+      - DEBIAN_REPOS=True
+      - OPNSENSE_REPOS=True
+    restart: unless-stopped

AiO_Container/entrypoint.sh

@@ -7,11 +7,15 @@ PGID="${PGID:-1600}"
 USER="${USER:-muyu}"
 GROUP="${USER:-muyu}"
 CHOWNPATH="/etc/debmirror"
-EXTRA_REPOS="${EXTRA_REPOS:-False}"
-SECURITY_REPOS="${SECURITY_REPOS:-True}"
+
+EXTRA_REPOS="${EXTRA_REPOS:-True}"
+PROXMOX_REPOS="${PROXMOX_REPOS:-True}"
+DOCKER_REPOS="${DOCKER_REPOS:-True}"
 DEBIAN_REPOS="${DEBIAN_REPOS:-True}"
 OPNSENSE_REPOS="${OPNSENSE_REPOS:-True}"
 
+THE_REPO="${THE_REPO:-https://tea.shupogaki.org/YuruC3/Repo-IP-lists}"
+
 # Create group if missing
 if ! getent group "$GROUP" >/dev/null 2>&1; then
     addgroup -g "$PGID" "$GROUP"
@@ -27,32 +31,50 @@ touch /etc/crontabs/cron-jobs
 
 # Create symlinks based on what is passed
 # Also add cronjobs when applicable 
-git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists
-if [[ "$EXTRA_REPOS" ]]; then
+git clone $THE_REPO
+
+if [ "$EXTRA_REPOS" = true ]; then
+    touch /etc/debmirror/Repo-IP-lists/CustomMirrorListV4
+    touch /etc/debmirror/Repo-IP-lists/CustomMirrorListV6
     ln -s /etc/debmirror/Repo-IP-lists/CustomMirrorListV4 /etc/debmirror/CustomMirrorListV4
     ln -s /etc/debmirror/Repo-IP-lists/CustomMirrorListV6 /etc/debmirror/CustomMirrorListV6
 
-    echo "00 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainCustom.py"  >> /etc/crontabs/cron-jobs
+    # echo "00 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainCustom.py"  >> /etc/crontabs/cron-jobs
 fi
-if [[ "$SECURITY_REPOS" ]]; then
-    ln -s /etc/debmirror/Repo-IP-lists/SecurityMirrorListV4 /etc/debmirror/SecurityMirrorListV4
-    ln -s /etc/debmirror/Repo-IP-lists/SecurityMirrorListV6 /etc/debmirror/SecurityMirrorListV6
+if [ "$PROXMOX_REPOS" = true ]; then
+    touch /etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV4
+    touch /etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV6
+    ln -s /etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV4 /etc/debmirror/ProxmoxMirrorListV4
+    ln -s /etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV6 /etc/debmirror/ProxmoxMirrorListV6
 
-    echo "05 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainSec.py"  >> /etc/crontabs/cron-jobs
+    # echo "05 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainSec.py"  >> /etc/crontabs/cron-jobs
 fi
-if [[ "$DEBIAN_REPOS" ]]; then
+if [ "$DOCKER_REPOS" = true ]; then
+    touch /etc/debmirror/Repo-IP-lists/DockerMirrorListV4
+    touch /etc/debmirror/Repo-IP-lists/DockerMirrorListV6
+    ln -s /etc/debmirror/Repo-IP-lists/DockerMirrorListV4 /etc/debmirror/DockerMirrorListV4
+    ln -s /etc/debmirror/Repo-IP-lists/DockerMirrorListV6 /etc/debmirror/DockerMirrorListV6
+
+    # echo "05 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainSec.py"  >> /etc/crontabs/cron-jobs
+fi
+if [ "$DEBIAN_REPOS" = true ]; then
+    touch /etc/debmirror/Repo-IP-lists/DebianMirrorListV4
+    touch /etc/debmirror/Repo-IP-lists/DebianMirrorListV6
     ln -s /etc/debmirror/Repo-IP-lists/DebianMirrorListV4 /etc/debmirror/DebianMirrorListV4 
     ln -s /etc/debmirror/Repo-IP-lists/DebianMirrorListV6 /etc/debmirror/DebianMirrorListV6 
 
-    echo "10 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py"  >> /etc/crontabs/cron-jobs
+    # echo "10 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py"  >> /etc/crontabs/cron-jobs
 fi
-if [[ "$OPNSENSE_REPOS" ]]; then
-    ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMIrrorListV4 /etc/debmirror/OPNsenseMIrrorListV4 
-    ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMIrrorListV6 /etc/debmirror/OPNsenseMIrrorListV6
+if [ "$OPNSENSE_REPOS" = true ]; then
+    touch /etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV4
+    touch /etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV6
+    ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV4 /etc/debmirror/OPNsenseMirrorListV4 
+    ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV6 /etc/debmirror/OPNsenseMirrorListV6
     
-    echo "15 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py"  >> /etc/crontabs/cron-jobs
+    # echo "15 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py"  >> /etc/crontabs/cron-jobs
 fi
 
+echo "00 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/main.py"  >> /etc/crontabs/cron-jobs
 echo "30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh"  >> /etc/crontabs/cron-jobs
 
 # Configure the crontab to work 

AiO_Container/gitPush.sh

@@ -7,18 +7,18 @@ GITURL="${GITURL:-6767}"
 GITREPOPATH="${GITREPOPATH:-6767}"
 GITURLPROTO="${GITURLPROTO:-http}"
 
-# if [[ "$GITEA_TOKEN" == "6767" || "$GITREPOPATH" == "6767" || "$GITURL" == "6767" ]]; then
-if [[ "$GITEA_TOKEN" == "6767" ]]; then
+# if [ "$GITEA_TOKEN" == "6767" || "$GITREPOPATH" == "6767" || "$GITURL" == "6767" ]; then
+if [ "$GITEA_TOKEN" == "6767" ]; then
     exit 1
 fi
 
 WORKPTH="${WORKPTH:-/etc/debmirror/}"
 REPO_DIR="${REPO_DIR:-$WORKPTH/Repo-IP-lists}"
-REPO_URL="${REPO_URL:-GITURLPROTO://GITURL/GITREPOPATH}"
+REPO_URL="${REPO_URL:-$GITURLPROTO://$GITURL/$GITREPOPATH}"
 
 
 # Clone repo if not exists
-if [[ ! -d "$REPO_DIR/.git" ]]; then
+if [ ! -d "$REPO_DIR/.git" ]; then
     echo "[$(date)] Cloning repository..."
     git clone "$REPO_URL" "$REPO_DIR"
 fi
@@ -33,8 +33,8 @@ git cherry-pick --abort 2>/dev/null || true
 git checkout main || git checkout -b main origin/main
 git pull --rebase --autostash
 
-# git remote set-url origin "${GITURLPROTO}://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
-git remote set-url origin "https://${GITEA_TOKEN}@YuruC3/Repo-IP-lists"
+git remote set-url origin "${GITURLPROTO}://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
+# git remote set-url origin "https://${GITEA_TOKEN}@YuruC3/Repo-IP-lists"
 
 
 git config user.name "UpdateBot"
@@ -44,7 +44,7 @@ git config user.email "UpdateBot@localhost.local"
 git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
 
 # Only proceed if there are staged changes
-if [[ ! git diff --quiet --cached ]]; then
+if [ ! git diff --quiet --cached ]; then
     echo "[$(date)] Committing and pushing changes..."
     git commit -m "Auto-update mirror list on $(date -Iseconds)"
     # git pull --rebase --autostash

README.md

@@ -26,4 +26,9 @@ Here is a rundown of all the options in docker-compose
 
 ## Run at home 
 
-if you want to run it at your place remember to change crontab timings and git personal token and so on.
+if you want to run it at your place remember to change crontab timings and git personal token and so on.
+
+
+<p align="center">
+    <img src="https://data.shupogaki.org/assets/addbdfbdffwf.jpeg" alt="">
+</p>