customList/.gitignore

@@ -1,3 +0,0 @@
-repo
-__pycache__
-.env

customList/Dockerfile

@@ -1,40 +0,0 @@
-FROM alpine:latest
-
-ENV EXTRAURL = "http://mdu.se/"
-ENV REPOFILE = "/etc/customMirrors/repoList.list"
-ENV DNSSRV = "1.1.1.1"
-
-RUN apk update && \
-    apk add python3 py3-pip git bash
-# inotify-tools
-
-RUN mkdir /etc/customMirrors
-WORKDIR /etc/customMirrors/
-
-RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \
-    ln -s /etc/customMirrors/Repo-IP-lists/MirrorListV4 /etc/customMirrors/MirrorListV4 && \
-    ln -s /etc/customMirrors/Repo-IP-lists/MirrorListV6 /etc/customMirrors/MirrorListV6 
-
-# RUN touch /etc/customMirrors/MirrorListV6 && \
-#     touch /etc/customMirrors/MirrorListV4
-
-COPY mainDocker.py .
-COPY whatDomain.py .
-COPY requirements.txt .
-COPY init.sh .
-
-RUN python3 -m venv venv && \
-    venv/bin/python3 -m pip install --upgrade pip && \
-    venv/bin/pip3 install -r requirements.txt
-
-COPY cron-jobs /etc/crontabs/
-RUN chmod 0644 /etc/crontabs/cron-jobs && \
-    crontab /etc/crontabs/cron-jobs
-
-COPY gitPush.sh .
-
-RUN chmod +x ./gitPush.sh
-
-CMD ["bash", "init.sh"]
-
-# CMD ["/usr/sbin/crond", "-f"]

customList/cron-jobs

@@ -1,11 +0,0 @@
-# python
-#20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
-25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
-# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
-# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
-#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
-#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
-
-# git push 
-30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh
-#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh

customList/customListGitVars.env

@@ -1,4 +0,0 @@
-GITURLPROTO=https
-GITURL=tea.shupogaki.org
-GITREPOPATH=YuruC3/Repo-IP-lists.git
-GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2

customList/customListURLS.env

@@ -1 +0,0 @@
-EXTRAURL=["https://www.pixiv.net/", "https://pixiv.net/", "https://i.pximg.net/", "https://pximg.net/", "iwara.tv/"]

customList/docker-compose.yml

@@ -1,18 +0,0 @@
----
-
-services:
-  debmirup:
-    container_name: Debian_Mirrors_Updater
-    build: ./
-    #image: yuruc3/debianrepolist:v0.3
-
-    environment:
-      - GITURLPROTO=https
-      - GITURL=tea.shupogaki.org
-      - GITREPOPATH=YuruC3/Repo-IP-lists.git
-      - GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2
-      - REPOFILE=repoList.list
-    volumes:
-      - ./repoList.list:/etc/customMirrors/repoList.list:ro
-
-    restart: unless-stopped

customList/gitPush.sh

@@ -1,87 +0,0 @@
-#!/bin/sh
-set -e
-
-WORKPTH="/etc/debmirror/"
-REPO_DIR="$WORKPTH/Repo-IP-lists"
-REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
-
-# Clone repo if not exists
-if [ ! -d "$REPO_DIR/.git" ]; then
-    echo "[$(date)] Cloning repository..."
-    git clone "$REPO_URL" "$REPO_DIR"
-fi
-
-cd "$REPO_DIR"
-
-# Abort previous rebase/cherry-pick if stuck
-git rebase --abort 2>/dev/null || true
-git cherry-pick --abort 2>/dev/null || true
-
-# Make sure we're on a clean 'main'
-git checkout main || git checkout -b main origin/main
-git pull --rebase --autostash
-
-git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
-
-
-git config user.name "UpdateBot"
-git config user.email "UpdateBot@localhost.local"
-
-# Stage the files
-git add MirrorListV4 MirrorListV6
-
-# Only proceed if there are staged changes
-if ! git diff --quiet --cached; then
-    echo "[$(date)] Committing and pushing changes..."
-    git commit -m "Auto-update mirror list on $(date -Iseconds)"
-    # git pull --rebase --autostash
-    git push --quiet
-    echo "[$(date)] Changes pushed."
-else
-    echo "[$(date)] No changes to commit or push."
-fi
-
-
-# #!/bin/sh
-# set -e
-
-# WORKPTH="/etc/debmirror/"
-# REPO_DIRd="$WORKPTH/Repo-IP-lists"
-# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
-
-
-# # Clone repo only if it doesn't already exist
-# if [ ! -d "$REPO_DIR/.git" ]; then
-#     echo "[$(date)] Cloning repository..."
-#     git clone "$REPO_URL" "$REPO_DIR"
-# fi
-
-
-# cd "$REPO_DIR"
-
-# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
-
-# git config user.name "UpdateBot"
-# git config user.email "UpdateBot@localhost.local"
-
-# # stage files
-# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
-
-# # If anything to commit locally, commit it now
-# if ! git diff --quiet --cached; then
-#     echo "[$(date)] Committing local changes before pulling"
-#     git commit -m "Auto-commit before pull on $(date -Iseconds)"
-# fi
-
-# # Now pull the latest
-# git pull --rebase --autostash
-
-# # Commit and push only if there's anything new staged
-# if git diff --quiet; then
-#     echo "[$(date)] No changes to commit."
-# else
-#     git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet
-#     git push --quiet
-#     echo "[$(date)] Changes pushed successfully."
-# fi
-

customList/init.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-set -e
-
-echo "EXTRAURL: $EXTRAURL"
-
-if [[ -f "$REPOFILE" ]]; then
-    echo "URL file exists"
-else
-    touch $REPOFILE
-    echo $EXTRAURL | tee $REPOFILE
-fi
-
-echo "nameserver $DNSSRV" > /etc/resolv.conf 
-echo "search local" >> /etc/resolv.conf
-
-exec /usr/sbin/crond -f

customList/mainDocker.py

@@ -1,85 +0,0 @@
-import requests, schedule, time, os
-# from bs4 import BeautifulSoup
-from whatDomain import *
-
-EXTRAURL = []
-
-repoListFile = open(os.getenv("REPOFILE", "repoList.list"), 'r')
-i = 0
-for repoUrl in repoListFile:
-    i += 1
-    print(repoUrl.strip())
-    EXTRAURL.append(repoUrl.strip())
-repoListFile.close()
-
-# EXTRAURL = list(os.getenv("EXTRAURL", "https://mdu.se/"))
-IPv4FILE = "/etc/customMirrors/MirrorListV4"
-IPv6FILE = "/etc/customMirrors/MirrorListV6"
-
-def sanitizeURL(inpurl: str):
-    if "https://" in inpurl:
-        outurl = inpurl[8:]
-    elif "http://" in inpurl:
-        outurl = inpurl[7:]
-    elif "http://" or "https://" not in url:
-        outurl = inpurl
-    else:
-        return inpurl
-
-    i = 0
-    for char in outurl:
-        i += 1
-        if char == "/":
-            outurl = outurl[:i]
-
-    if char == "/":
-        outurl = outurl[:-1]
-    return outurl
-
-def LeJob():
-
-    print("Starting lookup")
-
-    # print(LeMirrorDict)
-
-    with open(IPv4FILE, "w",) as fW:
-
-        for url in EXTRAURL:
-            goodurl = sanitizeURL(url)
-            # print(goodurl)
-
-            # ip4Dict = ermWhatATheIpFromDomainYaCrazy(url)
-            ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
-
-            try:
-                for key, ip in ip4Dict.items():
-                    print(ip + "/32")
-
-                    fW.write(ip + "/32" +  "\n")
-            except AttributeError: 
-                continue
-                        
-
-    with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
-
-        for url in EXTRAURL:
-            goodurl = sanitizeURL(url)
-            # print(goodurl)
-
-            # ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url)
-            ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
-
-            try:
-                for key, ip in ip6Dict.items():
-                    print(ip + "/128")
-
-                    fW.write(ip + "/128" + "\n")
-            except AttributeError: 
-                continue
-
-
-
-
-LeJob()
-
-print("Done")

customList/repoList.list

@@ -1,4 +0,0 @@
-https://www.pixiv.net/
-https://pixiv.net/
-https://i.pximg.net/
-https://pximg.net/

customList/requirements.txt

@@ -1,4 +0,0 @@
-#beautifulsoup4==4.13.4
-requests==2.32.3
-schedule==1.2.2
-nslookup==1.8.1

customList/whatDomain.py

@@ -1,129 +0,0 @@
-#from nslookup import Nslookup
-from typing import Optional, Annotated
-import dns, dns.resolver
-
-# https://www.codeunderscored.com/nslookup-python/
-
-def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
-    #dns_query = Nslookup()
-    """
-    Tells you what IPv4 address/es a domain point to.
-    Returns:
-        dict: A dictionary with IP addresses associated with that domain.
-
-    """
-
-    # i = 0
-    outDict = {} 
-
-    #result = dns_query.dns_lookup("example.com")
-    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
-    try:
-        result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
-    except dns.resolver.NoAnswer:
-        print("\nDNS ERROR")
-        print("No answer from dns server.\n")
-        return 1
-    except dns.resolver.NoNameservers:
-        print("\nDNS ERROR")
-        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
-        return 1
-    except dns.resolver.NXDOMAIN:
-        print("\nDNS ERROR")
-        print("The DNS query name does not exist.\n")
-        return 1
-    except dns.resolver.LifetimeTimeout:
-        print("\nDNS ERROR")
-        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
-        return 1
-    for i, something in enumerate(result):
-        outDict[i] = something.to_text()
-        # i += 1
-
-    return outDict
-
-def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
-    #dns_query = Nslookup()
-    """
-    Tells you what IPv6 address/es a domain point to.
-    Returns:
-        dict: A dictionary with IP addresses associated with that domain.
-
-    """
-
-
-    # i = 0
-    outDict = {} 
-
-    #result = dns_query.dns_lookup("example.com")
-    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
-    try:
-        result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
-    except dns.resolver.NoAnswer:
-        print("\nDNS ERROR")
-        print("No answer from dns server.\n")
-        return 1
-    except dns.resolver.NoNameservers:
-        print("\nDNS ERROR")
-        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
-        return 1
-    except dns.resolver.NXDOMAIN:
-        print("\nDNS ERROR")
-        print("The DNS query name does not exist.\n")
-        return 1
-    except dns.resolver.LifetimeTimeout:
-        print("\nDNS ERROR")
-        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
-        return 1
-    for i, something in enumerate(result):
-        outDict[i] = something.to_text()
-        # i += 1
-
-    return outDict
-
-
-def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]):
-    #dns_query = Nslookup()
-    """
-    Tells you what IPv6 address/es a domain point to.
-    Returns:
-        dict: A dictionary with IP addresses associated with that domain.
-
-    """
-
-    whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa"
-
-
-    # i = 0
-    outDict = {}
-
-    #result = dns_query.dns_lookup("example.com")
-    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
-    try:
-        result = dns.resolver.resolve(whatToCheck, 'PTR')
-    except dns.resolver.NoAnswer:
-        print("\nDNS ERROR")
-        print("No answer from dns server.\n")
-        return 1
-    except dns.resolver.NoNameservers:
-        print("\nDNS ERROR")
-        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
-        return 1
-    except dns.resolver.NXDOMAIN:
-        print("\nDNS ERROR")
-        print("The DNS query name does not exist.\n")
-        return 1
-    except dns.resolver.LifetimeTimeout:
-        print("\nDNS ERROR")
-        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
-        return 1
-    for i, something in enumerate(result):
-        outDict[i] = something.to_text()
-        # i += 1
-
-    return outDict
-
-
-#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
-#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
-#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))

docker/mainDocker.py

@@ -2,9 +2,13 @@ import requests, schedule, time, os
 from bs4 import BeautifulSoup
 from whatDomain import *
 
-DEBMIRRORURL = str(os.getenv("DEBMIRRORURL", "https://www.debian.org/mirror/list"))
-DEBSECURITYURL = str(os.getenv("DEBSECURITYURL", "https://security.debian.org/debian-security/"))
 
+if os.environ["DEBMIRRORURL"]:
+    DEBMIRRORURL = os.environ["DEBMIRRORURL"]
+else:
+    DEBMIRRORURL = "https://www.debian.org/mirror/list"
+
+DEBSECURITYURL = ["https://security.debian.org/debian-security/"]
 EXTRASURL = ["https://download.docker.com/linux/debian/", 
             # Double just to be sure. Even though they point to the same IP
             "http://download.proxmox.com/debian/", 
@@ -42,7 +46,7 @@ def sanitizeURL(inpurl: str):
     elif "http://" or "https://" not in url:
         outurl = inpurl
     else:
-        return 1
+        return -1
 
     i = 0
     for char in outurl:

docker/mainOPNsense.py

@@ -3,10 +3,10 @@ from bs4 import BeautifulSoup
 from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
 
 
-OPNSNSMIRRORURL = str(os.getenv("OPNSNSMIRRORURL", "https://opnsense.org/download/#full-mirror-listing"))
+
- 
+OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing"
-IPv4FILE = str(os.getenv("IPv4FILE", "/etc/debmirror/OPNS_MirrorListV4"))
+IPv4FILE = "/etc/debmirror/OPNS_MirrorListV4"
-IPv6FILE = str(os.getenv("IPv6FILE", "/etc/debmirror/OPNS_MirrorListV6"))
+IPv6FILE = "/etc/debmirror/OPNS_MirrorListV6"
 
 
 def sanitizeURL(inpurl: str):