Added additional container for custom urls

customList/.gitignore

@@ -0,0 +1,3 @@
+repo
+__pycache__
+.env

customList/Dockerfile

@@ -0,0 +1,40 @@
+FROM alpine:latest
+
+ENV EXTRAURL "http://mdu.se/"
+ENV REPOFILE "/etc/customMirrors/repoList.list"
+ENV DNSSRV "1.1.1.1"
+
+RUN apk update && \
+    apk add python3 py3-pip git bash
+# inotify-tools
+
+RUN mkdir /etc/customMirrors
+WORKDIR /etc/customMirrors/
+
+RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \
+    ln -s /etc/customMirrors/Repo-IP-lists/MirrorListV4 /etc/customMirrors/MirrorListV4 && \
+    ln -s /etc/customMirrors/Repo-IP-lists/MirrorListV6 /etc/customMirrors/MirrorListV6 
+
+# RUN touch /etc/customMirrors/MirrorListV6 && \
+#     touch /etc/customMirrors/MirrorListV4
+
+COPY mainDocker.py .
+COPY whatDomain.py .
+COPY requirements.txt .
+COPY init.sh .
+
+RUN python3 -m venv venv && \
+    venv/bin/python3 -m pip install --upgrade pip && \
+    venv/bin/pip3 install -r requirements.txt
+
+COPY cron-jobs /etc/crontabs/
+RUN chmod 0644 /etc/crontabs/cron-jobs && \
+    crontab /etc/crontabs/cron-jobs
+
+COPY gitPush.sh .
+
+RUN chmod +x ./gitPush.sh
+
+CMD ["bash", "init.sh"]
+
+# CMD ["/usr/sbin/crond", "-f"]

customList/cron-jobs

@@ -0,0 +1,11 @@
+# python
+#20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
+
+# git push 
+30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh
+#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh

customList/customListGitVars.env

@@ -0,0 +1,4 @@
+GITURLPROTO=https
+GITURL=tea.shupogaki.org
+GITREPOPATH=YuruC3/Repo-IP-lists.git
+GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2

customList/customListURLS.env

@@ -0,0 +1 @@
+EXTRAURL=["https://www.pixiv.net/", "https://pixiv.net/", "https://i.pximg.net/", "https://pximg.net/", "iwara.tv/"]

customList/docker-compose.yml

@@ -0,0 +1,18 @@
+---
+
+services:
+  debmirup:
+    container_name: Debian_Mirrors_Updater
+    build: ./
+    #image: yuruc3/debianrepolist:v0.3
+
+    environment:
+      - GITURLPROTO=https
+      - GITURL=tea.shupogaki.org
+      - GITREPOPATH=YuruC3/Repo-IP-lists.git
+      - GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2
+      - REPOFILE=repoList.list
+    volumes:
+      - ./repoList.list:/etc/customMirrors/repoList.list:ro
+
+    restart: unless-stopped

customList/gitPush.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+set -e
+
+WORKPTH="/etc/debmirror/"
+REPO_DIR="$WORKPTH/Repo-IP-lists"
+REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
+
+# Clone repo if not exists
+if [ ! -d "$REPO_DIR/.git" ]; then
+    echo "[$(date)] Cloning repository..."
+    git clone "$REPO_URL" "$REPO_DIR"
+fi
+
+cd "$REPO_DIR"
+
+# Abort previous rebase/cherry-pick if stuck
+git rebase --abort 2>/dev/null || true
+git cherry-pick --abort 2>/dev/null || true
+
+# Make sure we're on a clean 'main'
+git checkout main || git checkout -b main origin/main
+git pull --rebase --autostash
+
+git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
+
+
+git config user.name "UpdateBot"
+git config user.email "UpdateBot@localhost.local"
+
+# Stage the files
+git add MirrorListV4 MirrorListV6
+
+# Only proceed if there are staged changes
+if ! git diff --quiet --cached; then
+    echo "[$(date)] Committing and pushing changes..."
+    git commit -m "Auto-update mirror list on $(date -Iseconds)"
+    # git pull --rebase --autostash
+    git push --quiet
+    echo "[$(date)] Changes pushed."
+else
+    echo "[$(date)] No changes to commit or push."
+fi
+
+
+# #!/bin/sh
+# set -e
+
+# WORKPTH="/etc/debmirror/"
+# REPO_DIRd="$WORKPTH/Repo-IP-lists"
+# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
+
+
+# # Clone repo only if it doesn't already exist
+# if [ ! -d "$REPO_DIR/.git" ]; then
+#     echo "[$(date)] Cloning repository..."
+#     git clone "$REPO_URL" "$REPO_DIR"
+# fi
+
+
+# cd "$REPO_DIR"
+
+# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
+
+# git config user.name "UpdateBot"
+# git config user.email "UpdateBot@localhost.local"
+
+# # stage files
+# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
+
+# # If anything to commit locally, commit it now
+# if ! git diff --quiet --cached; then
+#     echo "[$(date)] Committing local changes before pulling"
+#     git commit -m "Auto-commit before pull on $(date -Iseconds)"
+# fi
+
+# # Now pull the latest
+# git pull --rebase --autostash
+
+# # Commit and push only if there's anything new staged
+# if git diff --quiet; then
+#     echo "[$(date)] No changes to commit."
+# else
+#     git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet
+#     git push --quiet
+#     echo "[$(date)] Changes pushed successfully."
+# fi
+

customList/init.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+echo "EXTRAURL: $EXTRAURL"
+
+if [[ -f "$REPOFILE" ]]; then
+    echo "URL file exists"
+else
+    touch $REPOFILE
+    echo $EXTRAURL | tee $REPOFILE
+fi
+
+echo "nameserver $DNSSRV" > /etc/resolv.conf 
+echo "search local" >> /etc/resolv.conf
+
+exec /usr/sbin/crond -f

customList/mainDocker.py

@@ -0,0 +1,85 @@
+import requests, schedule, time, os
+# from bs4 import BeautifulSoup
+from whatDomain import *
+
+EXTRAURL = []
+
+repoListFile = open(os.getenv("REPOFILE", "repoList.list"), 'r')
+i = 0
+for repoUrl in repoListFile:
+    i += 1
+    print(repoUrl.strip())
+    EXTRAURL.append(repoUrl.strip())
+repoListFile.close()
+
+# EXTRAURL = list(os.getenv("EXTRAURL", "https://mdu.se/"))
+IPv4FILE = "/etc/customMirrors/MirrorListV4"
+IPv6FILE = "/etc/customMirrors/MirrorListV6"
+
+def sanitizeURL(inpurl: str):
+    if "https://" in inpurl:
+        outurl = inpurl[8:]
+    elif "http://" in inpurl:
+        outurl = inpurl[7:]
+    elif "http://" or "https://" not in url:
+        outurl = inpurl
+    else:
+        return inpurl
+
+    i = 0
+    for char in outurl:
+        i += 1
+        if char == "/":
+            outurl = outurl[:i]
+
+    if char == "/":
+        outurl = outurl[:-1]
+    return outurl
+
+def LeJob():
+
+    print("Starting lookup")
+
+    # print(LeMirrorDict)
+
+    with open(IPv4FILE, "w",) as fW:
+
+        for url in EXTRAURL:
+            goodurl = sanitizeURL(url)
+            # print(goodurl)
+
+            # ip4Dict = ermWhatATheIpFromDomainYaCrazy(url)
+            ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
+
+            try:
+                for key, ip in ip4Dict.items():
+                    print(ip + "/32")
+
+                    fW.write(ip + "/32" +  "\n")
+            except AttributeError: 
+                continue
+                        
+
+    with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
+
+        for url in EXTRAURL:
+            goodurl = sanitizeURL(url)
+            # print(goodurl)
+
+            # ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url)
+            ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
+
+            try:
+                for key, ip in ip6Dict.items():
+                    print(ip + "/128")
+
+                    fW.write(ip + "/128" + "\n")
+            except AttributeError: 
+                continue
+
+
+
+
+LeJob()
+
+print("Done")

customList/requirements.txt

@@ -0,0 +1,4 @@
+#beautifulsoup4==4.13.4
+requests==2.32.3
+schedule==1.2.2
+nslookup==1.8.1

customList/whatDomain.py

@@ -0,0 +1,129 @@
+#from nslookup import Nslookup
+from typing import Optional, Annotated
+import dns, dns.resolver
+
+# https://www.codeunderscored.com/nslookup-python/
+
+def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
+    #dns_query = Nslookup()
+    """
+    Tells you what IPv4 address/es a domain point to.
+    Returns:
+        dict: A dictionary with IP addresses associated with that domain.
+
+    """
+
+    # i = 0
+    outDict = {} 
+
+    #result = dns_query.dns_lookup("example.com")
+    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
+    try:
+        result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
+    except dns.resolver.NoAnswer:
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
+    for i, something in enumerate(result):
+        outDict[i] = something.to_text()
+        # i += 1
+
+    return outDict
+
+def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
+    #dns_query = Nslookup()
+    """
+    Tells you what IPv6 address/es a domain point to.
+    Returns:
+        dict: A dictionary with IP addresses associated with that domain.
+
+    """
+
+
+    # i = 0
+    outDict = {} 
+
+    #result = dns_query.dns_lookup("example.com")
+    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
+    try:
+        result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
+    except dns.resolver.NoAnswer:
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
+    for i, something in enumerate(result):
+        outDict[i] = something.to_text()
+        # i += 1
+
+    return outDict
+
+
+def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]):
+    #dns_query = Nslookup()
+    """
+    Tells you what IPv6 address/es a domain point to.
+    Returns:
+        dict: A dictionary with IP addresses associated with that domain.
+
+    """
+
+    whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa"
+
+
+    # i = 0
+    outDict = {}
+
+    #result = dns_query.dns_lookup("example.com")
+    #result = Nslookup.dns_lookup(inpDomainNameOrSomething)
+    try:
+        result = dns.resolver.resolve(whatToCheck, 'PTR')
+    except dns.resolver.NoAnswer:
+        print("\nDNS ERROR")
+        print("No answer from dns server.\n")
+        return 1
+    except dns.resolver.NoNameservers:
+        print("\nDNS ERROR")
+        print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
+        return 1
+    except dns.resolver.NXDOMAIN:
+        print("\nDNS ERROR")
+        print("The DNS query name does not exist.\n")
+        return 1
+    except dns.resolver.LifetimeTimeout:
+        print("\nDNS ERROR")
+        print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
+        return 1
+    for i, something in enumerate(result):
+        outDict[i] = something.to_text()
+        # i += 1
+
+    return outDict
+
+
+#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
+#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
+#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))