diff --git a/customList/.gitignore b/customList/.gitignore new file mode 100644 index 0000000..d1c9345 --- /dev/null +++ b/customList/.gitignore @@ -0,0 +1,3 @@ +repo +__pycache__ +.env \ No newline at end of file diff --git a/customList/Dockerfile b/customList/Dockerfile new file mode 100644 index 0000000..02accfe --- /dev/null +++ b/customList/Dockerfile @@ -0,0 +1,40 @@ +FROM alpine:latest + +ENV EXTRAURL "http://mdu.se/" +ENV REPOFILE "/etc/customMirrors/repoList.list" +ENV DNSSRV "1.1.1.1" + +RUN apk update && \ + apk add python3 py3-pip git bash +# inotify-tools + +RUN mkdir /etc/customMirrors +WORKDIR /etc/customMirrors/ + +RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \ + ln -s /etc/customMirrors/Repo-IP-lists/MirrorListV4 /etc/customMirrors/MirrorListV4 && \ + ln -s /etc/customMirrors/Repo-IP-lists/MirrorListV6 /etc/customMirrors/MirrorListV6 + +# RUN touch /etc/customMirrors/MirrorListV6 && \ +# touch /etc/customMirrors/MirrorListV4 + +COPY mainDocker.py . +COPY whatDomain.py . +COPY requirements.txt . +COPY init.sh . + +RUN python3 -m venv venv && \ + venv/bin/python3 -m pip install --upgrade pip && \ + venv/bin/pip3 install -r requirements.txt + +COPY cron-jobs /etc/crontabs/ +RUN chmod 0644 /etc/crontabs/cron-jobs && \ + crontab /etc/crontabs/cron-jobs + +COPY gitPush.sh . + +RUN chmod +x ./gitPush.sh + +CMD ["bash", "init.sh"] + +# CMD ["/usr/sbin/crond", "-f"] diff --git a/customList/cron-jobs b/customList/cron-jobs new file mode 100644 index 0000000..38951cf --- /dev/null +++ b/customList/cron-jobs @@ -0,0 +1,11 @@ +# python +#20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py +25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py +# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py +#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py + +# git push +30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh +#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh \ No newline at end of file diff --git a/customList/customListGitVars.env b/customList/customListGitVars.env new file mode 100644 index 0000000..0b10295 --- /dev/null +++ b/customList/customListGitVars.env @@ -0,0 +1,4 @@ +GITURLPROTO=https +GITURL=tea.shupogaki.org +GITREPOPATH=YuruC3/Repo-IP-lists.git +GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2 \ No newline at end of file diff --git a/customList/customListURLS.env b/customList/customListURLS.env new file mode 100644 index 0000000..70a5b62 --- /dev/null +++ b/customList/customListURLS.env @@ -0,0 +1 @@ +EXTRAURL=["https://www.pixiv.net/", "https://pixiv.net/", "https://i.pximg.net/", "https://pximg.net/", "iwara.tv/"] \ No newline at end of file diff --git a/customList/docker-compose.yml b/customList/docker-compose.yml new file mode 100644 index 0000000..b49c341 --- /dev/null +++ b/customList/docker-compose.yml @@ -0,0 +1,18 @@ +--- + +services: + debmirup: + container_name: Debian_Mirrors_Updater + build: ./ + #image: yuruc3/debianrepolist:v0.3 + + environment: + - GITURLPROTO=https + - GITURL=tea.shupogaki.org + - GITREPOPATH=YuruC3/Repo-IP-lists.git + - GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2 + - REPOFILE=repoList.list + volumes: + - ./repoList.list:/etc/customMirrors/repoList.list:ro + + restart: unless-stopped diff --git a/customList/gitPush.sh b/customList/gitPush.sh new file mode 100644 index 0000000..e2490da --- /dev/null +++ b/customList/gitPush.sh @@ -0,0 +1,87 @@ +#!/bin/sh +set -e + +WORKPTH="/etc/debmirror/" +REPO_DIR="$WORKPTH/Repo-IP-lists" +REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}" + +# Clone repo if not exists +if [ ! -d "$REPO_DIR/.git" ]; then + echo "[$(date)] Cloning repository..." + git clone "$REPO_URL" "$REPO_DIR" +fi + +cd "$REPO_DIR" + +# Abort previous rebase/cherry-pick if stuck +git rebase --abort 2>/dev/null || true +git cherry-pick --abort 2>/dev/null || true + +# Make sure we're on a clean 'main' +git checkout main || git checkout -b main origin/main +git pull --rebase --autostash + +git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}" + + +git config user.name "UpdateBot" +git config user.email "UpdateBot@localhost.local" + +# Stage the files +git add MirrorListV4 MirrorListV6 + +# Only proceed if there are staged changes +if ! git diff --quiet --cached; then + echo "[$(date)] Committing and pushing changes..." + git commit -m "Auto-update mirror list on $(date -Iseconds)" + # git pull --rebase --autostash + git push --quiet + echo "[$(date)] Changes pushed." +else + echo "[$(date)] No changes to commit or push." +fi + + +# #!/bin/sh +# set -e + +# WORKPTH="/etc/debmirror/" +# REPO_DIRd="$WORKPTH/Repo-IP-lists" +# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}" + + +# # Clone repo only if it doesn't already exist +# if [ ! -d "$REPO_DIR/.git" ]; then +# echo "[$(date)] Cloning repository..." +# git clone "$REPO_URL" "$REPO_DIR" +# fi + + +# cd "$REPO_DIR" + +# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}" + +# git config user.name "UpdateBot" +# git config user.email "UpdateBot@localhost.local" + +# # stage files +# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6 + +# # If anything to commit locally, commit it now +# if ! git diff --quiet --cached; then +# echo "[$(date)] Committing local changes before pulling" +# git commit -m "Auto-commit before pull on $(date -Iseconds)" +# fi + +# # Now pull the latest +# git pull --rebase --autostash + +# # Commit and push only if there's anything new staged +# if git diff --quiet; then +# echo "[$(date)] No changes to commit." +# else +# git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet +# git push --quiet +# echo "[$(date)] Changes pushed successfully." +# fi + diff --git a/customList/init.sh b/customList/init.sh new file mode 100644 index 0000000..6e4f969 --- /dev/null +++ b/customList/init.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +echo "EXTRAURL: $EXTRAURL" + +if [[ -f "$REPOFILE" ]]; then + echo "URL file exists" +else + touch $REPOFILE + echo $EXTRAURL | tee $REPOFILE +fi + +echo "nameserver $DNSSRV" > /etc/resolv.conf +echo "search local" >> /etc/resolv.conf + +exec /usr/sbin/crond -f \ No newline at end of file diff --git a/customList/mainDocker.py b/customList/mainDocker.py new file mode 100644 index 0000000..69e49d8 --- /dev/null +++ b/customList/mainDocker.py @@ -0,0 +1,85 @@ +import requests, schedule, time, os +# from bs4 import BeautifulSoup +from whatDomain import * + +EXTRAURL = [] + +repoListFile = open(os.getenv("REPOFILE", "repoList.list"), 'r') +i = 0 +for repoUrl in repoListFile: + i += 1 + print(repoUrl.strip()) + EXTRAURL.append(repoUrl.strip()) +repoListFile.close() + +# EXTRAURL = list(os.getenv("EXTRAURL", "https://mdu.se/")) +IPv4FILE = "/etc/customMirrors/MirrorListV4" +IPv6FILE = "/etc/customMirrors/MirrorListV6" + +def sanitizeURL(inpurl: str): + if "https://" in inpurl: + outurl = inpurl[8:] + elif "http://" in inpurl: + outurl = inpurl[7:] + elif "http://" or "https://" not in url: + outurl = inpurl + else: + return inpurl + + i = 0 + for char in outurl: + i += 1 + if char == "/": + outurl = outurl[:i] + + if char == "/": + outurl = outurl[:-1] + return outurl + +def LeJob(): + + print("Starting lookup") + + # print(LeMirrorDict) + + with open(IPv4FILE, "w",) as fW: + + for url in EXTRAURL: + goodurl = sanitizeURL(url) + # print(goodurl) + + # ip4Dict = ermWhatATheIpFromDomainYaCrazy(url) + ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl) + + try: + for key, ip in ip4Dict.items(): + print(ip + "/32") + + fW.write(ip + "/32" + "\n") + except AttributeError: + continue + + + with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: + + for url in EXTRAURL: + goodurl = sanitizeURL(url) + # print(goodurl) + + # ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url) + ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl) + + try: + for key, ip in ip6Dict.items(): + print(ip + "/128") + + fW.write(ip + "/128" + "\n") + except AttributeError: + continue + + + + +LeJob() + +print("Done") \ No newline at end of file diff --git a/customList/requirements.txt b/customList/requirements.txt new file mode 100644 index 0000000..b3f0c80 --- /dev/null +++ b/customList/requirements.txt @@ -0,0 +1,4 @@ +#beautifulsoup4==4.13.4 +requests==2.32.3 +schedule==1.2.2 +nslookup==1.8.1 \ No newline at end of file diff --git a/customList/whatDomain.py b/customList/whatDomain.py new file mode 100644 index 0000000..70cb81e --- /dev/null +++ b/customList/whatDomain.py @@ -0,0 +1,129 @@ +#from nslookup import Nslookup +from typing import Optional, Annotated +import dns, dns.resolver + +# https://www.codeunderscored.com/nslookup-python/ + +def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]): + #dns_query = Nslookup() + """ + Tells you what IPv4 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(inpDomainNameOrSomething, 'A') + except dns.resolver.NoAnswer: + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + +def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]): + #dns_query = Nslookup() + """ + Tells you what IPv6 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA') + except dns.resolver.NoAnswer: + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + + +def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]): + #dns_query = Nslookup() + """ + Tells you what IPv6 address/es a domain point to. + Returns: + dict: A dictionary with IP addresses associated with that domain. + + """ + + whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa" + + + # i = 0 + outDict = {} + + #result = dns_query.dns_lookup("example.com") + #result = Nslookup.dns_lookup(inpDomainNameOrSomething) + try: + result = dns.resolver.resolve(whatToCheck, 'PTR') + except dns.resolver.NoAnswer: + print("\nDNS ERROR") + print("No answer from dns server.\n") + return 1 + except dns.resolver.NoNameservers: + print("\nDNS ERROR") + print("All nameservers failed to answer the query.\n Fix your DNS servers.\n") + return 1 + except dns.resolver.NXDOMAIN: + print("\nDNS ERROR") + print("The DNS query name does not exist.\n") + return 1 + except dns.resolver.LifetimeTimeout: + print("\nDNS ERROR") + print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n") + return 1 + for i, something in enumerate(result): + outDict[i] = something.to_text() + # i += 1 + + return outDict + + +#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net")) +#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net")) +#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))