Compare commits
7 Commits
6fde3dd9be
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 880da3b79d | |||
| 61e7560952 | |||
| 56ac33b666 | |||
| d2838646b3 | |||
| 8c61f4a51c | |||
|
|
75cb35b954 | ||
|
|
f572e9f10e |
3
customList/.gitignore
vendored
Normal file
3
customList/.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
repo
|
||||||
|
__pycache__
|
||||||
|
.env
|
||||||
28
customList/Dockerfile
Normal file
28
customList/Dockerfile
Normal file
@@ -0,0 +1,28 @@
|
|||||||
|
FROM alpine:latest
|
||||||
|
|
||||||
|
ENV REPOFILE="/etc/customMirrors/repoList.list" EXTRAURL="http://mdu.se/"
|
||||||
|
ENV DNSSRV="1.1.1.1"
|
||||||
|
ENV IPV4FILENAME="MirrorListV4" IPV6FILENAME="MirrorListV6"
|
||||||
|
ENV CRONTABSET="25 */4 * * *"
|
||||||
|
|
||||||
|
|
||||||
|
RUN apk update && \
|
||||||
|
apk add python3 py3-pip git bash
|
||||||
|
|
||||||
|
RUN mkdir /etc/customMirrors
|
||||||
|
WORKDIR /etc/customMirrors/
|
||||||
|
|
||||||
|
COPY mainDocker.py .
|
||||||
|
COPY whatDomain.py .
|
||||||
|
COPY requirements.txt .
|
||||||
|
COPY init.sh .
|
||||||
|
|
||||||
|
RUN python3 -m venv venv && \
|
||||||
|
venv/bin/python3 -m pip install --upgrade pip && \
|
||||||
|
venv/bin/pip3 install -r requirements.txt
|
||||||
|
|
||||||
|
COPY gitPush.sh .
|
||||||
|
|
||||||
|
RUN chmod +x ./gitPush.sh
|
||||||
|
|
||||||
|
CMD ["bash", "init.sh"]
|
||||||
11
customList/cron-jobs
Normal file
11
customList/cron-jobs
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
# python
|
||||||
|
#20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||||
|
25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||||
|
# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||||
|
# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||||
|
#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||||
|
#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||||
|
|
||||||
|
# git push
|
||||||
|
30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||||
|
#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||||
4
customList/customListGitVars.env
Normal file
4
customList/customListGitVars.env
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
GITURLPROTO=https
|
||||||
|
GITURL=tea.shupogaki.org
|
||||||
|
GITREPOPATH=YuruC3/Repo-IP-lists.git
|
||||||
|
GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2
|
||||||
1
customList/customListURLS.env
Normal file
1
customList/customListURLS.env
Normal file
@@ -0,0 +1 @@
|
|||||||
|
EXTRAURL=["https://www.pixiv.net/", "https://pixiv.net/", "https://i.pximg.net/", "https://pximg.net/", "iwara.tv/"]
|
||||||
20
customList/docker-compose.yml
Normal file
20
customList/docker-compose.yml
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
---
|
||||||
|
|
||||||
|
services:
|
||||||
|
debmirup:
|
||||||
|
container_name: Debian_Mirrors_Updater
|
||||||
|
build: ./
|
||||||
|
#image: yuruc3/debianrepolist:v0.3
|
||||||
|
|
||||||
|
environment:
|
||||||
|
- GITURLPROTO=https
|
||||||
|
- GITURL=tea.shupogaki.org
|
||||||
|
- GITREPOPATH=YuruC3/Repo-IP-lists.git
|
||||||
|
- GITEA_TOKEN=3947a16ee5c3a337d6ff9cfb32d361167d4099d7
|
||||||
|
- REPOFILE=repoList.list
|
||||||
|
- IPV4FILENAME=customListNv4
|
||||||
|
- IPV6FILENAME=customListNv6
|
||||||
|
volumes:
|
||||||
|
- ./repoList.list:/etc/customMirrors/repoList.list:ro
|
||||||
|
|
||||||
|
restart: unless-stopped
|
||||||
87
customList/gitPush.sh
Normal file
87
customList/gitPush.sh
Normal file
@@ -0,0 +1,87 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -e
|
||||||
|
|
||||||
|
WORKPTH="/etc/customMirrors/"
|
||||||
|
REPO_DIR="$WORKPTH/Repo-IP-lists"
|
||||||
|
REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
||||||
|
|
||||||
|
# Clone repo if not exists
|
||||||
|
if [ ! -d "$REPO_DIR/.git" ]; then
|
||||||
|
echo "[$(date)] Cloning repository..."
|
||||||
|
git clone "$REPO_URL" "$REPO_DIR"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$REPO_DIR"
|
||||||
|
|
||||||
|
# Abort previous rebase/cherry-pick if stuck
|
||||||
|
git rebase --abort 2>/dev/null || true
|
||||||
|
git cherry-pick --abort 2>/dev/null || true
|
||||||
|
|
||||||
|
# Make sure we're on a clean 'main'
|
||||||
|
git checkout main || git checkout -b main origin/main
|
||||||
|
git pull --rebase --autostash
|
||||||
|
|
||||||
|
git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
||||||
|
|
||||||
|
|
||||||
|
git config user.name "UpdateBot"
|
||||||
|
git config user.email "UpdateBot@localhost.local"
|
||||||
|
|
||||||
|
# Stage the files
|
||||||
|
git add $IPV4FILENAME $IPV6FILENAME
|
||||||
|
|
||||||
|
# Only proceed if there are staged changes
|
||||||
|
if ! git diff --quiet --cached; then
|
||||||
|
echo "[$(date)] Committing and pushing changes..."
|
||||||
|
git commit -m "Auto-update mirror list on $(date -Iseconds)"
|
||||||
|
# git pull --rebase --autostash
|
||||||
|
git push --quiet
|
||||||
|
echo "[$(date)] Changes pushed."
|
||||||
|
else
|
||||||
|
echo "[$(date)] No changes to commit or push."
|
||||||
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
# #!/bin/sh
|
||||||
|
# set -e
|
||||||
|
|
||||||
|
# WORKPTH="/etc/debmirror/"
|
||||||
|
# REPO_DIRd="$WORKPTH/Repo-IP-lists"
|
||||||
|
# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
||||||
|
|
||||||
|
|
||||||
|
# # Clone repo only if it doesn't already exist
|
||||||
|
# if [ ! -d "$REPO_DIR/.git" ]; then
|
||||||
|
# echo "[$(date)] Cloning repository..."
|
||||||
|
# git clone "$REPO_URL" "$REPO_DIR"
|
||||||
|
# fi
|
||||||
|
|
||||||
|
|
||||||
|
# cd "$REPO_DIR"
|
||||||
|
|
||||||
|
# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
||||||
|
|
||||||
|
# git config user.name "UpdateBot"
|
||||||
|
# git config user.email "UpdateBot@localhost.local"
|
||||||
|
|
||||||
|
# # stage files
|
||||||
|
# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
|
||||||
|
|
||||||
|
# # If anything to commit locally, commit it now
|
||||||
|
# if ! git diff --quiet --cached; then
|
||||||
|
# echo "[$(date)] Committing local changes before pulling"
|
||||||
|
# git commit -m "Auto-commit before pull on $(date -Iseconds)"
|
||||||
|
# fi
|
||||||
|
|
||||||
|
# # Now pull the latest
|
||||||
|
# git pull --rebase --autostash
|
||||||
|
|
||||||
|
# # Commit and push only if there's anything new staged
|
||||||
|
# if git diff --quiet; then
|
||||||
|
# echo "[$(date)] No changes to commit."
|
||||||
|
# else
|
||||||
|
# git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet
|
||||||
|
# git push --quiet
|
||||||
|
# echo "[$(date)] Changes pushed successfully."
|
||||||
|
# fi
|
||||||
|
|
||||||
23
customList/init.sh
Normal file
23
customList/init.sh
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
echo "EXTRAURL: $EXTRAURL"
|
||||||
|
|
||||||
|
if [[ -f "$REPOFILE" ]]; then
|
||||||
|
echo "URL file exists"
|
||||||
|
else
|
||||||
|
touch $REPOFILE
|
||||||
|
echo $EXTRAURL | tee $REPOFILE
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "nameserver $DNSSRV" > /etc/resolv.conf
|
||||||
|
echo "search local" >> /etc/resolv.conf
|
||||||
|
|
||||||
|
git clone https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}
|
||||||
|
ln -s /etc/customMirrors/Repo-IP-lists/$IPV4FILENAME /etc/customMirrors/$IPV4FILENAME
|
||||||
|
ln -s /etc/customMirrors/Repo-IP-lists/$IPV6FILENAME /etc/customMirrors/$IPV6FILENAME
|
||||||
|
|
||||||
|
echo -n "$CRONTABSET /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py" > /etc/crontabs/customListCron
|
||||||
|
|
||||||
|
exec /usr/sbin/crond -f
|
||||||
88
customList/mainDocker.py
Normal file
88
customList/mainDocker.py
Normal file
@@ -0,0 +1,88 @@
|
|||||||
|
import requests, schedule, time, os
|
||||||
|
# from bs4 import BeautifulSoup
|
||||||
|
from whatDomain import *
|
||||||
|
|
||||||
|
EXTRAURL = []
|
||||||
|
|
||||||
|
repoListFile = open(os.getenv("REPOFILE", "repoList.list"), 'r')
|
||||||
|
i = 0
|
||||||
|
for repoUrl in repoListFile:
|
||||||
|
i += 1
|
||||||
|
print(repoUrl.strip())
|
||||||
|
EXTRAURL.append(repoUrl.strip())
|
||||||
|
repoListFile.close()
|
||||||
|
|
||||||
|
IPV4FILENAME = str(os.getenv("IPV4FILENAME", "MirrorListV4"))
|
||||||
|
IPV6FILENAME = str(os.getenv("IPV6FILENAME", "MirrorListV6"))
|
||||||
|
|
||||||
|
# EXTRAURL = list(os.getenv("EXTRAURL", "https://mdu.se/"))
|
||||||
|
IPv4FILE = f"/etc/customMirrors/{IPV4FILENAME}"
|
||||||
|
IPv6FILE = f"/etc/customMirrors/{IPV6FILENAME}"
|
||||||
|
|
||||||
|
def sanitizeURL(inpurl: str):
|
||||||
|
if "https://" in inpurl:
|
||||||
|
outurl = inpurl[8:]
|
||||||
|
elif "http://" in inpurl:
|
||||||
|
outurl = inpurl[7:]
|
||||||
|
elif "http://" or "https://" not in url:
|
||||||
|
outurl = inpurl
|
||||||
|
else:
|
||||||
|
return inpurl
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
for char in outurl:
|
||||||
|
i += 1
|
||||||
|
if char == "/":
|
||||||
|
outurl = outurl[:i]
|
||||||
|
|
||||||
|
if char == "/":
|
||||||
|
outurl = outurl[:-1]
|
||||||
|
return outurl
|
||||||
|
|
||||||
|
def LeJob():
|
||||||
|
|
||||||
|
print("Starting lookup")
|
||||||
|
|
||||||
|
# print(LeMirrorDict)
|
||||||
|
|
||||||
|
with open(IPv4FILE, "w",) as fW:
|
||||||
|
|
||||||
|
for url in EXTRAURL:
|
||||||
|
goodurl = sanitizeURL(url)
|
||||||
|
# print(goodurl)
|
||||||
|
|
||||||
|
# ip4Dict = ermWhatATheIpFromDomainYaCrazy(url)
|
||||||
|
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
|
||||||
|
|
||||||
|
try:
|
||||||
|
for key, ip in ip4Dict.items():
|
||||||
|
print(ip + "/32")
|
||||||
|
|
||||||
|
fW.write(ip + "/32" + "\n")
|
||||||
|
except AttributeError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
|
with open(IPv6FILE, "w",) as fW:
|
||||||
|
|
||||||
|
for url in EXTRAURL:
|
||||||
|
goodurl = sanitizeURL(url)
|
||||||
|
# print(goodurl)
|
||||||
|
|
||||||
|
# ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url)
|
||||||
|
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
|
||||||
|
|
||||||
|
try:
|
||||||
|
for key, ip in ip6Dict.items():
|
||||||
|
print(ip + "/128")
|
||||||
|
|
||||||
|
fW.write(ip + "/128" + "\n")
|
||||||
|
except AttributeError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
LeJob()
|
||||||
|
|
||||||
|
print("Done")
|
||||||
5
customList/repoList.list
Normal file
5
customList/repoList.list
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
https://www.pixiv.net/
|
||||||
|
https://pixiv.net/
|
||||||
|
https://i.pximg.net/
|
||||||
|
https://pximg.net/
|
||||||
|
https://wiki.archlinux.org/
|
||||||
4
customList/repoList.list.example
Normal file
4
customList/repoList.list.example
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
https://www.pixiv.net/
|
||||||
|
https://pixiv.net/
|
||||||
|
https://i.pximg.net/
|
||||||
|
https://pximg.net/
|
||||||
4
customList/requirements.txt
Normal file
4
customList/requirements.txt
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
#beautifulsoup4==4.13.4
|
||||||
|
requests==2.32.3
|
||||||
|
schedule==1.2.2
|
||||||
|
nslookup==1.8.1
|
||||||
129
customList/whatDomain.py
Normal file
129
customList/whatDomain.py
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
#from nslookup import Nslookup
|
||||||
|
from typing import Optional, Annotated
|
||||||
|
import dns, dns.resolver
|
||||||
|
|
||||||
|
# https://www.codeunderscored.com/nslookup-python/
|
||||||
|
|
||||||
|
def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
|
||||||
|
#dns_query = Nslookup()
|
||||||
|
"""
|
||||||
|
Tells you what IPv4 address/es a domain point to.
|
||||||
|
Returns:
|
||||||
|
dict: A dictionary with IP addresses associated with that domain.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
# i = 0
|
||||||
|
outDict = {}
|
||||||
|
|
||||||
|
#result = dns_query.dns_lookup("example.com")
|
||||||
|
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||||
|
try:
|
||||||
|
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
|
||||||
|
except dns.resolver.NoAnswer:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("No answer from dns server.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.NoNameservers:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.NXDOMAIN:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("The DNS query name does not exist.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.LifetimeTimeout:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||||
|
return 1
|
||||||
|
for i, something in enumerate(result):
|
||||||
|
outDict[i] = something.to_text()
|
||||||
|
# i += 1
|
||||||
|
|
||||||
|
return outDict
|
||||||
|
|
||||||
|
def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
|
||||||
|
#dns_query = Nslookup()
|
||||||
|
"""
|
||||||
|
Tells you what IPv6 address/es a domain point to.
|
||||||
|
Returns:
|
||||||
|
dict: A dictionary with IP addresses associated with that domain.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
# i = 0
|
||||||
|
outDict = {}
|
||||||
|
|
||||||
|
#result = dns_query.dns_lookup("example.com")
|
||||||
|
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||||
|
try:
|
||||||
|
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
|
||||||
|
except dns.resolver.NoAnswer:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("No answer from dns server.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.NoNameservers:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.NXDOMAIN:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("The DNS query name does not exist.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.LifetimeTimeout:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||||
|
return 1
|
||||||
|
for i, something in enumerate(result):
|
||||||
|
outDict[i] = something.to_text()
|
||||||
|
# i += 1
|
||||||
|
|
||||||
|
return outDict
|
||||||
|
|
||||||
|
|
||||||
|
def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]):
|
||||||
|
#dns_query = Nslookup()
|
||||||
|
"""
|
||||||
|
Tells you what IPv6 address/es a domain point to.
|
||||||
|
Returns:
|
||||||
|
dict: A dictionary with IP addresses associated with that domain.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa"
|
||||||
|
|
||||||
|
|
||||||
|
# i = 0
|
||||||
|
outDict = {}
|
||||||
|
|
||||||
|
#result = dns_query.dns_lookup("example.com")
|
||||||
|
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||||
|
try:
|
||||||
|
result = dns.resolver.resolve(whatToCheck, 'PTR')
|
||||||
|
except dns.resolver.NoAnswer:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("No answer from dns server.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.NoNameservers:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.NXDOMAIN:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("The DNS query name does not exist.\n")
|
||||||
|
return 1
|
||||||
|
except dns.resolver.LifetimeTimeout:
|
||||||
|
print("\nDNS ERROR")
|
||||||
|
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||||
|
return 1
|
||||||
|
for i, something in enumerate(result):
|
||||||
|
outDict[i] = something.to_text()
|
||||||
|
# i += 1
|
||||||
|
|
||||||
|
return outDict
|
||||||
|
|
||||||
|
|
||||||
|
#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
|
||||||
|
#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
|
||||||
|
#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))
|
||||||
BIN
docker.tar.gz
Normal file
BIN
docker.tar.gz
Normal file
Binary file not shown.
@@ -14,11 +14,15 @@ WORKDIR /etc/debmirror/
|
|||||||
|
|
||||||
RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \
|
RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \
|
||||||
ln -s /etc/debmirror/Repo-IP-lists/MirrorListV4 /etc/debmirror/MirrorListV4 && \
|
ln -s /etc/debmirror/Repo-IP-lists/MirrorListV4 /etc/debmirror/MirrorListV4 && \
|
||||||
ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6
|
ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6 && \
|
||||||
|
ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV4 /etc/debmirror/OPNS_MirrorListV4 && \
|
||||||
|
ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV6 /etc/debmirror/OPNS_MirrorListV6
|
||||||
|
|
||||||
# RUN touch /etc/debmirror/MirrorListV6 && \
|
# RUN touch /etc/debmirror/MirrorListV6 && \
|
||||||
# touch /etc/debmirror/MirrorListV4
|
# touch /etc/debmirror/MirrorListV4
|
||||||
|
|
||||||
COPY mainDocker.py /etc/debmirror/
|
COPY mainDocker.py /etc/debmirror/
|
||||||
|
COPY mainOPNsense.py /etc/debmirror/
|
||||||
COPY whatDomain.py /etc/debmirror/
|
COPY whatDomain.py /etc/debmirror/
|
||||||
COPY requirements.txt /etc/debmirror/
|
COPY requirements.txt /etc/debmirror/
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
# python
|
# python
|
||||||
|
20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||||
|
25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||||
|
# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||||
# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||||
*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||||
|
#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||||
|
|
||||||
# git push
|
# git push
|
||||||
# 30 19 * * * /bin/sh /etc/debmirror/gitPush.sh
|
30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||||
*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh
|
#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||||
@@ -3,14 +3,14 @@
|
|||||||
services:
|
services:
|
||||||
debmirup:
|
debmirup:
|
||||||
container_name: Debian_Mirrors_Updater
|
container_name: Debian_Mirrors_Updater
|
||||||
build: ./
|
# build: ./
|
||||||
image: debmirrorupd:V2.0.1
|
image: yuruc3/debianrepolist:v0.3
|
||||||
environment:
|
environment:
|
||||||
- DEBMIRRORURL=https://www.debian.org/mirror/list
|
- DEBMIRRORURL=https://www.debian.org/mirror/list
|
||||||
- GITURLPROTO=https
|
- GITURLPROTO=https
|
||||||
- GITURL=tea.shupogaki.org
|
- GITURL=tea.shupogaki.org
|
||||||
- GITREPOPATH=YuruC3/Debain-repos.git
|
- GITREPOPATH=YuruC3/Repo-IP-lists.git
|
||||||
- GITEA_TOKEN=<CHANGHE_ME>
|
- GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2
|
||||||
- EXTRAREPOS=True
|
- EXTRAREPOS=True
|
||||||
- SECURITYREPOS=True
|
- SECURITYREPOS=True
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|||||||
@@ -5,39 +5,83 @@ WORKPTH="/etc/debmirror/"
|
|||||||
REPO_DIR="$WORKPTH/Repo-IP-lists"
|
REPO_DIR="$WORKPTH/Repo-IP-lists"
|
||||||
REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
||||||
|
|
||||||
|
# Clone repo if not exists
|
||||||
# Clone repo only if it doesn't already exist
|
|
||||||
if [ ! -d "$REPO_DIR/.git" ]; then
|
if [ ! -d "$REPO_DIR/.git" ]; then
|
||||||
echo "[$(date)] Cloning repository..."
|
echo "[$(date)] Cloning repository..."
|
||||||
git clone "$REPO_URL" "$REPO_DIR"
|
git clone "$REPO_URL" "$REPO_DIR"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
cd "$REPO_DIR"
|
cd "$REPO_DIR"
|
||||||
|
|
||||||
|
# Abort previous rebase/cherry-pick if stuck
|
||||||
|
git rebase --abort 2>/dev/null || true
|
||||||
|
git cherry-pick --abort 2>/dev/null || true
|
||||||
|
|
||||||
|
# Make sure we're on a clean 'main'
|
||||||
|
git checkout main || git checkout -b main origin/main
|
||||||
|
git pull --rebase --autostash
|
||||||
|
|
||||||
git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
||||||
|
|
||||||
|
|
||||||
git config user.name "UpdateBot"
|
git config user.name "UpdateBot"
|
||||||
git config user.email "UpdateBot@localhost.local"
|
git config user.email "UpdateBot@localhost.local"
|
||||||
|
|
||||||
# If anything to commit locally, commit it now
|
# Stage the files
|
||||||
if ! git diff --quiet; then
|
git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
|
||||||
echo "[$(date)] Committing local changes before pulling"
|
|
||||||
git commit -m "Auto-commit before pull on $(date -Iseconds)"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Now pull the latest
|
# Only proceed if there are staged changes
|
||||||
git pull --rebase --autostash
|
if ! git diff --quiet --cached; then
|
||||||
|
echo "[$(date)] Committing and pushing changes..."
|
||||||
# stage files
|
git commit -m "Auto-update mirror list on $(date -Iseconds)"
|
||||||
git add MirrorListV4 MirrorListV6
|
# git pull --rebase --autostash
|
||||||
|
|
||||||
# Commit and push only if there's anything new staged
|
|
||||||
if git diff --quiet; then
|
|
||||||
echo "[$(date)] No changes to commit."
|
|
||||||
else
|
|
||||||
git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet
|
|
||||||
git push --quiet
|
git push --quiet
|
||||||
echo "[$(date)] Changes pushed successfully."
|
echo "[$(date)] Changes pushed."
|
||||||
|
else
|
||||||
|
echo "[$(date)] No changes to commit or push."
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
|
||||||
|
# #!/bin/sh
|
||||||
|
# set -e
|
||||||
|
|
||||||
|
# WORKPTH="/etc/debmirror/"
|
||||||
|
# REPO_DIRd="$WORKPTH/Repo-IP-lists"
|
||||||
|
# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
||||||
|
|
||||||
|
|
||||||
|
# # Clone repo only if it doesn't already exist
|
||||||
|
# if [ ! -d "$REPO_DIR/.git" ]; then
|
||||||
|
# echo "[$(date)] Cloning repository..."
|
||||||
|
# git clone "$REPO_URL" "$REPO_DIR"
|
||||||
|
# fi
|
||||||
|
|
||||||
|
|
||||||
|
# cd "$REPO_DIR"
|
||||||
|
|
||||||
|
# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
||||||
|
|
||||||
|
# git config user.name "UpdateBot"
|
||||||
|
# git config user.email "UpdateBot@localhost.local"
|
||||||
|
|
||||||
|
# # stage files
|
||||||
|
# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
|
||||||
|
|
||||||
|
# # If anything to commit locally, commit it now
|
||||||
|
# if ! git diff --quiet --cached; then
|
||||||
|
# echo "[$(date)] Committing local changes before pulling"
|
||||||
|
# git commit -m "Auto-commit before pull on $(date -Iseconds)"
|
||||||
|
# fi
|
||||||
|
|
||||||
|
# # Now pull the latest
|
||||||
|
# git pull --rebase --autostash
|
||||||
|
|
||||||
|
# # Commit and push only if there's anything new staged
|
||||||
|
# if git diff --quiet; then
|
||||||
|
# echo "[$(date)] No changes to commit."
|
||||||
|
# else
|
||||||
|
# git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet
|
||||||
|
# git push --quiet
|
||||||
|
# echo "[$(date)] Changes pushed successfully."
|
||||||
|
# fi
|
||||||
|
|
||||||
|
|||||||
@@ -2,13 +2,9 @@ import requests, schedule, time, os
|
|||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
from whatDomain import *
|
from whatDomain import *
|
||||||
|
|
||||||
|
DEBMIRRORURL = str(os.getenv("DEBMIRRORURL", "https://www.debian.org/mirror/list"))
|
||||||
|
DEBSECURITYURL = str(os.getenv("DEBSECURITYURL", "https://security.debian.org/debian-security/"))
|
||||||
|
|
||||||
if os.environ["DEBMIRRORURL"]:
|
|
||||||
DEBMIRRORURL = os.environ["DEBMIRRORURL"]
|
|
||||||
else:
|
|
||||||
DEBMIRRORURL = "https://www.debian.org/mirror/list"
|
|
||||||
|
|
||||||
DEBSECURITYURL = ["https://security.debian.org/debian-security/"]
|
|
||||||
EXTRASURL = ["https://download.docker.com/linux/debian/",
|
EXTRASURL = ["https://download.docker.com/linux/debian/",
|
||||||
# Double just to be sure. Even though they point to the same IP
|
# Double just to be sure. Even though they point to the same IP
|
||||||
"http://download.proxmox.com/debian/",
|
"http://download.proxmox.com/debian/",
|
||||||
@@ -46,7 +42,7 @@ def sanitizeURL(inpurl: str):
|
|||||||
elif "http://" or "https://" not in url:
|
elif "http://" or "https://" not in url:
|
||||||
outurl = inpurl
|
outurl = inpurl
|
||||||
else:
|
else:
|
||||||
return -1
|
return 1
|
||||||
|
|
||||||
i = 0
|
i = 0
|
||||||
for char in outurl:
|
for char in outurl:
|
||||||
|
|||||||
135
docker/mainOPNsense.py
Normal file
135
docker/mainOPNsense.py
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
import requests, schedule, time
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
|
||||||
|
|
||||||
|
|
||||||
|
OPNSNSMIRRORURL = str(os.getenv("OPNSNSMIRRORURL", "https://opnsense.org/download/#full-mirror-listing"))
|
||||||
|
|
||||||
|
IPv4FILE = str(os.getenv("IPv4FILE", "/etc/debmirror/OPNS_MirrorListV4"))
|
||||||
|
IPv6FILE = str(os.getenv("IPv6FILE", "/etc/debmirror/OPNS_MirrorListV6"))
|
||||||
|
|
||||||
|
|
||||||
|
def sanitizeURL(inpurl: str):
|
||||||
|
if not "/" in inpurl[:-1]:
|
||||||
|
inpurl += "/"
|
||||||
|
if "https://" in inpurl:
|
||||||
|
outurl = inpurl[8:]
|
||||||
|
elif "http://" in inpurl:
|
||||||
|
outurl = inpurl[7:]
|
||||||
|
elif "http://" or "https://" not in url:
|
||||||
|
outurl = inpurl
|
||||||
|
else:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
# how the fuck does it work?
|
||||||
|
# I mean I wrote this but I don't know why does it work.
|
||||||
|
i = 0
|
||||||
|
for char in outurl:
|
||||||
|
i += 1
|
||||||
|
if char == "/":
|
||||||
|
outurl = outurl[:i]
|
||||||
|
|
||||||
|
if char == "/":
|
||||||
|
outurl = outurl[:-1]
|
||||||
|
return outurl
|
||||||
|
|
||||||
|
|
||||||
|
def getFreshData():
|
||||||
|
payload = requests.get(OPNSNSMIRRORURL)
|
||||||
|
soup = BeautifulSoup(payload.content, "html.parser")
|
||||||
|
return soup
|
||||||
|
|
||||||
|
def LeJob():
|
||||||
|
|
||||||
|
print("Starting lookup")
|
||||||
|
|
||||||
|
LeSoup = getFreshData()
|
||||||
|
|
||||||
|
# print(LeMirrorDict)
|
||||||
|
|
||||||
|
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
|
||||||
|
|
||||||
|
for data in LeSoup.find_all('div', class_='download_section'):
|
||||||
|
for a in data.find_all('a', href=True):
|
||||||
|
|
||||||
|
url = a['href']
|
||||||
|
|
||||||
|
saniturl = sanitizeURL(url)
|
||||||
|
|
||||||
|
# print(saniturl)
|
||||||
|
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||||
|
|
||||||
|
# print(IPv4Dict)
|
||||||
|
try:
|
||||||
|
for key, ip in IPv4Dict.items():
|
||||||
|
print(f"Found the ipv4: {ip}")
|
||||||
|
fW.write(ip + "/32" + "\n")
|
||||||
|
|
||||||
|
# If int returned from WhatDomain then it is error.
|
||||||
|
# Error type is described via prints from whatdomain functions
|
||||||
|
except AttributeError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
|
||||||
|
|
||||||
|
for data in LeSoup.find_all('div', class_='download_section'):
|
||||||
|
for a in data.find_all('a', href=True):
|
||||||
|
|
||||||
|
url = a['href']
|
||||||
|
|
||||||
|
saniturl = sanitizeURL(url)
|
||||||
|
|
||||||
|
# print(saniturl)
|
||||||
|
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||||
|
|
||||||
|
# print(IPv6Dict)
|
||||||
|
try:
|
||||||
|
for key, ip in IPv6Dict.items():
|
||||||
|
print(f"Found the ipv6: {ip}")
|
||||||
|
fW.write(ip + "/128" + "\n")
|
||||||
|
|
||||||
|
# If int returned from WhatDomain then it is error.
|
||||||
|
# Error type is described via prints from whatdomain functions
|
||||||
|
except AttributeError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# schedule.every().day.at("12:45").do(LeJob)
|
||||||
|
# schedule.every().day.at("17:44").do(LeJob)
|
||||||
|
|
||||||
|
# while True:
|
||||||
|
# schedule.run_pending()
|
||||||
|
# print("Waiting...")
|
||||||
|
# time.sleep(30) #Wait one minute
|
||||||
|
# # LeJob()
|
||||||
|
|
||||||
|
LeJob()
|
||||||
|
|
||||||
|
# gigalist = []
|
||||||
|
|
||||||
|
# payload = requests.get(OPNSNSMIRRORURL)
|
||||||
|
# soup = BeautifulSoup(payload.content, "html.parser")
|
||||||
|
|
||||||
|
|
||||||
|
# for data in soup.find_all('div', class_='download_section'):
|
||||||
|
# for a in data.find_all('a', href=True):
|
||||||
|
|
||||||
|
# url = a['href']
|
||||||
|
|
||||||
|
# saniturl = sanitizeURL(url)
|
||||||
|
|
||||||
|
# # print(saniturl)
|
||||||
|
# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||||
|
# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||||
|
|
||||||
|
# # print(IPv4Dict)
|
||||||
|
# try:
|
||||||
|
# for key, ip in IPv4Dict.items():
|
||||||
|
# print(f"Found the ipv4: {ip}")
|
||||||
|
|
||||||
|
# for key, ip in IPv6Dict.items():
|
||||||
|
# print(f"Found the ipv6: {ip}")
|
||||||
|
# # If int returned from WhatDomain then it is error.
|
||||||
|
# # Error type is described via prints from whatdomain functions
|
||||||
|
# except AttributeError:
|
||||||
|
# continue
|
||||||
|
|
||||||
Reference in New Issue
Block a user