Compare commits

..

8 Commits

Author SHA1 Message Date
c253398809 Code works now from how it looks 2026-01-25 00:04:48 +01:00
72e5bdfdf5 Changed if to POSIX [] instead of [[]] 2026-01-25 00:04:29 +01:00
01bafc6ba0 Added a funni README 2026-01-25 00:04:22 +01:00
98c5807b85 Fixed Dockerfile 2026-01-25 00:04:00 +01:00
0fb6f9baa5 Added docker-compose 2026-01-24 23:37:50 +01:00
1ad1e26f72 Adjusted Entrypoint so that it crontabs main.py 2026-01-24 22:55:01 +01:00
543ac2a724 Slight unrelated and unimportant edits 2026-01-24 22:53:51 +01:00
3c17b0a9ab Consolidated all functions into one 2026-01-24 22:53:27 +01:00
9 changed files with 495 additions and 35 deletions

View File

@@ -1,11 +1,12 @@
FROM alpine:latest
# https://docs.docker.com/reference/dockerfile/#environment-replacement
ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
# ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
# By default these are scraped
ENV EXTRA_REPOS=False
ENV SECURITY_REPOS=True
ENV EXTRA_REPOS=True
ENV PROXMOX_REPOS=True
ENV DOCKER_REPOS=True
ENV DEBIAN_REPOS=True
ENV OPNSENSE_REPOS=True
@@ -25,7 +26,9 @@ WORKDIR /etc/debmirror/
# touch /etc/debmirror/MirrorListV4
# Copy code
COPY ./code/* /etc/debmirror/
COPY ./code/main.py /etc/debmirror/
COPY ./code/whatDomain.py /etc/debmirror/
COPY ./entrypoint.sh /
# Copy important files
# COPY ./init.sh /etc/debmirror/

408
AiO_Container/code/main.py Normal file
View File

@@ -0,0 +1,408 @@
import requests, schedule, time, os
from bs4 import BeautifulSoup
from whatDomain import *
# import whatDomain
# Debian
DEBMIRRORURL = str(os.getenv("DEBMIRRORURL", "https://www.debian.org/mirror/list"))
DEBSECURITYURL = str(os.getenv("DEBSECURITYURL", "https://security.debian.org/debian-security/"))
DEBIANMIRRORLISTV4 = "/etc/debmirror/DebianMirrorListV4"
DEBIANMIRRORLISTV6 = "/etc/debmirror/DebianMirrorListV6"
target_countries = set([
# Europe
"Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark",
"Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland",
"Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland",
"Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland",
"United Kingdom", "Moldova",
# America
"Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
"Mexico", "Peru", "United States", "Uruguay", "Venezuela" #,
# Others
# "Security", "Extras"
])
# Proxmox
PROXURL = ["http://download.proxmox.com/debian/", "https://enterprise.proxmox.com/debian/pve/",]
PROXMOXMIRRORLISTV4 = "/etc/debmirror/ProxmoxMirrorListV4"
PROXMOXMIRRORLISTV6 = "/etc/debmirror/ProxmoxMirrorListV6"
# Docker
DOCKERURLS = ["https://download.docker.com/linux/debian/",
"https://nvidia.github.io/libnvidia-container/stable/deb/",
"https://nvidia.github.io/libnvidia-container/experimental/deb/"]
DOCKERMIRRORLISTV4 = "/etc/debmirror/DockerMirrorListV4"
DOCKERMIRRORLISTV6 = "/etc/debmirror/DockerMirrorListV6"
# OPNsense
OPNSNSMIRRORURL = str(os.getenv("OPNSNSMIRRORURL", "https://opnsense.org/download/#full-mirror-listing"))
OPNSENSEMIRRORLISTV4 = "/etc/debmirror/OPNsenseMirrorListV4"
OPNSENSEMIRRORLISTV6 = "/etc/debmirror/OPNsenseMirrorListV6"
# Custom
EXTRAURL = []
if os.getenv("EXTRA_REPOS", True):
try:
with open(os.getenv("REPOFILE", "customRepoList.list"), 'r') as repoListFile:
for repoUrl in repoListFile:
# print(repoUrl.strip())
EXTRAURL.append(repoUrl.strip())
except FileNotFoundError:
print("File not passed")
exit(2)
CUSTOMMIRRORLISTV4 = "/etc/debmirror/CustomMirrorListV4"
CUSTOMMIRRORLISTV6 = "/etc/debmirror/CustomMirrorListV6"
# Helper
def sanitizeURL(inpurl: str):
if "https://" in inpurl:
outurl = inpurl[8:]
elif "http://" in inpurl:
outurl = inpurl[7:]
# elif " " in inpurl:
# return 7
elif "http://" or "https://" not in inpurl:
outurl = inpurl
else:
return inpurl
i = 0
for char in outurl:
i += 1
if char == "/":
outurl = outurl[:i]
if char == "/":
outurl = outurl[:-1]
return outurl
# Debian
def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
outMirrorDict = {}
current_country = None
# Iterate through all table rows
for table in SoupInput.find_all("table"):
for row in table.find_all("tr"):
# Check for country name in a full-row header (<strong><big>)
strong = row.find("strong")
if strong:
country_name = strong.get_text(strip=True)
if country_name in target_countries:
current_country = country_name
else:
current_country = None
continue # move to next row
# Check for inline country name in first column
cols = row.find_all("td")
if len(cols) >= 2:
possible_country = cols[0].get_text(strip=True)
link_tag = cols[1].find("a", href=True)
if possible_country in target_countries:
current_country = possible_country
if current_country and link_tag:
url = link_tag['href']
if current_country not in outMirrorDict:
outMirrorDict[current_country] = []
outMirrorDict[current_country].append(url)
# if os.environ["SECURITYREPOS"]:
outMirrorDict.update({"Security": DEBSECURITYURL})
return outMirrorDict
def debianJob():
print("\nStarting lookup for Debian")
payload = requests.get(DEBMIRRORURL)
LeSoup = BeautifulSoup(payload.content, "html.parser")
LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)
# print(LeMirrorDict)
with open(DEBIANMIRRORLISTV4, "r",) as fR, open(DEBIANMIRRORLISTV4, "w",) as fW:
for key, urls in LeMirrorDict.items():
# print(urls)
if key in target_countries:
for url in urls:
# print(url)
if url not in fR:
goodurl = sanitizeURL(url)
if goodurl == 7:
continue
# print(goodurl)
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
try:
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/32" + "\n")
except AttributeError:
continue
with open(DEBIANMIRRORLISTV6, "r",) as fR, open(DEBIANMIRRORLISTV6, "w",) as fW:
for key, urls in LeMirrorDict.items():
if key in target_countries:
for url in urls:
if url not in fR:
goodurl = sanitizeURL(url)
if goodurl == 7:
continue
# print(goodurl)
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
try:
for key, ip in ip6Dict.items():
# print(ip)
fW.write(ip + "/128" + "\n")
except AttributeError:
continue
# Proxmox
def proxmoxJob():
print("\nStarting lookup for Proxmox")
with open(PROXMOXMIRRORLISTV4, "r",) as fR, open(PROXMOXMIRRORLISTV4, "w",) as fW:
for url in PROXURL:
if url not in fR:
sanitizedURL = sanitizeURL(url)
if sanitizedURL == 7:
continue
ip4Dict = ermWhatATheIpFromDomainYaCrazy(sanitizedURL)
try:
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/32" + "\n")
except AttributeError:
continue
with open(PROXMOXMIRRORLISTV6, "r",) as fR, open(PROXMOXMIRRORLISTV6, "w",) as fW:
for url in PROXURL:
if url not in fR:
sanitizedURL = sanitizeURL(url)
if sanitizedURL == 7:
continue
ip4Dict = ermWhatAAAATheIpFromDomainYaCrazy(sanitizedURL)
try:
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/128" + "\n")
except AttributeError:
continue
# OPNsense
def opnsenseJob():
print("\nStarting lookup for OPNSense")
payload = requests.get(OPNSNSMIRRORURL)
LeOPNSoup = BeautifulSoup(payload.content, "html.parser")
# print(LeMirrorDict)
with open(OPNSENSEMIRRORLISTV4, "r",) as fR, open(OPNSENSEMIRRORLISTV4, "w",) as fW:
for data in LeOPNSoup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
url = a['href']
saniturl = sanitizeURL(url)
if saniturl == 7:
continue
# print(saniturl)
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
# print(IPv4Dict)
try:
for key, ip in IPv4Dict.items():
print(f"Found the ipv4: {ip}")
fW.write(ip + "/32" + "\n")
# If int returned from WhatDomain then it is error.
# Error type is described via prints from whatdomain functions
except AttributeError:
continue
with open(OPNSENSEMIRRORLISTV6, "r",) as fR, open(OPNSENSEMIRRORLISTV6, "w",) as fW:
for data in LeOPNSoup.find_all('div', class_='download_section'):
for a in data.find_all('a', href=True):
url = a['href']
saniturl = sanitizeURL(url)
if saniturl == 7:
continue
# print(saniturl)
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
# print(IPv6Dict)
try:
for key, ip in IPv6Dict.items():
print(f"Found the ipv6: {ip}")
fW.write(ip + "/128" + "\n")
# If int returned from WhatDomain then it is error.
# Error type is described via prints from whatdomain functions
except AttributeError:
continue
# Docker
def dockerJob():
print("\nStarting lookup for Docker")
with open(DOCKERMIRRORLISTV4, "r",) as fR, open(DOCKERMIRRORLISTV4, "w",) as fW:
for url in DOCKERURLS:
if url not in fR:
sanitizedURL = sanitizeURL(url)
if sanitizedURL == 7:
continue
ip4Dict = ermWhatATheIpFromDomainYaCrazy(sanitizedURL)
try:
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/32" + "\n")
except AttributeError:
continue
with open(DOCKERMIRRORLISTV6, "r",) as fR, open(DOCKERMIRRORLISTV6, "w",) as fW:
for url in DOCKERURLS:
if url not in fR:
sanitizedURL = sanitizeURL(url)
if sanitizedURL == 7:
continue
ip4Dict = ermWhatAAAATheIpFromDomainYaCrazy(sanitizedURL)
try:
for key, ip in ip4Dict.items():
print(ip)
fW.write(ip + "/128" + "\n")
except AttributeError:
continue
# Custom
def customLinksJob():
print("\nStarting lookup for Custom domains")
with open(CUSTOMMIRRORLISTV4, "w",) as fW, open(CUSTOMMIRRORLISTV4, "r",) as fR:
for url in EXTRAURL:
# print(url)
goodurl = sanitizeURL(url)
# print(goodurl)
if url not in fR:
if goodurl == 7:
continue
# print(goodurl)
# ip4Dict = ermWhatATheIpFromDomainYaCrazy(url)
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
try:
for key, ip in ip4Dict.items():
print(ip + "/32")
fW.write(ip + "/32" + "\n")
except AttributeError:
continue
with open(CUSTOMMIRRORLISTV6, "w",) as fW, open(CUSTOMMIRRORLISTV6, "r",) as fR:
for url in EXTRAURL:
goodurl = sanitizeURL(url)
# print(goodurl)
if url not in fR:
if goodurl == 7:
continue
# print(goodurl)
# ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url)
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
try:
for key, ip in ip6Dict.items():
print(ip + "/128")
fW.write(ip + "/128" + "\n")
except AttributeError:
continue
if __name__ == "__main__":
if os.getenv("EXTRA_REPOS", True):
customLinksJob()
# if os.getenv("PROXMOX_REPOS", True):
# proxmoxJob()
# if os.getenv("DOCKER_REPOS", True):
# dockerJob()
# if os.getenv("DEBIAN_REPOS", True):
# debianJob()
# if os.getenv("OPNSENSE_REPOS", True):
# opnsenseJob()

View File

@@ -1,6 +1,7 @@
import requests, schedule, time, os
# from bs4 import BeautifulSoup
from whatDomain import *
# from whatDomain import *
import whatDomain
EXTRAURL = []
@@ -90,7 +91,7 @@ def LeJob():
if __name__ == "__main__":
LeJob()
print("Done")

View File

@@ -10,7 +10,7 @@ EXTRASURL = ["https://download.docker.com/linux/debian/",
"http://download.proxmox.com/debian/",
"https://enterprise.proxmox.com/debian/pve/",
# That's for nvidia docker toolkit something something
# stuff that makes containers use nvenc and shit
# stuff that makes containers use nvenc
"https://nvidia.github.io/libnvidia-container/stable/deb/",
"https://nvidia.github.io/libnvidia-container/experimental/deb/"]

View File

@@ -124,6 +124,8 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
return outDict
#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))
if __name__ == "__main__":
print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))

View File

@@ -0,0 +1,19 @@
---
services:
debmirup:
container_name: Debian_Mirrors_Updater
# build: ./
image: tea.shupogaki.org/yuruc3/debianrepolist:v0.4.1
environment:
- GITURLPROTO=https
- GITURL=tea.shupogaki.org
- GITREPOPATH=YuruC3/Repo-IP-lists.git
- GITEA_TOKEN=6767
- EXTRA_REPOS=True
- PROXMOX_REPOS=True
- DOCKER_REPOS=True
- DEBIAN_REPOS=True
- OPNSENSE_REPOS=True
restart: unless-stopped

View File

@@ -7,11 +7,15 @@ PGID="${PGID:-1600}"
USER="${USER:-muyu}"
GROUP="${USER:-muyu}"
CHOWNPATH="/etc/debmirror"
EXTRA_REPOS="${EXTRA_REPOS:-False}"
SECURITY_REPOS="${SECURITY_REPOS:-True}"
EXTRA_REPOS="${EXTRA_REPOS:-True}"
PROXMOX_REPOS="${PROXMOX_REPOS:-True}"
DOCKER_REPOS="${DOCKER_REPOS:-True}"
DEBIAN_REPOS="${DEBIAN_REPOS:-True}"
OPNSENSE_REPOS="${OPNSENSE_REPOS:-True}"
THE_REPO="${THE_REPO:-https://tea.shupogaki.org/YuruC3/Repo-IP-lists}"
# Create group if missing
if ! getent group "$GROUP" >/dev/null 2>&1; then
addgroup -g "$PGID" "$GROUP"
@@ -27,32 +31,50 @@ touch /etc/crontabs/cron-jobs
# Create symlinks based on what is passed
# Also add cronjobs when applicable
git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists
if [[ "$EXTRA_REPOS" ]]; then
git clone $THE_REPO
if [ "$EXTRA_REPOS" = true ]; then
touch /etc/debmirror/Repo-IP-lists/CustomMirrorListV4
touch /etc/debmirror/Repo-IP-lists/CustomMirrorListV6
ln -s /etc/debmirror/Repo-IP-lists/CustomMirrorListV4 /etc/debmirror/CustomMirrorListV4
ln -s /etc/debmirror/Repo-IP-lists/CustomMirrorListV6 /etc/debmirror/CustomMirrorListV6
echo "00 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainCustom.py" >> /etc/crontabs/cron-jobs
# echo "00 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainCustom.py" >> /etc/crontabs/cron-jobs
fi
if [[ "$SECURITY_REPOS" ]]; then
ln -s /etc/debmirror/Repo-IP-lists/SecurityMirrorListV4 /etc/debmirror/SecurityMirrorListV4
ln -s /etc/debmirror/Repo-IP-lists/SecurityMirrorListV6 /etc/debmirror/SecurityMirrorListV6
if [ "$PROXMOX_REPOS" = true ]; then
touch /etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV4
touch /etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV6
ln -s /etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV4 /etc/debmirror/ProxmoxMirrorListV4
ln -s /etc/debmirror/Repo-IP-lists/ProxmoxMirrorListV6 /etc/debmirror/ProxmoxMirrorListV6
echo "05 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainSec.py" >> /etc/crontabs/cron-jobs
# echo "05 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainSec.py" >> /etc/crontabs/cron-jobs
fi
if [[ "$DEBIAN_REPOS" ]]; then
if [ "$DOCKER_REPOS" = true ]; then
touch /etc/debmirror/Repo-IP-lists/DockerMirrorListV4
touch /etc/debmirror/Repo-IP-lists/DockerMirrorListV6
ln -s /etc/debmirror/Repo-IP-lists/DockerMirrorListV4 /etc/debmirror/DockerMirrorListV4
ln -s /etc/debmirror/Repo-IP-lists/DockerMirrorListV6 /etc/debmirror/DockerMirrorListV6
# echo "05 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainSec.py" >> /etc/crontabs/cron-jobs
fi
if [ "$DEBIAN_REPOS" = true ]; then
touch /etc/debmirror/Repo-IP-lists/DebianMirrorListV4
touch /etc/debmirror/Repo-IP-lists/DebianMirrorListV6
ln -s /etc/debmirror/Repo-IP-lists/DebianMirrorListV4 /etc/debmirror/DebianMirrorListV4
ln -s /etc/debmirror/Repo-IP-lists/DebianMirrorListV6 /etc/debmirror/DebianMirrorListV6
echo "10 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py" >> /etc/crontabs/cron-jobs
# echo "10 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py" >> /etc/crontabs/cron-jobs
fi
if [[ "$OPNSENSE_REPOS" ]]; then
ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMIrrorListV4 /etc/debmirror/OPNsenseMIrrorListV4
ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMIrrorListV6 /etc/debmirror/OPNsenseMIrrorListV6
if [ "$OPNSENSE_REPOS" = true ]; then
touch /etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV4
touch /etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV6
ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV4 /etc/debmirror/OPNsenseMirrorListV4
ln -s /etc/debmirror/Repo-IP-lists/OPNsenseMirrorListV6 /etc/debmirror/OPNsenseMirrorListV6
echo "15 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py" >> /etc/crontabs/cron-jobs
# echo "15 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py" >> /etc/crontabs/cron-jobs
fi
echo "00 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/main.py" >> /etc/crontabs/cron-jobs
echo "30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh" >> /etc/crontabs/cron-jobs
# Configure the crontab to work

View File

@@ -7,18 +7,18 @@ GITURL="${GITURL:-6767}"
GITREPOPATH="${GITREPOPATH:-6767}"
GITURLPROTO="${GITURLPROTO:-http}"
# if [[ "$GITEA_TOKEN" == "6767" || "$GITREPOPATH" == "6767" || "$GITURL" == "6767" ]]; then
if [[ "$GITEA_TOKEN" == "6767" ]]; then
# if [ "$GITEA_TOKEN" == "6767" || "$GITREPOPATH" == "6767" || "$GITURL" == "6767" ]; then
if [ "$GITEA_TOKEN" == "6767" ]; then
exit 1
fi
WORKPTH="${WORKPTH:-/etc/debmirror/}"
REPO_DIR="${REPO_DIR:-$WORKPTH/Repo-IP-lists}"
REPO_URL="${REPO_URL:-GITURLPROTO://GITURL/GITREPOPATH}"
REPO_URL="${REPO_URL:-$GITURLPROTO://$GITURL/$GITREPOPATH}"
# Clone repo if not exists
if [[ ! -d "$REPO_DIR/.git" ]]; then
if [ ! -d "$REPO_DIR/.git" ]; then
echo "[$(date)] Cloning repository..."
git clone "$REPO_URL" "$REPO_DIR"
fi
@@ -33,8 +33,8 @@ git cherry-pick --abort 2>/dev/null || true
git checkout main || git checkout -b main origin/main
git pull --rebase --autostash
# git remote set-url origin "${GITURLPROTO}://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
git remote set-url origin "https://${GITEA_TOKEN}@YuruC3/Repo-IP-lists"
git remote set-url origin "${GITURLPROTO}://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
# git remote set-url origin "https://${GITEA_TOKEN}@YuruC3/Repo-IP-lists"
git config user.name "UpdateBot"
@@ -44,7 +44,7 @@ git config user.email "UpdateBot@localhost.local"
git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
# Only proceed if there are staged changes
if [[ ! git diff --quiet --cached ]]; then
if [ ! git diff --quiet --cached ]; then
echo "[$(date)] Committing and pushing changes..."
git commit -m "Auto-update mirror list on $(date -Iseconds)"
# git pull --rebase --autostash

View File

@@ -27,3 +27,8 @@ Here is a rundown of all the options in docker-compose
## Run at home
if you want to run it at your place remember to change crontab timings and git personal token and so on.
<p align="center">
<img src="https://data.shupogaki.org/assets/addbdfbdffwf.jpeg" alt="">
</p>