From f3a28cb10ffc9aceacb33422f0af7a57ae92b15f Mon Sep 17 00:00:00 2001 From: YuruC3 <98943911+YuruC3@@users.noreply.github.com> Date: Sun, 11 May 2025 19:16:07 +0200 Subject: [PATCH] V2.0.1 aka it's working now --- README.md | 22 ++++++++++++++++++++-- code/main.py | 12 +++++++++++- docker/.gitignore | 2 +- docker/Dockerfile | 2 ++ docker/docker-compose.yml | 10 +++++++--- docker/gitPush.sh | 14 ++++++++++---- docker/mainDocker.py | 16 ++++++++++++++-- 7 files changed, 65 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 926d66b..4d9c258 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,26 @@ It scrapes URL's of debian mirrors for later use in opnsense as an alias that on I'll be trying to maintain up to date list, aka just run a docker container and forget about it propably. -Also includes IPs for security.debian.org +Also includes IPs for security.debian.org, docker, proxmox and nvidia container toolkit + +## Docker + +Here is a rundown of all the options in docker-compose + +```DEBMIRRORURL``` is a link to Debian's mirror list. You don't have to fill it as I don't think they'll be changing that. I hope so at least. + +```GITURLPROTO``` protocol of git server. Usually https + +```GITURL``` address of git server. Fox example, github.com + +```GITREPOPATH``` is the path to your repository. For example, YuruC3/pmpw (shameless plug for my other project) + +```GITEA_TOKEN``` is a personal token used by git to authenticate. + +```EXTRAREPOS``` can be True of False. It includes IP's for security updates for debian. Pretty usefull. + +```SECURITYREPOS``` has IPs for things like docker, the whole proxmox suite and nvidia container toolkit. ## Run at home -if you want to run it at your place remember to change crontab timings and git personal token. \ No newline at end of file +if you want to run it at your place remember to change crontab timings and git personal token and so on. \ No newline at end of file diff --git a/code/main.py b/code/main.py index f4cc125..7a98dff 100644 --- a/code/main.py +++ b/code/main.py @@ -4,6 +4,15 @@ from whatDomain import * DEBSECURITYURL = ["https://security.debian.org/debian-security/"] +EXTRASURL = ["https://download.docker.com/linux/debian/", + # Double just to be sure. Even though they point to the same IP + "http://download.proxmox.com/debian/", + "https://enterprise.proxmox.com/debian/pve/", + # That's for nvidia docker toolkit something something + # stuff that makes containers use nvenc and shit + "https://nvidia.github.io/libnvidia-container/stable/deb/", + "https://nvidia.github.io/libnvidia-container/experimental/deb/"] + DEBMIRRORURL = "https://www.debian.org/mirror/list" IPv4FILE = "../MirrorListV4" IPv6FILE = "../MirrorListV6" @@ -21,7 +30,7 @@ target_countries = set([ "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador", "Mexico", "Peru", "United States", "Uruguay", "Venezuela", # Others - "Security" + "Security", "Extras" ]) @@ -84,6 +93,7 @@ def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup): outMirrorDict[current_country].append(url) outMirrorDict.update({"Security": DEBSECURITYURL}) + outMirrorDict.update({"Extras": EXTRASURL}) return outMirrorDict diff --git a/docker/.gitignore b/docker/.gitignore index 233416d..d1c9345 100644 --- a/docker/.gitignore +++ b/docker/.gitignore @@ -1,3 +1,3 @@ repo __pycache__ -/.env \ No newline at end of file +.env \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 68838f6..3c92766 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,6 +2,8 @@ FROM alpine:latest # https://docs.docker.com/reference/dockerfile/#environment-replacement ENV DEBMIRRORURL="https://www.debian.org/mirror/list" +ENV EXTRAREPOS=True +ENV SECURITYREPOS=True RUN apk update && \ apk add python3 py3-pip git diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ff8791f..0c40460 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -4,9 +4,13 @@ services: debmirup: container_name: Debian_Mirrors_Updater build: ./ - image: debmirrorupd:V1.1 + image: debmirrorupd:V2.0.1 environment: - DEBMIRRORURL=https://www.debian.org/mirror/list - - GITEA_TOKEN= + - GITURLPROTO=https + - GITURL=tea.shupogaki.org + - GITREPOPATH=YuruC3/Debain-repos.git + - GITEA_TOKEN= + - EXTRAREPOS=True + - SECURITYREPOS=True restart: unless-stopped - privileged: false diff --git a/docker/gitPush.sh b/docker/gitPush.sh index 25757f3..3689be9 100644 --- a/docker/gitPush.sh +++ b/docker/gitPush.sh @@ -3,7 +3,7 @@ set -e WORKPTH="/etc/debmirror/" REPO_DIR="$WORKPTH/Debain-repos" -REPO_URL="https://tea.shupogaki.org/YuruC3/Debain-repos.git" +REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}" # Clone repo only if it doesn't already exist @@ -15,13 +15,19 @@ fi cd "$REPO_DIR" -git remote set-url origin "https://${GITEA_TOKEN}@tea.shupogaki.org/YuruC3/Debain-repos.git" +git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}" git config user.name "UpdateBot" git config user.email "UpdateBot@localhost.local" -# pull changes -git pull +# If anything to commit locally, commit it now +if ! git diff --cached --quiet; then + echo "[$(date)] Committing local changes before pulling" + git commit -m "Auto-commit before pull on $(date -Iseconds)" +fi + +# Now pull the latest +git pull --rebase --autostash # stage files git add MirrorListV4 MirrorListV6 diff --git a/docker/mainDocker.py b/docker/mainDocker.py index f976bef..0cb0ed8 100644 --- a/docker/mainDocker.py +++ b/docker/mainDocker.py @@ -9,6 +9,15 @@ else: DEBMIRRORURL = "https://www.debian.org/mirror/list" DEBSECURITYURL = ["https://security.debian.org/debian-security/"] +EXTRASURL = ["https://download.docker.com/linux/debian/", + # Double just to be sure. Even though they point to the same IP + "http://download.proxmox.com/debian/", + "https://enterprise.proxmox.com/debian/pve/", + # That's for nvidia docker toolkit something something + # stuff that makes containers use nvenc and shit + "https://nvidia.github.io/libnvidia-container/stable/deb/", + "https://nvidia.github.io/libnvidia-container/experimental/deb/"] + IPv4FILE = "/etc/debmirror/MirrorListV4" IPv6FILE = "/etc/debmirror/MirrorListV6" @@ -25,7 +34,7 @@ target_countries = set([ "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador", "Mexico", "Peru", "United States", "Uruguay", "Venezuela", # Others - "Security" + "Security", "Extras" ]) @@ -87,7 +96,10 @@ def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup): outMirrorDict[current_country] = [] outMirrorDict[current_country].append(url) - outMirrorDict.update({"Security": DEBSECURITYURL}) + if os.environ["SECURITYREPOS"]: + outMirrorDict.update({"Security": DEBSECURITYURL}) + if os.environ["EXTRAREPOS"]: + outMirrorDict.update({"Extras": EXTRASURL}) return outMirrorDict