V2.0.1 aka it's working now

README.md

@@ -4,8 +4,26 @@ It scrapes URL's of debian mirrors for later use in opnsense as an alias that on
 
 I'll be trying to maintain up to date list, aka just run a docker container and forget about it propably.
 
-Also includes IPs for security.debian.org 
+Also includes IPs for security.debian.org, docker, proxmox and nvidia container toolkit 
+
+## Docker
+
+Here is a rundown of all the options in docker-compose
+
+```DEBMIRRORURL``` is a link to Debian's mirror list. You don't have to fill it as I don't think they'll be changing that. I hope so at least.
+
+```GITURLPROTO``` protocol of git server. Usually https
+
+```GITURL``` address of git server. Fox example, github.com
+
+```GITREPOPATH``` is the path to your repository. For example, YuruC3/pmpw (shameless plug for my other project)
+
+```GITEA_TOKEN``` is a personal token used by git to authenticate. 
+
+```EXTRAREPOS``` can be True of False. It includes IP's for security updates for debian. Pretty usefull.
+
+```SECURITYREPOS``` has IPs for things like docker, the whole proxmox suite and nvidia container toolkit.
 
 ## Run at home 
 
-if you want to run it at your place remember to change crontab timings and git personal token.
+if you want to run it at your place remember to change crontab timings and git personal token and so on.

code/main.py

@@ -4,6 +4,15 @@ from whatDomain import *
 
 
 DEBSECURITYURL = ["https://security.debian.org/debian-security/"]
+EXTRASURL = ["https://download.docker.com/linux/debian/", 
+            # Double just to be sure. Even though they point to the same IP
+            "http://download.proxmox.com/debian/", 
+            "https://enterprise.proxmox.com/debian/pve/",
+            # That's for nvidia docker toolkit something something
+            # stuff that makes containers use nvenc and shit
+            "https://nvidia.github.io/libnvidia-container/stable/deb/",
+            "https://nvidia.github.io/libnvidia-container/experimental/deb/"]
+
 DEBMIRRORURL = "https://www.debian.org/mirror/list"
 IPv4FILE = "../MirrorListV4"
 IPv6FILE = "../MirrorListV6"
@@ -21,7 +30,7 @@ target_countries = set([
     "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
     "Mexico", "Peru", "United States", "Uruguay", "Venezuela",
     # Others
-    "Security"
+    "Security", "Extras"
 ])
 
 
@@ -84,6 +93,7 @@ def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
                     outMirrorDict[current_country].append(url)
 
     outMirrorDict.update({"Security": DEBSECURITYURL})
+    outMirrorDict.update({"Extras": EXTRASURL})
 
     return outMirrorDict
 

docker/.gitignore

@@ -1,3 +1,3 @@
 repo
 __pycache__
-/.env
+.env

docker/Dockerfile

@@ -2,6 +2,8 @@ FROM alpine:latest
 
 # https://docs.docker.com/reference/dockerfile/#environment-replacement
 ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
+ENV EXTRAREPOS=True
+ENV SECURITYREPOS=True
 
 RUN apk update && \
     apk add python3 py3-pip git  

docker/docker-compose.yml

@@ -4,9 +4,13 @@ services:
   debmirup:
     container_name: Debian_Mirrors_Updater
     build: ./
-    image: debmirrorupd:V1.1
+    image: debmirrorupd:V2.0.1
     environment:
       - DEBMIRRORURL=https://www.debian.org/mirror/list
-      - GITEA_TOKEN=<CHANGE_ME>
+      - GITURLPROTO=https
+      - GITURL=tea.shupogaki.org
+      - GITREPOPATH=YuruC3/Debain-repos.git
+      - GITEA_TOKEN=<CHANGHE_ME>
+      - EXTRAREPOS=True
+      - SECURITYREPOS=True
     restart: unless-stopped
-    privileged: false

docker/gitPush.sh

@@ -3,7 +3,7 @@ set -e
 
 WORKPTH="/etc/debmirror/"
 REPO_DIR="$WORKPTH/Debain-repos"
-REPO_URL="https://tea.shupogaki.org/YuruC3/Debain-repos.git"
+REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
 
 
 # Clone repo only if it doesn't already exist
@@ -15,13 +15,19 @@ fi
 
 cd "$REPO_DIR"
 
-git remote set-url origin "https://${GITEA_TOKEN}@tea.shupogaki.org/YuruC3/Debain-repos.git"
+git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
 
 git config user.name "UpdateBot"
 git config user.email "UpdateBot@localhost.local"
 
-# pull changes
+# If anything to commit locally, commit it now
-git pull 
+if ! git diff --cached --quiet; then
+    echo "[$(date)] Committing local changes before pulling"
+    git commit -m "Auto-commit before pull on $(date -Iseconds)"
+fi
+
+# Now pull the latest
+git pull --rebase --autostash
 
 # stage files
 git add MirrorListV4 MirrorListV6

docker/mainDocker.py

@@ -9,6 +9,15 @@ else:
     DEBMIRRORURL = "https://www.debian.org/mirror/list"
 
 DEBSECURITYURL = ["https://security.debian.org/debian-security/"]
+EXTRASURL = ["https://download.docker.com/linux/debian/", 
+            # Double just to be sure. Even though they point to the same IP
+            "http://download.proxmox.com/debian/", 
+            "https://enterprise.proxmox.com/debian/pve/",
+            # That's for nvidia docker toolkit something something
+            # stuff that makes containers use nvenc and shit
+            "https://nvidia.github.io/libnvidia-container/stable/deb/",
+            "https://nvidia.github.io/libnvidia-container/experimental/deb/"]
+
 IPv4FILE = "/etc/debmirror/MirrorListV4"
 IPv6FILE = "/etc/debmirror/MirrorListV6"
 
@@ -25,7 +34,7 @@ target_countries = set([
     "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
     "Mexico", "Peru", "United States", "Uruguay", "Venezuela",
     # Others
-    "Security"
+    "Security", "Extras"
 ])
 
 
@@ -87,7 +96,10 @@ def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
                         outMirrorDict[current_country] = []
                     outMirrorDict[current_country].append(url)
 
-    outMirrorDict.update({"Security": DEBSECURITYURL})
+    if os.environ["SECURITYREPOS"]:
+        outMirrorDict.update({"Security": DEBSECURITYURL})
+    if os.environ["EXTRAREPOS"]:
+        outMirrorDict.update({"Extras": EXTRASURL})
 
     return outMirrorDict