Some more changes.

docker/Dockerfile

@@ -14,11 +14,15 @@ WORKDIR /etc/debmirror/
 
 RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \
     ln -s /etc/debmirror/Repo-IP-lists/MirrorListV4 /etc/debmirror/MirrorListV4 && \
-    ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6
+    ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6 && \
+    ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV4 /etc/debmirror/OPNS_MirrorListV4 && \
+    ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV6 /etc/debmirror/OPNS_MirrorListV6
+
 # RUN touch /etc/debmirror/MirrorListV6 && \
 #     touch /etc/debmirror/MirrorListV4
 
 COPY mainDocker.py /etc/debmirror/
+COPY mainOPNsense.py /etc/debmirror/
 COPY whatDomain.py /etc/debmirror/
 COPY requirements.txt /etc/debmirror/
 

docker/cron-jobs

@@ -1,7 +1,8 @@
 # python
 # 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
-*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py 
+#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py 
 
 # git push 
 # 30 19 * * * /bin/sh /etc/debmirror/gitPush.sh
-*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh
+#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh

docker/gitPush.sh

@@ -20,8 +20,11 @@ git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
 git config user.name "UpdateBot"
 git config user.email "UpdateBot@localhost.local"
 
+# stage files
+git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
+
 # If anything to commit locally, commit it now
-if ! git diff --quiet; then
+if ! git diff --quiet --cached; then
     echo "[$(date)] Committing local changes before pulling"
     git commit -m "Auto-commit before pull on $(date -Iseconds)"
 fi
@@ -29,9 +32,6 @@ fi
 # Now pull the latest
 git pull --rebase --autostash
 
-# stage files
-git add MirrorListV4 MirrorListV6
-
 # Commit and push only if there's anything new staged
 if git diff --quiet; then
     echo "[$(date)] No changes to commit."

docker/mainOPNsense.py

@@ -0,0 +1,135 @@
+import requests, schedule, time
+from bs4 import BeautifulSoup
+from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
+
+
+
+OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing"
+IPv4FILE = "/etc/debmirror/OPNS_MirrorListV4"
+IPv6FILE = "/etc/debmirror/OPNS_MirrorListV6"
+
+
+def sanitizeURL(inpurl: str):
+    if not "/" in inpurl[:-1]:
+        inpurl += "/" 
+    if "https://" in inpurl:
+        outurl = inpurl[8:]
+    elif "http://" in inpurl:
+        outurl = inpurl[7:]
+    elif "http://" or "https://" not in url:
+        outurl = inpurl
+    else:
+        return -1
+
+    # how the fuck does it work?
+    # I mean I wrote this but I don't know why does it work.
+    i = 0
+    for char in outurl:
+        i += 1
+        if char == "/":
+            outurl = outurl[:i]
+
+    if char == "/":
+        outurl = outurl[:-1]
+    return outurl
+
+
+def getFreshData():
+    payload = requests.get(OPNSNSMIRRORURL)
+    soup = BeautifulSoup(payload.content, "html.parser")
+    return soup
+
+def LeJob():
+
+    print("Starting lookup")
+
+    LeSoup = getFreshData()
+
+    # print(LeMirrorDict)
+
+    with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
+
+        for data in LeSoup.find_all('div', class_='download_section'):
+            for a in data.find_all('a', href=True):
+
+                url = a['href']
+
+                saniturl = sanitizeURL(url)
+
+                # print(saniturl)
+                IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
+
+                # print(IPv4Dict)
+                try:
+                    for key, ip in IPv4Dict.items():
+                        print(f"Found the ipv4: {ip}")
+                        fW.write(ip + "/32" +  "\n")
+
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue 
+
+    with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
+
+        for data in LeSoup.find_all('div', class_='download_section'):
+            for a in data.find_all('a', href=True):
+                
+                url = a['href']
+
+                saniturl = sanitizeURL(url)
+
+                # print(saniturl)
+                IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
+
+                # print(IPv6Dict)
+                try:
+                    for key, ip in IPv6Dict.items():
+                        print(f"Found the ipv6: {ip}")
+                        fW.write(ip + "/128" + "\n")
+
+                # If int returned from WhatDomain then it is error.
+                # Error type is described via prints from whatdomain functions
+                except AttributeError:
+                    continue
+
+# schedule.every().day.at("12:45").do(LeJob)
+# schedule.every().day.at("17:44").do(LeJob)
+
+# while True:
+#     schedule.run_pending()
+#     print("Waiting...")
+#     time.sleep(30)  #Wait one minute
+#     # LeJob()
+
+LeJob()
+
+# gigalist = []
+
+# payload = requests.get(OPNSNSMIRRORURL)
+# soup = BeautifulSoup(payload.content, "html.parser")
+
+
+# for data in soup.find_all('div', class_='download_section'):
+#     for a in data.find_all('a', href=True):
+
+#         url = a['href']
+
+#         saniturl = sanitizeURL(url)
+
+#         # print(saniturl)
+#         IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
+#         IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
+
+#         # print(IPv4Dict)
+#         try:
+#             for key, ip in IPv4Dict.items():
+#                 print(f"Found the ipv4: {ip}")
+
+#             for key, ip in IPv6Dict.items():
+#                 print(f"Found the ipv6: {ip}")
+#         # If int returned from WhatDomain then it is error.
+#         # Error type is described via prints from whatdomain functions
+#         except AttributeError:
+#             continue
+