From 0d5db658ef280f402204c30d7ffa667aaa6c8f3e Mon Sep 17 00:00:00 2001 From: YuruC3 <98943911+YuruC3@@users.noreply.github.com> Date: Sun, 11 May 2025 14:06:58 +0200 Subject: [PATCH] Forgot to make docker folder visible. bruh --- .gitignore | 2 +- docker/.env | 0 docker/Dockerfile | 27 +++++++ docker/cron-jobs | 6 ++ docker/docker-compose.yml | 11 +++ docker/gitPush.sh | 12 ++++ docker/mainDocker.py | 147 ++++++++++++++++++++++++++++++++++++++ docker/requirements.txt | 3 + 8 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 docker/.env create mode 100644 docker/Dockerfile create mode 100644 docker/cron-jobs create mode 100644 docker/docker-compose.yml create mode 100644 docker/gitPush.sh create mode 100644 docker/mainDocker.py create mode 100644 docker/requirements.txt diff --git a/.gitignore b/.gitignore index 90169eb..66f8dc0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ venv -docker \ No newline at end of file +# docker \ No newline at end of file diff --git a/docker/.env b/docker/.env new file mode 100644 index 0000000..e69de29 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..22adabb --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,27 @@ +FROM alpine:latest + +# https://docs.docker.com/reference/dockerfile/#environment-replacement +ENV DEBMIRRORURL="https://www.debian.org/mirror/list" + +RUN apk update && \ + apk add python3 py3-pip git +# inotify-tools + +RUN mkdir /etc/debmirror +WORKDIR /etc/debmirror/ + +COPY ./mainDocker.py /etc/debmirror/ +COPY ./requirements.txt /etc/debmirror/ + +RUN python3 -m venv venv && \ + venv/bin/python3 -m pip install --upgrade pip && \ + venv/bin/pip3 install -r requirements.txt + +COPY ./cron-jobs /etc/crontabs/ +COPY ./gitPush.sh /etc/crontabs/ + +RUN chmod +x /etc/crontabs/gitPush.sh + +RUN git clone https://tea.shupogaki.org/YuruC3/Debain-repos + +CMD ["/usr/sbin/crond"] diff --git a/docker/cron-jobs b/docker/cron-jobs new file mode 100644 index 0000000..aaeaf18 --- /dev/null +++ b/docker/cron-jobs @@ -0,0 +1,6 @@ +# python +25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py + +# git push + +30 19 * * * /bin/sh /etc/debmirror/gitPush.sh \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..7c3de75 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,11 @@ +--- + +services: + debmirup: + container_name: Debian_Mirrors_Updater + build: ./ + image: DebMirrorLists:V1 + environment: + - DEBMIRRORURL=https://www.debian.org/mirror/list + restart: unless-stopped + privileged: false diff --git a/docker/gitPush.sh b/docker/gitPush.sh new file mode 100644 index 0000000..6e94aae --- /dev/null +++ b/docker/gitPush.sh @@ -0,0 +1,12 @@ +#!/bin/sh +GITPATH="/etc/debmirror/" + +# https://github.com/mujtaba1747/git-autocommit/tree/master + +inotifywait --recursive -qq -e modify,close_write $GITPATH +cd $GITPATH &> /dev/null +git pull &> /dev/null +git add --all &> /dev/null +now=$(date) &> /dev/null +git commit -m "Auto-Commit at: $now" &> /dev/null +git push -u origin main &> /dev/null diff --git a/docker/mainDocker.py b/docker/mainDocker.py new file mode 100644 index 0000000..4e6b7f8 --- /dev/null +++ b/docker/mainDocker.py @@ -0,0 +1,147 @@ +import requests, schedule, time, os +from bs4 import BeautifulSoup +from whatDomain import * + + +if os.environ["DEBMIRRORURL"]: + DEBMIRRORURL = str(os.environ["DEBMIRRORURL"]) +else: + DEBMIRRORURL = "https://www.debian.org/mirror/list" + +IPv4FILE = "./MirrorListV4" +IPv6FILE = "./MirrorListV6" + + +# Define EU and American countries +target_countries = set([ + # Europe + "Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark", + "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland", + "Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland", + "Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland", + "United Kingdom", "Moldova", + # America + "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador", + "Mexico", "Peru", "United States", "Uruguay", "Venezuela" +]) + + +def sanitizeURL(inpurl: str): + if "https://" in inpurl: + outurl = inpurl[8:] + elif "http://" in inpurl: + outurl = inpurl[7:] + elif "http://" or "https://" not in url: + outurl = inpurl + else: + return -1 + + i = 0 + for char in outurl: + i += 1 + if char == "/": + outurl = outurl[:i] + + if char == "/": + outurl = outurl[:-1] + return outurl + + +def getFreshData(): + payload = requests.get(DEBMIRRORURL) + soup = BeautifulSoup(payload.content, "html.parser") + return soup + + +def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup): + + outMirrorDict = {} + current_country = None + + # Iterate through all table rows + for table in SoupInput.find_all("table"): + for row in table.find_all("tr"): + # Check for country name in a full-row header () + strong = row.find("strong") + if strong: + country_name = strong.get_text(strip=True) + if country_name in target_countries: + current_country = country_name + else: + current_country = None + continue # move to next row + + # Check for inline country name in first column + cols = row.find_all("td") + if len(cols) >= 2: + possible_country = cols[0].get_text(strip=True) + link_tag = cols[1].find("a", href=True) + if possible_country in target_countries: + current_country = possible_country + if current_country and link_tag: + url = link_tag['href'] + if current_country not in outMirrorDict: + outMirrorDict[current_country] = [] + outMirrorDict[current_country].append(url) + + return outMirrorDict + +def LeJob(): + + print("Starting lookup") + + LeSoup = getFreshData() + + LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup) + + # print(LeMirrorDict) + + with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW: + + for key, urls in LeMirrorDict.items(): + # print(urls) + if key in target_countries: + + for url in urls: + # print(url) + + if url not in fR: + + goodurl = sanitizeURL(url) + # print(goodurl) + + ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl) + + if ip4Dict == -1: + continue + + for key, ip in ip4Dict.items(): + # print(ip) + + fW.write(ip + "/32" + "\n") + + + with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW: + + for key, urls in LeMirrorDict.items(): + if key in target_countries: + + for url in urls: + + if url not in fR: + + goodurl = sanitizeURL(url) + # print(goodurl) + + ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl) + + if ip6Dict == -1: + continue + + for key, ip in ip6Dict.items(): + # print(ip) + + fW.write(ip + "/128" + "\n") + + +LeJob() diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 0000000..e779096 --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1,3 @@ +beautifulsoup4==4.13.4 +requests==2.32.3 +schedule==1.2.2 \ No newline at end of file