Forgot to make docker folder visible. bruh
This commit is contained in:
parent
291e0a4949
commit
0d5db658ef
2
.gitignore
vendored
2
.gitignore
vendored
@ -1,2 +1,2 @@
|
|||||||
venv
|
venv
|
||||||
docker
|
# docker
|
0
docker/.env
Normal file
0
docker/.env
Normal file
27
docker/Dockerfile
Normal file
27
docker/Dockerfile
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
FROM alpine:latest
|
||||||
|
|
||||||
|
# https://docs.docker.com/reference/dockerfile/#environment-replacement
|
||||||
|
ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
|
||||||
|
|
||||||
|
RUN apk update && \
|
||||||
|
apk add python3 py3-pip git
|
||||||
|
# inotify-tools
|
||||||
|
|
||||||
|
RUN mkdir /etc/debmirror
|
||||||
|
WORKDIR /etc/debmirror/
|
||||||
|
|
||||||
|
COPY ./mainDocker.py /etc/debmirror/
|
||||||
|
COPY ./requirements.txt /etc/debmirror/
|
||||||
|
|
||||||
|
RUN python3 -m venv venv && \
|
||||||
|
venv/bin/python3 -m pip install --upgrade pip && \
|
||||||
|
venv/bin/pip3 install -r requirements.txt
|
||||||
|
|
||||||
|
COPY ./cron-jobs /etc/crontabs/
|
||||||
|
COPY ./gitPush.sh /etc/crontabs/
|
||||||
|
|
||||||
|
RUN chmod +x /etc/crontabs/gitPush.sh
|
||||||
|
|
||||||
|
RUN git clone https://tea.shupogaki.org/YuruC3/Debain-repos
|
||||||
|
|
||||||
|
CMD ["/usr/sbin/crond"]
|
6
docker/cron-jobs
Normal file
6
docker/cron-jobs
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
# python
|
||||||
|
25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||||
|
|
||||||
|
# git push
|
||||||
|
|
||||||
|
30 19 * * * /bin/sh /etc/debmirror/gitPush.sh
|
11
docker/docker-compose.yml
Normal file
11
docker/docker-compose.yml
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
---
|
||||||
|
|
||||||
|
services:
|
||||||
|
debmirup:
|
||||||
|
container_name: Debian_Mirrors_Updater
|
||||||
|
build: ./
|
||||||
|
image: DebMirrorLists:V1
|
||||||
|
environment:
|
||||||
|
- DEBMIRRORURL=https://www.debian.org/mirror/list
|
||||||
|
restart: unless-stopped
|
||||||
|
privileged: false
|
12
docker/gitPush.sh
Normal file
12
docker/gitPush.sh
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
GITPATH="/etc/debmirror/"
|
||||||
|
|
||||||
|
# https://github.com/mujtaba1747/git-autocommit/tree/master
|
||||||
|
|
||||||
|
inotifywait --recursive -qq -e modify,close_write $GITPATH
|
||||||
|
cd $GITPATH &> /dev/null
|
||||||
|
git pull &> /dev/null
|
||||||
|
git add --all &> /dev/null
|
||||||
|
now=$(date) &> /dev/null
|
||||||
|
git commit -m "Auto-Commit at: $now" &> /dev/null
|
||||||
|
git push -u origin main &> /dev/null
|
147
docker/mainDocker.py
Normal file
147
docker/mainDocker.py
Normal file
@ -0,0 +1,147 @@
|
|||||||
|
import requests, schedule, time, os
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from whatDomain import *
|
||||||
|
|
||||||
|
|
||||||
|
if os.environ["DEBMIRRORURL"]:
|
||||||
|
DEBMIRRORURL = str(os.environ["DEBMIRRORURL"])
|
||||||
|
else:
|
||||||
|
DEBMIRRORURL = "https://www.debian.org/mirror/list"
|
||||||
|
|
||||||
|
IPv4FILE = "./MirrorListV4"
|
||||||
|
IPv6FILE = "./MirrorListV6"
|
||||||
|
|
||||||
|
|
||||||
|
# Define EU and American countries
|
||||||
|
target_countries = set([
|
||||||
|
# Europe
|
||||||
|
"Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark",
|
||||||
|
"Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland",
|
||||||
|
"Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland",
|
||||||
|
"Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland",
|
||||||
|
"United Kingdom", "Moldova",
|
||||||
|
# America
|
||||||
|
"Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
|
||||||
|
"Mexico", "Peru", "United States", "Uruguay", "Venezuela"
|
||||||
|
])
|
||||||
|
|
||||||
|
|
||||||
|
def sanitizeURL(inpurl: str):
|
||||||
|
if "https://" in inpurl:
|
||||||
|
outurl = inpurl[8:]
|
||||||
|
elif "http://" in inpurl:
|
||||||
|
outurl = inpurl[7:]
|
||||||
|
elif "http://" or "https://" not in url:
|
||||||
|
outurl = inpurl
|
||||||
|
else:
|
||||||
|
return -1
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
for char in outurl:
|
||||||
|
i += 1
|
||||||
|
if char == "/":
|
||||||
|
outurl = outurl[:i]
|
||||||
|
|
||||||
|
if char == "/":
|
||||||
|
outurl = outurl[:-1]
|
||||||
|
return outurl
|
||||||
|
|
||||||
|
|
||||||
|
def getFreshData():
|
||||||
|
payload = requests.get(DEBMIRRORURL)
|
||||||
|
soup = BeautifulSoup(payload.content, "html.parser")
|
||||||
|
return soup
|
||||||
|
|
||||||
|
|
||||||
|
def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
|
||||||
|
|
||||||
|
outMirrorDict = {}
|
||||||
|
current_country = None
|
||||||
|
|
||||||
|
# Iterate through all table rows
|
||||||
|
for table in SoupInput.find_all("table"):
|
||||||
|
for row in table.find_all("tr"):
|
||||||
|
# Check for country name in a full-row header (<strong><big>)
|
||||||
|
strong = row.find("strong")
|
||||||
|
if strong:
|
||||||
|
country_name = strong.get_text(strip=True)
|
||||||
|
if country_name in target_countries:
|
||||||
|
current_country = country_name
|
||||||
|
else:
|
||||||
|
current_country = None
|
||||||
|
continue # move to next row
|
||||||
|
|
||||||
|
# Check for inline country name in first column
|
||||||
|
cols = row.find_all("td")
|
||||||
|
if len(cols) >= 2:
|
||||||
|
possible_country = cols[0].get_text(strip=True)
|
||||||
|
link_tag = cols[1].find("a", href=True)
|
||||||
|
if possible_country in target_countries:
|
||||||
|
current_country = possible_country
|
||||||
|
if current_country and link_tag:
|
||||||
|
url = link_tag['href']
|
||||||
|
if current_country not in outMirrorDict:
|
||||||
|
outMirrorDict[current_country] = []
|
||||||
|
outMirrorDict[current_country].append(url)
|
||||||
|
|
||||||
|
return outMirrorDict
|
||||||
|
|
||||||
|
def LeJob():
|
||||||
|
|
||||||
|
print("Starting lookup")
|
||||||
|
|
||||||
|
LeSoup = getFreshData()
|
||||||
|
|
||||||
|
LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)
|
||||||
|
|
||||||
|
# print(LeMirrorDict)
|
||||||
|
|
||||||
|
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
|
||||||
|
|
||||||
|
for key, urls in LeMirrorDict.items():
|
||||||
|
# print(urls)
|
||||||
|
if key in target_countries:
|
||||||
|
|
||||||
|
for url in urls:
|
||||||
|
# print(url)
|
||||||
|
|
||||||
|
if url not in fR:
|
||||||
|
|
||||||
|
goodurl = sanitizeURL(url)
|
||||||
|
# print(goodurl)
|
||||||
|
|
||||||
|
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
|
||||||
|
|
||||||
|
if ip4Dict == -1:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for key, ip in ip4Dict.items():
|
||||||
|
# print(ip)
|
||||||
|
|
||||||
|
fW.write(ip + "/32" + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
|
||||||
|
|
||||||
|
for key, urls in LeMirrorDict.items():
|
||||||
|
if key in target_countries:
|
||||||
|
|
||||||
|
for url in urls:
|
||||||
|
|
||||||
|
if url not in fR:
|
||||||
|
|
||||||
|
goodurl = sanitizeURL(url)
|
||||||
|
# print(goodurl)
|
||||||
|
|
||||||
|
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
|
||||||
|
|
||||||
|
if ip6Dict == -1:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for key, ip in ip6Dict.items():
|
||||||
|
# print(ip)
|
||||||
|
|
||||||
|
fW.write(ip + "/128" + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
LeJob()
|
3
docker/requirements.txt
Normal file
3
docker/requirements.txt
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
beautifulsoup4==4.13.4
|
||||||
|
requests==2.32.3
|
||||||
|
schedule==1.2.2
|
Loading…
x
Reference in New Issue
Block a user