Debain-repos/docker/mainDocker.py

import requests, schedule, time, os
from bs4 import BeautifulSoup
from whatDomain import *


if os.environ["DEBMIRRORURL"]:
    DEBMIRRORURL = os.environ["DEBMIRRORURL"]
else:
    DEBMIRRORURL = "https://www.debian.org/mirror/list"

IPv4FILE = "./MirrorListV4"
IPv6FILE = "./MirrorListV6"


# Define EU and American countries
target_countries = set([
    # Europe
    "Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark",
    "Estonia", "Finland", "France", "Germany", "Greece", "Hungary", "Iceland",
    "Ireland", "Italy", "Latvia", "Lithuania", "Netherlands", "Norway", "Poland",
    "Portugal", "Romania", "Slovakia", "Slovenia", "Spain", "Sweden", "Switzerland",
    "United Kingdom", "Moldova",
    # America
    "Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
    "Mexico", "Peru", "United States", "Uruguay", "Venezuela"
])


def sanitizeURL(inpurl: str):
    if "https://" in inpurl:
        outurl = inpurl[8:]
    elif "http://" in inpurl:
        outurl = inpurl[7:]
    elif "http://" or "https://" not in url:
        outurl = inpurl
    else:
        return -1

    i = 0
    for char in outurl:
        i += 1
        if char == "/":
            outurl = outurl[:i]

    if char == "/":
        outurl = outurl[:-1]
    return outurl


def getFreshData():
    payload = requests.get(DEBMIRRORURL)
    soup = BeautifulSoup(payload.content, "html.parser")
    return soup


def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):

    outMirrorDict = {}
    current_country = None

    # Iterate through all table rows
    for table in SoupInput.find_all("table"):
        for row in table.find_all("tr"):
            # Check for country name in a full-row header (<strong><big>)
            strong = row.find("strong")
            if strong:
                country_name = strong.get_text(strip=True)
                if country_name in target_countries:
                    current_country = country_name
                else:
                    current_country = None
                continue  # move to next row

            # Check for inline country name in first column
            cols = row.find_all("td")
            if len(cols) >= 2:
                possible_country = cols[0].get_text(strip=True)
                link_tag = cols[1].find("a", href=True)
                if possible_country in target_countries:
                    current_country = possible_country
                if current_country and link_tag:
                    url = link_tag['href']
                    if current_country not in outMirrorDict:
                        outMirrorDict[current_country] = []
                    outMirrorDict[current_country].append(url)

    return outMirrorDict

def LeJob():

    print("Starting lookup")

    LeSoup = getFreshData()

    LeMirrorDict = sanitizeUrlsGodWhatTheFuckIsThis(LeSoup)

    # print(LeMirrorDict)

    with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:

        for key, urls in LeMirrorDict.items():
            # print(urls)
            if key in target_countries:
                
                for url in urls:
                    # print(url)
                    
                    if url not in fR:

                        goodurl = sanitizeURL(url)
                        # print(goodurl)

                        ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)

                        if ip4Dict == -1:
                            continue

                        for key, ip in ip4Dict.items():
                            # print(ip)

                            fW.write(ip + "/32" +  "\n")
                        

    with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:

        for key, urls in LeMirrorDict.items():
            if key in target_countries:
                
                for url in urls:
                    
                    if url not in fR:

                        goodurl = sanitizeURL(url)
                        # print(goodurl)

                        ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)

                        if ip6Dict == -1:
                            continue

                        for key, ip in ip6Dict.items():
                            # print(ip)

                            fW.write(ip + "/128" + "\n")



LeJob()