Compare commits
61 Commits
a945c3c78b
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 880da3b79d | |||
| 61e7560952 | |||
| 56ac33b666 | |||
| d2838646b3 | |||
| 8c61f4a51c | |||
|
|
75cb35b954 | ||
|
|
f572e9f10e | ||
|
|
6fde3dd9be | ||
|
|
a3842d9dc7 | ||
|
|
f82a8f5f39 | ||
|
|
daa6a5f9d5 | ||
|
|
29598232b4 | ||
|
|
01c11c986d | ||
|
|
af47dc5233 | ||
|
|
6aed390cf7 | ||
|
|
bc781217e2 | ||
|
|
9724ffdf9c | ||
|
|
cec5575ff2 | ||
|
|
b977ddb47a | ||
| 82668dba46 | |||
|
|
17423793f1 | ||
|
|
6beadaba5f | ||
|
|
8a5375c5c8 | ||
|
|
c265a83300 | ||
|
|
02d5484903 | ||
|
|
5a7a22bfe7 | ||
|
|
7ac5a1e420 | ||
|
|
33c63463cb | ||
|
|
a117ddd1e3 | ||
|
|
83acfacc89 | ||
|
|
d55ed33c9b | ||
|
|
317048dd59 | ||
|
|
5e9a80be07 | ||
|
|
74bae2d70d | ||
|
|
2d1bb18d6b | ||
|
|
6c7f0f7dc6 | ||
|
|
032c82c37a | ||
|
|
bd3a1ea817 | ||
|
|
e9f9c5e90d | ||
|
|
6718fb0cd0 | ||
|
|
b9ede7b9a4 | ||
|
|
a56cf99c17 | ||
|
|
04c4f4f878 | ||
|
|
a95eec3ff9 | ||
|
|
d7c4812310 | ||
|
|
d6d288da33 | ||
|
|
ad03f87096 | ||
|
|
90b24875cf | ||
|
|
099f6f3b43 | ||
|
|
fdf5bf8cc8 | ||
|
|
dab7ab6974 | ||
|
|
82bc1d5910 | ||
|
|
b6c0bcabdf | ||
| 6bf0b209f2 | |||
|
|
ff6b92703f | ||
|
|
b038e81de5 | ||
|
|
f3a28cb10f | ||
|
|
32052f5cd5 | ||
|
|
19d7cd3d58 | ||
|
|
a97da67111 | ||
|
|
4dcc7cc640 |
56
.gitea/workflows/builddockerImage.yaml
Normal file
56
.gitea/workflows/builddockerImage.yaml
Normal file
@@ -0,0 +1,56 @@
|
||||
# This workflow uses actions that are not certified by GitHub.
|
||||
# They are provided by a third-party and are governed by
|
||||
# separate terms of service, privacy policy, and support
|
||||
# documentation.
|
||||
|
||||
# GitHub recommends pinning actions to a commit SHA.
|
||||
# To get a newer version, you will need to update the SHA.
|
||||
# You can also reference a tag or branch, but the action may change without warning.
|
||||
|
||||
name: Publish Docker image
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
jobs:
|
||||
push_to_registry:
|
||||
name: Push Docker image to Docker Hub
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
attestations: write
|
||||
id-token: write
|
||||
steps:
|
||||
- name: Check out the repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Extract metadata (tags, labels) for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
|
||||
with:
|
||||
images: homelab/debianrepochecker
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: push
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
|
||||
with:
|
||||
context: .
|
||||
file: ./../../docker/Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
- name: Generate artifact attestation
|
||||
uses: actions/attest-build-provenance@v2
|
||||
with:
|
||||
subject-name: index.docker.io/my-docker-hub-namespace/my-docker-hub-repository
|
||||
subject-digest: ${{ steps.push.outputs.digest }}
|
||||
push-to-registry: true
|
||||
66
MirrorListV4
66
MirrorListV4
@@ -64,15 +64,15 @@
|
||||
89.102.0.150/32
|
||||
195.113.250.60/32
|
||||
217.31.202.63/32
|
||||
194.71.11.165/32
|
||||
194.71.11.163/32
|
||||
194.71.11.165/32
|
||||
194.71.11.163/32
|
||||
194.71.11.165/32
|
||||
46.30.211.12/32
|
||||
130.225.254.116/32
|
||||
77.247.64.34/32
|
||||
194.71.11.165/32
|
||||
194.71.11.163/32
|
||||
194.71.11.165/32
|
||||
194.71.11.163/32
|
||||
194.71.11.165/32
|
||||
195.140.195.56/32
|
||||
@@ -119,8 +119,8 @@
|
||||
137.226.34.46/32
|
||||
80.237.136.138/32
|
||||
129.143.116.10/32
|
||||
212.201.68.60/32
|
||||
212.201.68.61/32
|
||||
212.201.68.60/32
|
||||
134.109.228.1/32
|
||||
139.174.32.6/32
|
||||
132.180.8.60/32
|
||||
@@ -148,13 +148,14 @@
|
||||
78.129.164.123/32
|
||||
185.165.170.20/32
|
||||
85.94.199.210/32
|
||||
203.178.137.175/32
|
||||
153.127.75.11/32
|
||||
203.178.137.175/32
|
||||
210.117.237.2/32
|
||||
85.94.199.210/32
|
||||
109.73.80.190/32
|
||||
|
||||
151.0.128.28/32
|
||||
193.206.214.14/32
|
||||
193.206.214.15/32
|
||||
85.94.199.210/32
|
||||
130.251.19.2/32
|
||||
140.105.48.55/32
|
||||
@@ -167,8 +168,8 @@
|
||||
178.17.160.48/32
|
||||
178.17.160.48/32
|
||||
130.89.149.21/32
|
||||
103.2.187.6/32
|
||||
103.2.187.5/32
|
||||
103.2.187.6/32
|
||||
163.7.134.112/32
|
||||
130.89.149.21/32
|
||||
130.89.149.21/32
|
||||
@@ -225,12 +226,12 @@
|
||||
82.194.78.250/32
|
||||
194.71.11.165/32
|
||||
194.71.11.163/32
|
||||
194.71.11.163/32
|
||||
194.71.11.165/32
|
||||
194.71.11.163/32
|
||||
130.235.34.30/32
|
||||
193.11.30.180/32
|
||||
194.71.11.163/32
|
||||
194.71.11.165/32
|
||||
194.71.11.163/32
|
||||
194.132.225.213/32
|
||||
94.103.202.216/32
|
||||
109.74.12.226/32
|
||||
@@ -239,16 +240,16 @@
|
||||
193.140.98.183/32
|
||||
129.132.53.171/32
|
||||
129.132.53.171/32
|
||||
188.185.29.10/32
|
||||
188.184.79.248/32
|
||||
188.184.91.52/32
|
||||
188.184.23.33/32
|
||||
188.185.8.192/32
|
||||
188.184.90.134/32
|
||||
188.184.111.122/32
|
||||
188.184.105.5/32
|
||||
188.185.29.10/32
|
||||
188.184.90.134/32
|
||||
188.185.8.192/32
|
||||
188.184.79.248/32
|
||||
188.184.23.33/32
|
||||
188.184.104.250/32
|
||||
188.184.91.52/32
|
||||
188.185.25.162/32
|
||||
188.184.111.122/32
|
||||
188.185.65.208/32
|
||||
83.166.143.42/32
|
||||
83.166.143.43/32
|
||||
@@ -264,10 +265,10 @@
|
||||
90.207.225.196/32
|
||||
94.229.65.150/32
|
||||
213.230.211.73/32
|
||||
93.93.131.18/32
|
||||
46.235.229.21/32
|
||||
163.1.160.163/32
|
||||
93.93.131.18/32
|
||||
163.1.221.67/32
|
||||
163.1.160.163/32
|
||||
80.87.134.17/32
|
||||
85.13.241.50/32
|
||||
212.219.56.184/32
|
||||
@@ -275,23 +276,23 @@
|
||||
185.28.240.10/32
|
||||
5.180.211.190/32
|
||||
64.50.236.52/32
|
||||
208.80.154.139/32
|
||||
64.50.233.100/32
|
||||
208.80.154.139/32
|
||||
208.80.154.139/32
|
||||
64.50.233.100/32
|
||||
64.50.236.52/32
|
||||
208.80.154.139/32
|
||||
92.119.16.152/32
|
||||
4.153.4.61/32
|
||||
172.179.119.208/32
|
||||
128.180.2.105/32
|
||||
128.30.3.141/32
|
||||
128.226.118.14/32
|
||||
128.226.118.13/32
|
||||
128.226.118.15/32
|
||||
128.226.118.12/32
|
||||
128.226.118.15/32
|
||||
128.226.118.14/32
|
||||
108.61.5.92/32
|
||||
64.50.236.52/32
|
||||
64.50.233.100/32
|
||||
140.211.166.134/32
|
||||
64.50.233.100/32
|
||||
128.135.10.29/32
|
||||
77.247.126.176/32
|
||||
108.86.224.23/32
|
||||
@@ -319,6 +320,7 @@
|
||||
94.154.159.241/32
|
||||
128.211.151.252/32
|
||||
138.247.115.248/32
|
||||
168.100.147.101/32
|
||||
200.14.116.3/32
|
||||
190.105.216.42/32
|
||||
45.162.204.37/32
|
||||
@@ -340,3 +342,21 @@
|
||||
176.124.104.176/32
|
||||
185.100.87.86/32
|
||||
164.73.233.2/32
|
||||
151.101.194.132/32
|
||||
151.101.130.132/32
|
||||
151.101.2.132/32
|
||||
151.101.66.132/32
|
||||
3.164.240.105/32
|
||||
3.164.240.119/32
|
||||
3.164.240.73/32
|
||||
3.164.240.123/32
|
||||
45.84.67.184/32
|
||||
45.84.67.184/32
|
||||
185.199.110.153/32
|
||||
185.199.108.153/32
|
||||
185.199.111.153/32
|
||||
185.199.109.153/32
|
||||
185.199.110.153/32
|
||||
185.199.111.153/32
|
||||
185.199.109.153/32
|
||||
185.199.108.153/32
|
||||
|
||||
68
MirrorListV6
68
MirrorListV6
@@ -50,10 +50,10 @@
|
||||
2a02:8301:0:2::150/128
|
||||
2001:718:1007:48::1:60/128
|
||||
2001:1488:ffff::63/128
|
||||
2001:6b0:19::165/128
|
||||
2001:6b0:19::163/128
|
||||
2001:6b0:19::163/128
|
||||
2001:6b0:19::165/128
|
||||
2001:6b0:19::165/128
|
||||
2001:6b0:19::163/128
|
||||
2a02:2350::211:12/128
|
||||
2001:878:346::116/128
|
||||
2a02:4de0:21::2/128
|
||||
@@ -95,14 +95,14 @@
|
||||
2a00:8a60:e012:a00::21/128
|
||||
2a01:488:10:1::50ed:888a/128
|
||||
2001:7c0:700::10/128
|
||||
2a00:5ba0:8000:e:92e2:baff:fe36:3e88/128
|
||||
2a00:5ba0:8000:e:92e2:baff:fe36:3cd0/128
|
||||
2a00:5ba0:8000:e:92e2:baff:fe36:3e88/128
|
||||
2001:638:911:b0e:134:109:228:1/128
|
||||
2001:638:605:20:1::6/128
|
||||
2001:638:208:ef1b:0:ff:fe00:4/128
|
||||
2001:4c80:40:628:215:5dff:fe01:be02/128
|
||||
2001:7c0:2041:48::42/128
|
||||
2001:7c0:2041:48::41/128
|
||||
2001:7c0:2041:48::42/128
|
||||
2003:d1:3012::400/128
|
||||
2a00:f48:1007:32::30/128
|
||||
2a07:6fc0:1:0:73::59/128
|
||||
@@ -124,7 +124,7 @@
|
||||
2001:320:237::2/128
|
||||
2001:4b78:2000:1::1/128
|
||||
2a00:1cc8:0:ff0a::2/128
|
||||
2001:760:ffff:b6:4:100:0:72/128
|
||||
2001:760:ffff:b6:4:100:0:73/128
|
||||
2001:4b78:2000:1::1/128
|
||||
2001:760:2e03::ca:26:0/128
|
||||
2a05:7cc0::91:211:244:15/128
|
||||
@@ -132,15 +132,15 @@
|
||||
2a00:1dc0::f00d/128
|
||||
2a00:1dc0::f00d/128
|
||||
2001:67c:2564:a120::21/128
|
||||
2404:e400::443:b/128
|
||||
2404:e400::443:a/128
|
||||
2404:e400::443:b/128
|
||||
2404:138:4000::/128
|
||||
2001:67c:2564:a120::21/128
|
||||
2001:67c:2564:a120::21/128
|
||||
2a14:3f87:c800:2980::2980/128
|
||||
2001:678:d64:564d::443/128
|
||||
2a01:1b0:7999:419::3/128
|
||||
2a00:1630:3400::214/128
|
||||
2a00:1630:3400::213/128
|
||||
2a00:1768:1000:22:1::1/128
|
||||
2a0f:ca80:1337::da0:c1c5/128
|
||||
2a02:6ea0:c036::10/128
|
||||
@@ -180,11 +180,11 @@
|
||||
2001:720:1c14::ffff:c192:4b13/128
|
||||
2001:6b0:19::165/128
|
||||
2001:6b0:19::163/128
|
||||
2001:6b0:19::163/128
|
||||
2001:6b0:19::165/128
|
||||
2001:6b0:19::163/128
|
||||
2001:6b0:5:50::180/128
|
||||
2001:6b0:19::163/128
|
||||
2001:6b0:19::165/128
|
||||
2001:6b0:19::163/128
|
||||
2a01:8e01:c1ea:6c5c::5/128
|
||||
2a01:8e01:c1ea:6c5c::8/128
|
||||
2a02:750:9::e8/128
|
||||
@@ -192,16 +192,16 @@
|
||||
2001:e10:2000:240:e643:4bff:fee8:a63c/128
|
||||
2001:67c:10ec:3dd1::42/128
|
||||
2001:67c:10ec:3dd1::42/128
|
||||
2001:1458:d00:67::100:429/128
|
||||
2001:1458:201:e4::100:22c/128
|
||||
2001:1458:d00:63::100:106/128
|
||||
2001:1458:d00:34::100:328/128
|
||||
2001:1458:d00:2c::100:229/128
|
||||
2001:1458:d00:6f::100:89/128
|
||||
2001:1458:201:e4::100:35c/128
|
||||
2001:1458:d00:2d::100:1cb/128
|
||||
2001:1458:d00:1::100:408/128
|
||||
2001:1458:d00:6f::100:89/128
|
||||
2001:1458:d00:63::100:106/128
|
||||
2001:1458:d00:2d::100:d4/128
|
||||
2001:1458:d00:2c::100:229/128
|
||||
2001:1458:201:e4::100:35c/128
|
||||
2001:1458:201:e4::100:22c/128
|
||||
2001:1458:d00:67::100:429/128
|
||||
2001:1458:d00:34::100:328/128
|
||||
2001:1458:d00:1::100:408/128
|
||||
2001:1458:d00:68::100:e4/128
|
||||
2001:1600:0:aaaa::2:2/128
|
||||
2001:1600:0:aaaa::2:1/128
|
||||
@@ -214,23 +214,23 @@
|
||||
2a01:550:0:402::2:1/128
|
||||
2001:1b40:5600:ff80:f8ee::1/128
|
||||
2a01:70:1:211:2::73/128
|
||||
2a00:1098:1::5/128
|
||||
2a00:1098:0:86:1000::25/128
|
||||
2a00:1098:1::5/128
|
||||
2a01:c0:2:3d::2/128
|
||||
2001:630:341:12::184/128
|
||||
2a0d:5082:0:7::2/128
|
||||
2620:0:861:2:208:80:154:139/128
|
||||
2600:3402:200:227::2/128
|
||||
2600:3404:200:237::2/128
|
||||
2600:3404:200:237::2/128
|
||||
2620:0:861:2:208:80:154:139/128
|
||||
2600:3402:200:227::2/128
|
||||
2600:3402:200:227::2/128
|
||||
2600:3404:200:237::2/128
|
||||
2620:0:861:2:208:80:154:139/128
|
||||
2a10:4dc0:2:5::2/128
|
||||
2603:1030:403:3::cee/128
|
||||
2603:1020:203:3::a/128
|
||||
2001:19f0:1595:1403::1092/128
|
||||
2600:3402:200:227::2/128
|
||||
2605:bc80:3010::134/128
|
||||
2600:3404:200:237::2/128
|
||||
2600:3402:200:227::2/128
|
||||
2a0b:f301:2:7::2/128
|
||||
2604:2dc0:200:1b57::a/128
|
||||
2604:fbc0:4::2/128
|
||||
@@ -262,3 +262,25 @@
|
||||
2001:67c:744:7::2/128
|
||||
2a06:1700:0:38::1/128
|
||||
2001:1328:54:fff::2/128
|
||||
2a04:4e42::644/128
|
||||
2a04:4e42:200::644/128
|
||||
2a04:4e42:400::644/128
|
||||
2a04:4e42:600::644/128
|
||||
2600:9000:2764:600:3:db06:4200:93a1/128
|
||||
2600:9000:2764:fa00:3:db06:4200:93a1/128
|
||||
2600:9000:2764:e800:3:db06:4200:93a1/128
|
||||
2600:9000:2764:7400:3:db06:4200:93a1/128
|
||||
2600:9000:2764:b000:3:db06:4200:93a1/128
|
||||
2600:9000:2764:4800:3:db06:4200:93a1/128
|
||||
2600:9000:2764:c600:3:db06:4200:93a1/128
|
||||
2600:9000:2764:2e00:3:db06:4200:93a1/128
|
||||
2607:5300:400:7d00::80/128
|
||||
2607:5300:400:7d00::80/128
|
||||
2606:50c0:8003::153/128
|
||||
2606:50c0:8000::153/128
|
||||
2606:50c0:8001::153/128
|
||||
2606:50c0:8002::153/128
|
||||
2606:50c0:8002::153/128
|
||||
2606:50c0:8000::153/128
|
||||
2606:50c0:8003::153/128
|
||||
2606:50c0:8001::153/128
|
||||
|
||||
97
OPNsense_aliases.json
Normal file
97
OPNsense_aliases.json
Normal file
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"aliases": {
|
||||
"alias": {
|
||||
|
||||
"19a5bc81-a08d-4b1e-9368-cd7c5a92a447": {
|
||||
"enabled": "1",
|
||||
"name": "RFC1918",
|
||||
"type": "network",
|
||||
"path_expression": "",
|
||||
"proto": "",
|
||||
"interface": "",
|
||||
"counters": "0",
|
||||
"updatefreq": "",
|
||||
"content": "10.0.0.0/8\n172.16.0.0/12\n192.168.0.0/16",
|
||||
"password": "",
|
||||
"username": "",
|
||||
"authtype": "",
|
||||
"categories": "",
|
||||
"current_items": "3",
|
||||
"last_updated": "2024-11-07T15:05:47.481822",
|
||||
"description": ""
|
||||
},
|
||||
"b5c3e490-2222-4e5a-98aa-fb3cf07d1fff": {
|
||||
"enabled": "1",
|
||||
"name": "CloudflareV4",
|
||||
"type": "urltable",
|
||||
"path_expression": "",
|
||||
"proto": "",
|
||||
"interface": "",
|
||||
"counters": "0",
|
||||
"updatefreq": "1",
|
||||
"content": "https://www.cloudflare.com/ips-v4/",
|
||||
"password": "",
|
||||
"username": "",
|
||||
"authtype": "",
|
||||
"categories": "",
|
||||
"current_items": "15",
|
||||
"last_updated": "2024-11-07T18:30:34.207147",
|
||||
"description": "Cloudflare servers for proxying IPv4"
|
||||
},
|
||||
"67f72dc5-960b-437c-b774-b6bc006dc6d5": {
|
||||
"enabled": "1",
|
||||
"name": "CloudflareV6",
|
||||
"type": "urltable",
|
||||
"path_expression": "",
|
||||
"proto": "",
|
||||
"interface": "",
|
||||
"counters": "0",
|
||||
"updatefreq": "1",
|
||||
"content": "https://www.cloudflare.com/ips-v6/",
|
||||
"password": "",
|
||||
"username": "",
|
||||
"authtype": "",
|
||||
"categories": "",
|
||||
"current_items": "7",
|
||||
"last_updated": "2024-11-07T18:30:34.431479",
|
||||
"description": "Cloudflare servers for proxying IPv6"
|
||||
},
|
||||
"dc901506-ff81-406e-bbed-a43g009588ba": {
|
||||
"enabled": "1",
|
||||
"name": "AptUpdateV4",
|
||||
"type": "urltable",
|
||||
"path_expression": "",
|
||||
"proto": "",
|
||||
"interface": "",
|
||||
"counters": "0",
|
||||
"updatefreq": "1",
|
||||
"content": "https://tea.shupogaki.org/YuruC3/Debain-repos/raw/branch/main/MirrorListV4",
|
||||
"password": "",
|
||||
"username": "",
|
||||
"authtype": "",
|
||||
"categories": "",
|
||||
"current_items": "",
|
||||
"last_updated": "",
|
||||
"description": ""
|
||||
},
|
||||
"dc901506-ff81-406e-bbed-a43c009588ba": {
|
||||
"enabled": "1",
|
||||
"name": "AptUpdateV6",
|
||||
"type": "urltable",
|
||||
"path_expression": "",
|
||||
"proto": "",
|
||||
"interface": "",
|
||||
"counters": "0",
|
||||
"updatefreq": "1",
|
||||
"content": "https://tea.shupogaki.org/YuruC3/Debain-repos/raw/branch/main/MirrorListV6",
|
||||
"password": "",
|
||||
"username": "",
|
||||
"authtype": "",
|
||||
"categories": "",
|
||||
"current_items": "",
|
||||
"last_updated": "",
|
||||
"description": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
26
README.md
26
README.md
@@ -1,3 +1,29 @@
|
||||
# A list of EU and American Debian mirror URLs
|
||||
|
||||
It scrapes URL's of debian mirrors for later use in opnsense as an alias that ony allows access to debian repositories
|
||||
|
||||
I'll be trying to maintain up to date list, aka just run a docker container and forget about it propably.
|
||||
|
||||
Also includes IPs for security.debian.org, docker, proxmox and nvidia container toolkit
|
||||
|
||||
## Docker
|
||||
|
||||
Here is a rundown of all the options in docker-compose
|
||||
|
||||
```DEBMIRRORURL``` is a link to Debian's mirror list. You don't have to fill it as I don't think they'll be changing that. I hope so at least.
|
||||
|
||||
```GITURLPROTO``` protocol of git server. Usually https
|
||||
|
||||
```GITURL``` address of git server. Fox example, github.com
|
||||
|
||||
```GITREPOPATH``` is the path to your repository. For example, YuruC3/pmpw (shameless plug for my other project)
|
||||
|
||||
```GITEA_TOKEN``` is a personal token used by git to authenticate.
|
||||
|
||||
```EXTRAREPOS``` can be True of False. It includes IP's for security updates for debian. Pretty usefull.
|
||||
|
||||
```SECURITYREPOS``` has IPs for things like docker, the whole proxmox suite and nvidia container toolkit.
|
||||
|
||||
## Run at home
|
||||
|
||||
if you want to run it at your place remember to change crontab timings and git personal token and so on.
|
||||
346
code/main.log
Normal file
346
code/main.log
Normal file
@@ -0,0 +1,346 @@
|
||||
Waiting...
|
||||
Waiting...
|
||||
Waiting...
|
||||
Starting lookup
|
||||
213.129.232.18
|
||||
82.209.230.71
|
||||
213.129.232.18
|
||||
144.208.213.156
|
||||
83.164.133.195
|
||||
89.106.208.14
|
||||
213.129.232.18
|
||||
129.27.3.13
|
||||
46.102.157.26
|
||||
195.234.45.114
|
||||
195.234.45.114
|
||||
193.138.101.179
|
||||
193.190.198.27
|
||||
195.234.45.114
|
||||
185.111.204.146
|
||||
200.236.31.3
|
||||
200.236.31.3
|
||||
200.236.31.3
|
||||
200.237.203.89
|
||||
170.233.231.184
|
||||
143.106.7.135
|
||||
143.208.160.8
|
||||
200.133.233.210
|
||||
62.44.96.11
|
||||
62.44.96.11
|
||||
212.50.0.19
|
||||
195.85.215.251
|
||||
185.117.82.66
|
||||
84.43.191.5
|
||||
217.79.83.126
|
||||
62.44.96.11
|
||||
87.121.121.4
|
||||
78.128.80.70
|
||||
78.128.80.69
|
||||
207.210.46.249
|
||||
207.210.46.249
|
||||
199.188.220.121
|
||||
207.210.46.249
|
||||
129.97.134.71
|
||||
208.89.84.55
|
||||
99.199.64.121
|
||||
137.82.116.21
|
||||
138.204.231.181
|
||||
45.125.0.6
|
||||
138.204.231.181
|
||||
190.93.84.7
|
||||
190.97.169.220
|
||||
138.186.10.35
|
||||
138.204.231.181
|
||||
146.83.205.200
|
||||
161.53.160.11
|
||||
161.53.160.11
|
||||
161.53.160.11
|
||||
213.191.133.160
|
||||
78.128.211.127
|
||||
78.128.211.127
|
||||
81.91.81.11
|
||||
95.168.211.41
|
||||
147.32.160.17
|
||||
78.128.211.127
|
||||
147.32.127.196
|
||||
147.228.57.10
|
||||
147.229.176.19
|
||||
89.102.0.150
|
||||
195.113.250.60
|
||||
217.31.202.63
|
||||
194.71.11.165
|
||||
194.71.11.163
|
||||
194.71.11.163
|
||||
194.71.11.165
|
||||
46.30.211.12
|
||||
130.225.254.116
|
||||
77.247.64.34
|
||||
194.71.11.165
|
||||
194.71.11.163
|
||||
194.71.11.163
|
||||
194.71.11.165
|
||||
195.140.195.56
|
||||
193.166.3.3
|
||||
212.27.32.66
|
||||
212.27.32.66
|
||||
46.105.185.24
|
||||
213.32.5.7
|
||||
145.238.186.56
|
||||
193.48.57.38
|
||||
212.27.32.66
|
||||
193.55.175.161
|
||||
149.62.156.14
|
||||
82.67.51.130
|
||||
147.94.19.20
|
||||
132.227.74.17
|
||||
77.95.64.41
|
||||
194.167.156.234
|
||||
193.49.184.24
|
||||
130.79.200.5
|
||||
194.214.218.193
|
||||
80.93.84.15
|
||||
80.67.163.159
|
||||
194.254.245.145
|
||||
136.243.89.31
|
||||
62.138.222.4
|
||||
129.102.1.37
|
||||
137.226.34.46
|
||||
141.76.2.4
|
||||
45.125.0.6
|
||||
137.226.34.46
|
||||
141.76.2.4
|
||||
193.175.72.15
|
||||
141.76.2.4
|
||||
85.25.128.62
|
||||
62.201.161.86
|
||||
129.187.10.100
|
||||
194.8.197.22
|
||||
134.169.192.5
|
||||
91.199.118.172
|
||||
147.142.19.60
|
||||
131.188.12.211
|
||||
134.76.12.6
|
||||
137.226.34.46
|
||||
80.237.136.138
|
||||
129.143.116.10
|
||||
212.201.68.60
|
||||
212.201.68.61
|
||||
134.109.228.1
|
||||
139.174.32.6
|
||||
132.180.8.60
|
||||
130.75.2.22
|
||||
131.246.123.4
|
||||
134.93.178.166
|
||||
129.69.0.41
|
||||
129.69.0.42
|
||||
217.243.224.144
|
||||
212.83.32.30
|
||||
5.1.73.59
|
||||
37.58.58.140
|
||||
185.3.234.216
|
||||
217.72.206.21
|
||||
139.20.16.130
|
||||
194.29.226.88
|
||||
185.41.106.155
|
||||
82.98.66.230
|
||||
62.133.35.14
|
||||
213.202.193.253
|
||||
213.209.102.29
|
||||
194.95.104.50
|
||||
109.69.69.17
|
||||
78.129.164.123
|
||||
78.129.164.123
|
||||
185.165.170.20
|
||||
85.94.199.210
|
||||
203.178.137.175
|
||||
153.127.75.11
|
||||
210.117.237.2
|
||||
85.94.199.210
|
||||
109.73.80.190
|
||||
151.0.128.28
|
||||
90.147.160.73
|
||||
85.94.199.210
|
||||
130.251.19.2
|
||||
140.105.48.55
|
||||
158.129.159.169
|
||||
158.129.159.169
|
||||
158.129.159.169
|
||||
83.171.27.40
|
||||
91.211.244.15
|
||||
178.17.160.48
|
||||
178.17.160.48
|
||||
178.17.160.48
|
||||
130.89.149.21
|
||||
103.2.187.5
|
||||
103.2.187.6
|
||||
163.7.134.112
|
||||
130.89.149.21
|
||||
130.89.149.21
|
||||
23.151.136.174
|
||||
185.244.24.35
|
||||
83.137.149.3
|
||||
109.200.196.213
|
||||
85.159.239.121
|
||||
50.118.225.16
|
||||
185.229.191.144
|
||||
5.79.108.33
|
||||
185.15.209.238
|
||||
45.84.31.11
|
||||
78.142.193.130
|
||||
45.88.5.78
|
||||
185.183.113.131
|
||||
185.246.188.7
|
||||
194.127.172.131
|
||||
129.240.118.47
|
||||
129.240.118.47
|
||||
129.240.118.47
|
||||
153.19.251.221
|
||||
153.19.251.221
|
||||
149.156.96.11
|
||||
193.219.28.2
|
||||
150.254.173.17
|
||||
153.19.251.222
|
||||
193.136.216.34
|
||||
85.143.112.112
|
||||
193.136.216.34
|
||||
193.136.216.34
|
||||
193.137.214.36
|
||||
193.136.164.6
|
||||
23.147.168.189
|
||||
195.80.174.186
|
||||
195.80.174.186
|
||||
91.221.4.48
|
||||
88.212.10.12
|
||||
195.80.174.186
|
||||
213.129.232.18
|
||||
213.129.232.18
|
||||
130.89.148.12
|
||||
130.89.148.12
|
||||
80.64.47.13
|
||||
193.145.15.20
|
||||
95.20.11.14
|
||||
193.146.32.81
|
||||
147.83.91.172
|
||||
150.214.5.134
|
||||
193.144.61.75
|
||||
91.132.103.246
|
||||
193.146.75.19
|
||||
155.210.4.97
|
||||
82.194.78.250
|
||||
194.71.11.165
|
||||
194.71.11.163
|
||||
194.71.11.163
|
||||
194.71.11.165
|
||||
130.235.34.30
|
||||
193.11.30.180
|
||||
194.71.11.163
|
||||
194.71.11.165
|
||||
194.132.225.213
|
||||
94.103.202.216
|
||||
109.74.12.226
|
||||
129.132.53.171
|
||||
140.110.240.80
|
||||
193.140.98.183
|
||||
129.132.53.171
|
||||
129.132.53.171
|
||||
188.184.90.134
|
||||
188.184.105.5
|
||||
188.185.65.208
|
||||
188.184.104.250
|
||||
188.184.111.122
|
||||
188.185.29.10
|
||||
188.184.79.248
|
||||
188.184.23.33
|
||||
188.184.91.52
|
||||
188.185.8.192
|
||||
188.185.25.162
|
||||
83.166.143.42
|
||||
83.166.143.43
|
||||
109.202.202.202
|
||||
212.25.8.232
|
||||
185.73.240.250
|
||||
89.145.166.16
|
||||
78.129.164.123
|
||||
78.129.164.123
|
||||
83.170.94.3
|
||||
78.158.65.70
|
||||
78.129.164.123
|
||||
90.207.225.196
|
||||
94.229.65.150
|
||||
213.230.211.73
|
||||
46.235.229.21
|
||||
93.93.131.18
|
||||
163.1.160.163
|
||||
163.1.221.67
|
||||
80.87.134.17
|
||||
85.13.241.50
|
||||
212.219.56.184
|
||||
81.179.123.56
|
||||
185.28.240.10
|
||||
5.180.211.190
|
||||
64.50.233.100
|
||||
64.50.236.52
|
||||
208.80.154.139
|
||||
208.80.154.139
|
||||
64.50.233.100
|
||||
64.50.236.52
|
||||
92.119.16.152
|
||||
20.61.228.89
|
||||
128.180.2.105
|
||||
128.30.3.141
|
||||
128.226.118.14
|
||||
128.226.118.15
|
||||
128.226.118.12
|
||||
128.226.118.13
|
||||
108.61.5.92
|
||||
64.50.236.52
|
||||
140.211.166.134
|
||||
64.50.233.100
|
||||
128.135.10.29
|
||||
77.247.126.176
|
||||
108.86.224.23
|
||||
15.204.41.243
|
||||
204.157.3.70
|
||||
161.129.154.250
|
||||
184.105.240.111
|
||||
76.252.192.212
|
||||
216.127.35.166
|
||||
148.137.11.75
|
||||
192.158.27.72
|
||||
206.82.17.213
|
||||
141.219.180.70
|
||||
104.189.183.57
|
||||
162.255.118.133
|
||||
169.229.200.70
|
||||
208.100.4.53
|
||||
69.166.55.80
|
||||
208.80.154.139
|
||||
142.147.88.7
|
||||
23.145.168.14
|
||||
207.244.94.80
|
||||
185.142.32.136
|
||||
74.208.240.17
|
||||
94.154.159.241
|
||||
128.211.151.252
|
||||
138.247.115.248
|
||||
200.14.116.3
|
||||
190.105.216.42
|
||||
45.162.204.37
|
||||
185.37.252.138
|
||||
212.205.74.89
|
||||
152.66.115.246
|
||||
45.67.159.244
|
||||
89.111.47.130
|
||||
91.220.43.19
|
||||
185.242.104.18
|
||||
132.248.59.73
|
||||
37.251.174.43
|
||||
81.196.9.142
|
||||
89.213.174.152
|
||||
91.188.226.250
|
||||
188.241.222.52
|
||||
5.154.224.26
|
||||
5.83.47.15
|
||||
176.124.104.176
|
||||
185.100.87.86
|
||||
164.73.233.2
|
||||
36
code/main.py
36
code/main.py
@@ -2,12 +2,23 @@ import requests, schedule, time
|
||||
from bs4 import BeautifulSoup
|
||||
from whatDomain import *
|
||||
|
||||
|
||||
DEBSECURITYURL = ["https://security.debian.org/debian-security/"]
|
||||
EXTRASURL = ["https://download.docker.com/linux/debian/",
|
||||
# Double just to be sure. Even though they point to the same IP
|
||||
"http://download.proxmox.com/debian/",
|
||||
"https://enterprise.proxmox.com/debian/pve/",
|
||||
# That's for nvidia docker toolkit something something
|
||||
# stuff that makes containers use nvenc and shit
|
||||
"https://nvidia.github.io/libnvidia-container/stable/deb/",
|
||||
"https://nvidia.github.io/libnvidia-container/experimental/deb/"]
|
||||
|
||||
DEBMIRRORURL = "https://www.debian.org/mirror/list"
|
||||
IPv4FILE = "../MirrorListV4"
|
||||
IPv6FILE = "../MirrorListV6"
|
||||
|
||||
|
||||
# Define EU and American countries
|
||||
# Define EU and American countries as well as Security for security updates
|
||||
target_countries = set([
|
||||
# Europe
|
||||
"Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark",
|
||||
@@ -17,7 +28,9 @@ target_countries = set([
|
||||
"United Kingdom", "Moldova",
|
||||
# America
|
||||
"Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
|
||||
"Mexico", "Peru", "United States", "Uruguay", "Venezuela"
|
||||
"Mexico", "Peru", "United States", "Uruguay", "Venezuela",
|
||||
# Others
|
||||
"Security", "Extras"
|
||||
])
|
||||
|
||||
|
||||
@@ -79,6 +92,9 @@ def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
|
||||
outMirrorDict[current_country] = []
|
||||
outMirrorDict[current_country].append(url)
|
||||
|
||||
outMirrorDict.update({"Security": DEBSECURITYURL})
|
||||
outMirrorDict.update({"Extras": EXTRASURL})
|
||||
|
||||
return outMirrorDict
|
||||
|
||||
def LeJob():
|
||||
@@ -107,13 +123,13 @@ def LeJob():
|
||||
|
||||
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
|
||||
|
||||
if ip4Dict == -1:
|
||||
continue
|
||||
|
||||
try:
|
||||
for key, ip in ip4Dict.items():
|
||||
# print(ip)
|
||||
print(ip)
|
||||
|
||||
fW.write(ip + "/32" + "\n")
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
|
||||
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
|
||||
@@ -130,17 +146,17 @@ def LeJob():
|
||||
|
||||
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
|
||||
|
||||
if ip6Dict == -1:
|
||||
continue
|
||||
|
||||
try:
|
||||
for key, ip in ip6Dict.items():
|
||||
# print(ip)
|
||||
|
||||
fW.write(ip + "/128" + "\n")
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
|
||||
# schedule.every().day.at("12:45").do(LeJob)
|
||||
schedule.every().day.at("12:40").do(LeJob)
|
||||
schedule.every().day.at("17:44").do(LeJob)
|
||||
|
||||
while True:
|
||||
schedule.run_pending()
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
beautifulsoup4==4.13.4
|
||||
requests==2.32.3
|
||||
schedule==1.2.2
|
||||
nslookup==1.8.1
|
||||
@@ -21,7 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
|
||||
try:
|
||||
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
|
||||
except dns.resolver.NoAnswer:
|
||||
return -1
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
@@ -46,7 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "
|
||||
try:
|
||||
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
|
||||
except dns.resolver.NoAnswer:
|
||||
return -1
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
@@ -74,7 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
|
||||
try:
|
||||
result = dns.resolver.resolve(whatToCheck, 'PTR')
|
||||
except dns.resolver.NoAnswer:
|
||||
return -1
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
|
||||
3
customList/.gitignore
vendored
Normal file
3
customList/.gitignore
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
repo
|
||||
__pycache__
|
||||
.env
|
||||
28
customList/Dockerfile
Normal file
28
customList/Dockerfile
Normal file
@@ -0,0 +1,28 @@
|
||||
FROM alpine:latest
|
||||
|
||||
ENV REPOFILE="/etc/customMirrors/repoList.list" EXTRAURL="http://mdu.se/"
|
||||
ENV DNSSRV="1.1.1.1"
|
||||
ENV IPV4FILENAME="MirrorListV4" IPV6FILENAME="MirrorListV6"
|
||||
ENV CRONTABSET="25 */4 * * *"
|
||||
|
||||
|
||||
RUN apk update && \
|
||||
apk add python3 py3-pip git bash
|
||||
|
||||
RUN mkdir /etc/customMirrors
|
||||
WORKDIR /etc/customMirrors/
|
||||
|
||||
COPY mainDocker.py .
|
||||
COPY whatDomain.py .
|
||||
COPY requirements.txt .
|
||||
COPY init.sh .
|
||||
|
||||
RUN python3 -m venv venv && \
|
||||
venv/bin/python3 -m pip install --upgrade pip && \
|
||||
venv/bin/pip3 install -r requirements.txt
|
||||
|
||||
COPY gitPush.sh .
|
||||
|
||||
RUN chmod +x ./gitPush.sh
|
||||
|
||||
CMD ["bash", "init.sh"]
|
||||
11
customList/cron-jobs
Normal file
11
customList/cron-jobs
Normal file
@@ -0,0 +1,11 @@
|
||||
# python
|
||||
#20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||
25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||
# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||
# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||
#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||
#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||
|
||||
# git push
|
||||
30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||
#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||
4
customList/customListGitVars.env
Normal file
4
customList/customListGitVars.env
Normal file
@@ -0,0 +1,4 @@
|
||||
GITURLPROTO=https
|
||||
GITURL=tea.shupogaki.org
|
||||
GITREPOPATH=YuruC3/Repo-IP-lists.git
|
||||
GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2
|
||||
1
customList/customListURLS.env
Normal file
1
customList/customListURLS.env
Normal file
@@ -0,0 +1 @@
|
||||
EXTRAURL=["https://www.pixiv.net/", "https://pixiv.net/", "https://i.pximg.net/", "https://pximg.net/", "iwara.tv/"]
|
||||
20
customList/docker-compose.yml
Normal file
20
customList/docker-compose.yml
Normal file
@@ -0,0 +1,20 @@
|
||||
---
|
||||
|
||||
services:
|
||||
debmirup:
|
||||
container_name: Debian_Mirrors_Updater
|
||||
build: ./
|
||||
#image: yuruc3/debianrepolist:v0.3
|
||||
|
||||
environment:
|
||||
- GITURLPROTO=https
|
||||
- GITURL=tea.shupogaki.org
|
||||
- GITREPOPATH=YuruC3/Repo-IP-lists.git
|
||||
- GITEA_TOKEN=3947a16ee5c3a337d6ff9cfb32d361167d4099d7
|
||||
- REPOFILE=repoList.list
|
||||
- IPV4FILENAME=customListNv4
|
||||
- IPV6FILENAME=customListNv6
|
||||
volumes:
|
||||
- ./repoList.list:/etc/customMirrors/repoList.list:ro
|
||||
|
||||
restart: unless-stopped
|
||||
87
customList/gitPush.sh
Normal file
87
customList/gitPush.sh
Normal file
@@ -0,0 +1,87 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
WORKPTH="/etc/customMirrors/"
|
||||
REPO_DIR="$WORKPTH/Repo-IP-lists"
|
||||
REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
||||
|
||||
# Clone repo if not exists
|
||||
if [ ! -d "$REPO_DIR/.git" ]; then
|
||||
echo "[$(date)] Cloning repository..."
|
||||
git clone "$REPO_URL" "$REPO_DIR"
|
||||
fi
|
||||
|
||||
cd "$REPO_DIR"
|
||||
|
||||
# Abort previous rebase/cherry-pick if stuck
|
||||
git rebase --abort 2>/dev/null || true
|
||||
git cherry-pick --abort 2>/dev/null || true
|
||||
|
||||
# Make sure we're on a clean 'main'
|
||||
git checkout main || git checkout -b main origin/main
|
||||
git pull --rebase --autostash
|
||||
|
||||
git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
||||
|
||||
|
||||
git config user.name "UpdateBot"
|
||||
git config user.email "UpdateBot@localhost.local"
|
||||
|
||||
# Stage the files
|
||||
git add $IPV4FILENAME $IPV6FILENAME
|
||||
|
||||
# Only proceed if there are staged changes
|
||||
if ! git diff --quiet --cached; then
|
||||
echo "[$(date)] Committing and pushing changes..."
|
||||
git commit -m "Auto-update mirror list on $(date -Iseconds)"
|
||||
# git pull --rebase --autostash
|
||||
git push --quiet
|
||||
echo "[$(date)] Changes pushed."
|
||||
else
|
||||
echo "[$(date)] No changes to commit or push."
|
||||
fi
|
||||
|
||||
|
||||
# #!/bin/sh
|
||||
# set -e
|
||||
|
||||
# WORKPTH="/etc/debmirror/"
|
||||
# REPO_DIRd="$WORKPTH/Repo-IP-lists"
|
||||
# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
||||
|
||||
|
||||
# # Clone repo only if it doesn't already exist
|
||||
# if [ ! -d "$REPO_DIR/.git" ]; then
|
||||
# echo "[$(date)] Cloning repository..."
|
||||
# git clone "$REPO_URL" "$REPO_DIR"
|
||||
# fi
|
||||
|
||||
|
||||
# cd "$REPO_DIR"
|
||||
|
||||
# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
||||
|
||||
# git config user.name "UpdateBot"
|
||||
# git config user.email "UpdateBot@localhost.local"
|
||||
|
||||
# # stage files
|
||||
# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
|
||||
|
||||
# # If anything to commit locally, commit it now
|
||||
# if ! git diff --quiet --cached; then
|
||||
# echo "[$(date)] Committing local changes before pulling"
|
||||
# git commit -m "Auto-commit before pull on $(date -Iseconds)"
|
||||
# fi
|
||||
|
||||
# # Now pull the latest
|
||||
# git pull --rebase --autostash
|
||||
|
||||
# # Commit and push only if there's anything new staged
|
||||
# if git diff --quiet; then
|
||||
# echo "[$(date)] No changes to commit."
|
||||
# else
|
||||
# git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet
|
||||
# git push --quiet
|
||||
# echo "[$(date)] Changes pushed successfully."
|
||||
# fi
|
||||
|
||||
23
customList/init.sh
Normal file
23
customList/init.sh
Normal file
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
echo "EXTRAURL: $EXTRAURL"
|
||||
|
||||
if [[ -f "$REPOFILE" ]]; then
|
||||
echo "URL file exists"
|
||||
else
|
||||
touch $REPOFILE
|
||||
echo $EXTRAURL | tee $REPOFILE
|
||||
fi
|
||||
|
||||
echo "nameserver $DNSSRV" > /etc/resolv.conf
|
||||
echo "search local" >> /etc/resolv.conf
|
||||
|
||||
git clone https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}
|
||||
ln -s /etc/customMirrors/Repo-IP-lists/$IPV4FILENAME /etc/customMirrors/$IPV4FILENAME
|
||||
ln -s /etc/customMirrors/Repo-IP-lists/$IPV6FILENAME /etc/customMirrors/$IPV6FILENAME
|
||||
|
||||
echo -n "$CRONTABSET /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py" > /etc/crontabs/customListCron
|
||||
|
||||
exec /usr/sbin/crond -f
|
||||
88
customList/mainDocker.py
Normal file
88
customList/mainDocker.py
Normal file
@@ -0,0 +1,88 @@
|
||||
import requests, schedule, time, os
|
||||
# from bs4 import BeautifulSoup
|
||||
from whatDomain import *
|
||||
|
||||
EXTRAURL = []
|
||||
|
||||
repoListFile = open(os.getenv("REPOFILE", "repoList.list"), 'r')
|
||||
i = 0
|
||||
for repoUrl in repoListFile:
|
||||
i += 1
|
||||
print(repoUrl.strip())
|
||||
EXTRAURL.append(repoUrl.strip())
|
||||
repoListFile.close()
|
||||
|
||||
IPV4FILENAME = str(os.getenv("IPV4FILENAME", "MirrorListV4"))
|
||||
IPV6FILENAME = str(os.getenv("IPV6FILENAME", "MirrorListV6"))
|
||||
|
||||
# EXTRAURL = list(os.getenv("EXTRAURL", "https://mdu.se/"))
|
||||
IPv4FILE = f"/etc/customMirrors/{IPV4FILENAME}"
|
||||
IPv6FILE = f"/etc/customMirrors/{IPV6FILENAME}"
|
||||
|
||||
def sanitizeURL(inpurl: str):
|
||||
if "https://" in inpurl:
|
||||
outurl = inpurl[8:]
|
||||
elif "http://" in inpurl:
|
||||
outurl = inpurl[7:]
|
||||
elif "http://" or "https://" not in url:
|
||||
outurl = inpurl
|
||||
else:
|
||||
return inpurl
|
||||
|
||||
i = 0
|
||||
for char in outurl:
|
||||
i += 1
|
||||
if char == "/":
|
||||
outurl = outurl[:i]
|
||||
|
||||
if char == "/":
|
||||
outurl = outurl[:-1]
|
||||
return outurl
|
||||
|
||||
def LeJob():
|
||||
|
||||
print("Starting lookup")
|
||||
|
||||
# print(LeMirrorDict)
|
||||
|
||||
with open(IPv4FILE, "w",) as fW:
|
||||
|
||||
for url in EXTRAURL:
|
||||
goodurl = sanitizeURL(url)
|
||||
# print(goodurl)
|
||||
|
||||
# ip4Dict = ermWhatATheIpFromDomainYaCrazy(url)
|
||||
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
|
||||
|
||||
try:
|
||||
for key, ip in ip4Dict.items():
|
||||
print(ip + "/32")
|
||||
|
||||
fW.write(ip + "/32" + "\n")
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
|
||||
with open(IPv6FILE, "w",) as fW:
|
||||
|
||||
for url in EXTRAURL:
|
||||
goodurl = sanitizeURL(url)
|
||||
# print(goodurl)
|
||||
|
||||
# ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(url)
|
||||
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
|
||||
|
||||
try:
|
||||
for key, ip in ip6Dict.items():
|
||||
print(ip + "/128")
|
||||
|
||||
fW.write(ip + "/128" + "\n")
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
|
||||
|
||||
|
||||
LeJob()
|
||||
|
||||
print("Done")
|
||||
5
customList/repoList.list
Normal file
5
customList/repoList.list
Normal file
@@ -0,0 +1,5 @@
|
||||
https://www.pixiv.net/
|
||||
https://pixiv.net/
|
||||
https://i.pximg.net/
|
||||
https://pximg.net/
|
||||
https://wiki.archlinux.org/
|
||||
4
customList/repoList.list.example
Normal file
4
customList/repoList.list.example
Normal file
@@ -0,0 +1,4 @@
|
||||
https://www.pixiv.net/
|
||||
https://pixiv.net/
|
||||
https://i.pximg.net/
|
||||
https://pximg.net/
|
||||
4
customList/requirements.txt
Normal file
4
customList/requirements.txt
Normal file
@@ -0,0 +1,4 @@
|
||||
#beautifulsoup4==4.13.4
|
||||
requests==2.32.3
|
||||
schedule==1.2.2
|
||||
nslookup==1.8.1
|
||||
129
customList/whatDomain.py
Normal file
129
customList/whatDomain.py
Normal file
@@ -0,0 +1,129 @@
|
||||
#from nslookup import Nslookup
|
||||
from typing import Optional, Annotated
|
||||
import dns, dns.resolver
|
||||
|
||||
# https://www.codeunderscored.com/nslookup-python/
|
||||
|
||||
def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
|
||||
#dns_query = Nslookup()
|
||||
"""
|
||||
Tells you what IPv4 address/es a domain point to.
|
||||
Returns:
|
||||
dict: A dictionary with IP addresses associated with that domain.
|
||||
|
||||
"""
|
||||
|
||||
# i = 0
|
||||
outDict = {}
|
||||
|
||||
#result = dns_query.dns_lookup("example.com")
|
||||
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||
try:
|
||||
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
|
||||
except dns.resolver.NoAnswer:
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
|
||||
return outDict
|
||||
|
||||
def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
|
||||
#dns_query = Nslookup()
|
||||
"""
|
||||
Tells you what IPv6 address/es a domain point to.
|
||||
Returns:
|
||||
dict: A dictionary with IP addresses associated with that domain.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# i = 0
|
||||
outDict = {}
|
||||
|
||||
#result = dns_query.dns_lookup("example.com")
|
||||
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||
try:
|
||||
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
|
||||
except dns.resolver.NoAnswer:
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
|
||||
return outDict
|
||||
|
||||
|
||||
def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]):
|
||||
#dns_query = Nslookup()
|
||||
"""
|
||||
Tells you what IPv6 address/es a domain point to.
|
||||
Returns:
|
||||
dict: A dictionary with IP addresses associated with that domain.
|
||||
|
||||
"""
|
||||
|
||||
whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa"
|
||||
|
||||
|
||||
# i = 0
|
||||
outDict = {}
|
||||
|
||||
#result = dns_query.dns_lookup("example.com")
|
||||
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||
try:
|
||||
result = dns.resolver.resolve(whatToCheck, 'PTR')
|
||||
except dns.resolver.NoAnswer:
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
|
||||
return outDict
|
||||
|
||||
|
||||
#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
|
||||
#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
|
||||
#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))
|
||||
BIN
docker.tar.gz
Normal file
BIN
docker.tar.gz
Normal file
Binary file not shown.
@@ -2,17 +2,27 @@ FROM alpine:latest
|
||||
|
||||
# https://docs.docker.com/reference/dockerfile/#environment-replacement
|
||||
ENV DEBMIRRORURL="https://www.debian.org/mirror/list"
|
||||
ENV EXTRAREPOS=True
|
||||
ENV SECURITYREPOS=True
|
||||
|
||||
RUN apk update && \
|
||||
apk add python3 py3-pip git
|
||||
# inotify-tools
|
||||
|
||||
RUN mkdir /etc/debmirror && \
|
||||
touch /etc/debmirror/MirrorListV6 && \
|
||||
touch /etc/debmirror/MirrorListV4
|
||||
RUN mkdir /etc/debmirror
|
||||
WORKDIR /etc/debmirror/
|
||||
|
||||
RUN git clone https://tea.shupogaki.org/YuruC3/Repo-IP-lists && \
|
||||
ln -s /etc/debmirror/Repo-IP-lists/MirrorListV4 /etc/debmirror/MirrorListV4 && \
|
||||
ln -s /etc/debmirror/Repo-IP-lists/MirrorListV6 /etc/debmirror/MirrorListV6 && \
|
||||
ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV4 /etc/debmirror/OPNS_MirrorListV4 && \
|
||||
ln -s /etc/debmirror/Repo-IP-lists/OPNS_MirrorListV6 /etc/debmirror/OPNS_MirrorListV6
|
||||
|
||||
# RUN touch /etc/debmirror/MirrorListV6 && \
|
||||
# touch /etc/debmirror/MirrorListV4
|
||||
|
||||
COPY mainDocker.py /etc/debmirror/
|
||||
COPY mainOPNsense.py /etc/debmirror/
|
||||
COPY whatDomain.py /etc/debmirror/
|
||||
COPY requirements.txt /etc/debmirror/
|
||||
|
||||
@@ -21,8 +31,11 @@ RUN python3 -m venv venv && \
|
||||
venv/bin/pip3 install -r requirements.txt
|
||||
|
||||
COPY cron-jobs /etc/crontabs/
|
||||
COPY gitPush.sh /etc/crontabs/
|
||||
RUN chmod 0644 /etc/crontabs/cron-jobs && \
|
||||
crontab /etc/crontabs/cron-jobs
|
||||
|
||||
RUN chmod +x /etc/crontabs/gitPush.sh
|
||||
COPY gitPush.sh /etc/debmirror/
|
||||
|
||||
CMD ["/usr/sbin/crond"]
|
||||
RUN chmod +x /etc/debmirror/gitPush.sh
|
||||
|
||||
CMD ["/usr/sbin/crond", "-f"]
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
# python
|
||||
25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||
20 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||
25 */4 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||
# 20 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||
# 25 19 * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||
#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainDocker.py
|
||||
#*/3 * * * * /etc/debmirror/venv/bin/python3 /etc/debmirror/mainOPNsense.py
|
||||
|
||||
# git push
|
||||
30 19 * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||
30 */4 * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||
#*/1 * * * * /bin/sh /etc/debmirror/gitPush.sh
|
||||
@@ -3,9 +3,14 @@
|
||||
services:
|
||||
debmirup:
|
||||
container_name: Debian_Mirrors_Updater
|
||||
build: ./
|
||||
image: DebMirrorLists:V1
|
||||
# build: ./
|
||||
image: yuruc3/debianrepolist:v0.3
|
||||
environment:
|
||||
- DEBMIRRORURL=https://www.debian.org/mirror/list
|
||||
- GITURLPROTO=https
|
||||
- GITURL=tea.shupogaki.org
|
||||
- GITREPOPATH=YuruC3/Repo-IP-lists.git
|
||||
- GITEA_TOKEN=0938a2033324b987bbcb2976b56d147a9a00d8a2
|
||||
- EXTRAREPOS=True
|
||||
- SECURITYREPOS=True
|
||||
restart: unless-stopped
|
||||
privileged: false
|
||||
|
||||
@@ -2,33 +2,86 @@
|
||||
set -e
|
||||
|
||||
WORKPTH="/etc/debmirror/"
|
||||
REPO_DIR="$WORKPTH/Debain-repos"
|
||||
REPO_URL="https://$GITEA_TOKEN@tea.shupogaki.org/YuruC3/Debain-repos.git"
|
||||
REPO_DIR="$WORKPTH/Repo-IP-lists"
|
||||
REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
||||
|
||||
# Clone repo only if it doesn't already exist
|
||||
# Clone repo if not exists
|
||||
if [ ! -d "$REPO_DIR/.git" ]; then
|
||||
echo "[$(date)] Cloning repository..."
|
||||
git clone "$REPO_URL" "$REPO_DIR"
|
||||
fi
|
||||
|
||||
|
||||
cd "$REPO_DIR"
|
||||
|
||||
# Abort previous rebase/cherry-pick if stuck
|
||||
git rebase --abort 2>/dev/null || true
|
||||
git cherry-pick --abort 2>/dev/null || true
|
||||
|
||||
# Make sure we're on a clean 'main'
|
||||
git checkout main || git checkout -b main origin/main
|
||||
git pull --rebase --autostash
|
||||
|
||||
git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
||||
|
||||
|
||||
git config user.name "UpdateBot"
|
||||
git config user.email "UpdateBot@localhost.local"
|
||||
|
||||
# pull changes
|
||||
git pull
|
||||
# Stage the files
|
||||
git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
|
||||
|
||||
# check if modified
|
||||
if git diff --quiet && git diff --cached --quiet; then
|
||||
echo "No changes to commit."
|
||||
# if yes, push
|
||||
else
|
||||
git add MirrorsV4 MirrorsV6
|
||||
# Only proceed if there are staged changes
|
||||
if ! git diff --quiet --cached; then
|
||||
echo "[$(date)] Committing and pushing changes..."
|
||||
git commit -m "Auto-update mirror list on $(date -Iseconds)"
|
||||
git push
|
||||
echo "Changes pushed successfully."
|
||||
# git pull --rebase --autostash
|
||||
git push --quiet
|
||||
echo "[$(date)] Changes pushed."
|
||||
else
|
||||
echo "[$(date)] No changes to commit or push."
|
||||
fi
|
||||
|
||||
|
||||
# #!/bin/sh
|
||||
# set -e
|
||||
|
||||
# WORKPTH="/etc/debmirror/"
|
||||
# REPO_DIRd="$WORKPTH/Repo-IP-lists"
|
||||
# REPO_URL="${GITURLPROTO}://${GITURL}/${GITREPOPATH}"
|
||||
|
||||
|
||||
# # Clone repo only if it doesn't already exist
|
||||
# if [ ! -d "$REPO_DIR/.git" ]; then
|
||||
# echo "[$(date)] Cloning repository..."
|
||||
# git clone "$REPO_URL" "$REPO_DIR"
|
||||
# fi
|
||||
|
||||
|
||||
# cd "$REPO_DIR"
|
||||
|
||||
# git remote set-url origin "https://${GITEA_TOKEN}@${GITURL}/${GITREPOPATH}"
|
||||
|
||||
# git config user.name "UpdateBot"
|
||||
# git config user.email "UpdateBot@localhost.local"
|
||||
|
||||
# # stage files
|
||||
# git add MirrorListV4 MirrorListV6 OPNS_MirrorListV4 OPNS_MirrorListV6
|
||||
|
||||
# # If anything to commit locally, commit it now
|
||||
# if ! git diff --quiet --cached; then
|
||||
# echo "[$(date)] Committing local changes before pulling"
|
||||
# git commit -m "Auto-commit before pull on $(date -Iseconds)"
|
||||
# fi
|
||||
|
||||
# # Now pull the latest
|
||||
# git pull --rebase --autostash
|
||||
|
||||
# # Commit and push only if there's anything new staged
|
||||
# if git diff --quiet; then
|
||||
# echo "[$(date)] No changes to commit."
|
||||
# else
|
||||
# git commit -a -m "Auto-update mirror list on $(date -Iseconds)" --quiet
|
||||
# git push --quiet
|
||||
# echo "[$(date)] Changes pushed successfully."
|
||||
# fi
|
||||
|
||||
|
||||
@@ -2,17 +2,23 @@ import requests, schedule, time, os
|
||||
from bs4 import BeautifulSoup
|
||||
from whatDomain import *
|
||||
|
||||
DEBMIRRORURL = str(os.getenv("DEBMIRRORURL", "https://www.debian.org/mirror/list"))
|
||||
DEBSECURITYURL = str(os.getenv("DEBSECURITYURL", "https://security.debian.org/debian-security/"))
|
||||
|
||||
if os.environ["DEBMIRRORURL"]:
|
||||
DEBMIRRORURL = os.environ["DEBMIRRORURL"]
|
||||
else:
|
||||
DEBMIRRORURL = "https://www.debian.org/mirror/list"
|
||||
EXTRASURL = ["https://download.docker.com/linux/debian/",
|
||||
# Double just to be sure. Even though they point to the same IP
|
||||
"http://download.proxmox.com/debian/",
|
||||
"https://enterprise.proxmox.com/debian/pve/",
|
||||
# That's for nvidia docker toolkit something something
|
||||
# stuff that makes containers use nvenc and shit
|
||||
"https://nvidia.github.io/libnvidia-container/stable/deb/",
|
||||
"https://nvidia.github.io/libnvidia-container/experimental/deb/"]
|
||||
|
||||
IPv4FILE = "./MirrorListV4"
|
||||
IPv6FILE = "./MirrorListV6"
|
||||
IPv4FILE = "/etc/debmirror/MirrorListV4"
|
||||
IPv6FILE = "/etc/debmirror/MirrorListV6"
|
||||
|
||||
|
||||
# Define EU and American countries
|
||||
# Define EU and American countries as well as Security for security updates
|
||||
target_countries = set([
|
||||
# Europe
|
||||
"Austria", "Belgium", "Bulgaria", "Croatia", "Czech Republic", "Denmark",
|
||||
@@ -22,7 +28,9 @@ target_countries = set([
|
||||
"United Kingdom", "Moldova",
|
||||
# America
|
||||
"Argentina", "Brazil", "Canada", "Chile", "Colombia", "Costa Rica", "Ecuador",
|
||||
"Mexico", "Peru", "United States", "Uruguay", "Venezuela"
|
||||
"Mexico", "Peru", "United States", "Uruguay", "Venezuela",
|
||||
# Others
|
||||
"Security", "Extras"
|
||||
])
|
||||
|
||||
|
||||
@@ -34,7 +42,7 @@ def sanitizeURL(inpurl: str):
|
||||
elif "http://" or "https://" not in url:
|
||||
outurl = inpurl
|
||||
else:
|
||||
return -1
|
||||
return 1
|
||||
|
||||
i = 0
|
||||
for char in outurl:
|
||||
@@ -84,6 +92,11 @@ def sanitizeUrlsGodWhatTheFuckIsThis(SoupInput: BeautifulSoup):
|
||||
outMirrorDict[current_country] = []
|
||||
outMirrorDict[current_country].append(url)
|
||||
|
||||
if os.environ["SECURITYREPOS"]:
|
||||
outMirrorDict.update({"Security": DEBSECURITYURL})
|
||||
if os.environ["EXTRAREPOS"]:
|
||||
outMirrorDict.update({"Extras": EXTRASURL})
|
||||
|
||||
return outMirrorDict
|
||||
|
||||
def LeJob():
|
||||
@@ -112,13 +125,13 @@ def LeJob():
|
||||
|
||||
ip4Dict = ermWhatATheIpFromDomainYaCrazy(goodurl)
|
||||
|
||||
if ip4Dict == -1:
|
||||
continue
|
||||
|
||||
try:
|
||||
for key, ip in ip4Dict.items():
|
||||
# print(ip)
|
||||
print(ip)
|
||||
|
||||
fW.write(ip + "/32" + "\n")
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
|
||||
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
|
||||
@@ -135,14 +148,16 @@ def LeJob():
|
||||
|
||||
ip6Dict = ermWhatAAAATheIpFromDomainYaCrazy(goodurl)
|
||||
|
||||
if ip6Dict == -1:
|
||||
continue
|
||||
|
||||
try:
|
||||
for key, ip in ip6Dict.items():
|
||||
# print(ip)
|
||||
|
||||
fW.write(ip + "/128" + "\n")
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
|
||||
|
||||
LeJob()
|
||||
|
||||
print("Done")
|
||||
135
docker/mainOPNsense.py
Normal file
135
docker/mainOPNsense.py
Normal file
@@ -0,0 +1,135 @@
|
||||
import requests, schedule, time
|
||||
from bs4 import BeautifulSoup
|
||||
from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
|
||||
|
||||
|
||||
OPNSNSMIRRORURL = str(os.getenv("OPNSNSMIRRORURL", "https://opnsense.org/download/#full-mirror-listing"))
|
||||
|
||||
IPv4FILE = str(os.getenv("IPv4FILE", "/etc/debmirror/OPNS_MirrorListV4"))
|
||||
IPv6FILE = str(os.getenv("IPv6FILE", "/etc/debmirror/OPNS_MirrorListV6"))
|
||||
|
||||
|
||||
def sanitizeURL(inpurl: str):
|
||||
if not "/" in inpurl[:-1]:
|
||||
inpurl += "/"
|
||||
if "https://" in inpurl:
|
||||
outurl = inpurl[8:]
|
||||
elif "http://" in inpurl:
|
||||
outurl = inpurl[7:]
|
||||
elif "http://" or "https://" not in url:
|
||||
outurl = inpurl
|
||||
else:
|
||||
return -1
|
||||
|
||||
# how the fuck does it work?
|
||||
# I mean I wrote this but I don't know why does it work.
|
||||
i = 0
|
||||
for char in outurl:
|
||||
i += 1
|
||||
if char == "/":
|
||||
outurl = outurl[:i]
|
||||
|
||||
if char == "/":
|
||||
outurl = outurl[:-1]
|
||||
return outurl
|
||||
|
||||
|
||||
def getFreshData():
|
||||
payload = requests.get(OPNSNSMIRRORURL)
|
||||
soup = BeautifulSoup(payload.content, "html.parser")
|
||||
return soup
|
||||
|
||||
def LeJob():
|
||||
|
||||
print("Starting lookup")
|
||||
|
||||
LeSoup = getFreshData()
|
||||
|
||||
# print(LeMirrorDict)
|
||||
|
||||
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
|
||||
|
||||
for data in LeSoup.find_all('div', class_='download_section'):
|
||||
for a in data.find_all('a', href=True):
|
||||
|
||||
url = a['href']
|
||||
|
||||
saniturl = sanitizeURL(url)
|
||||
|
||||
# print(saniturl)
|
||||
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
# print(IPv4Dict)
|
||||
try:
|
||||
for key, ip in IPv4Dict.items():
|
||||
print(f"Found the ipv4: {ip}")
|
||||
fW.write(ip + "/32" + "\n")
|
||||
|
||||
# If int returned from WhatDomain then it is error.
|
||||
# Error type is described via prints from whatdomain functions
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
|
||||
|
||||
for data in LeSoup.find_all('div', class_='download_section'):
|
||||
for a in data.find_all('a', href=True):
|
||||
|
||||
url = a['href']
|
||||
|
||||
saniturl = sanitizeURL(url)
|
||||
|
||||
# print(saniturl)
|
||||
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
# print(IPv6Dict)
|
||||
try:
|
||||
for key, ip in IPv6Dict.items():
|
||||
print(f"Found the ipv6: {ip}")
|
||||
fW.write(ip + "/128" + "\n")
|
||||
|
||||
# If int returned from WhatDomain then it is error.
|
||||
# Error type is described via prints from whatdomain functions
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
# schedule.every().day.at("12:45").do(LeJob)
|
||||
# schedule.every().day.at("17:44").do(LeJob)
|
||||
|
||||
# while True:
|
||||
# schedule.run_pending()
|
||||
# print("Waiting...")
|
||||
# time.sleep(30) #Wait one minute
|
||||
# # LeJob()
|
||||
|
||||
LeJob()
|
||||
|
||||
# gigalist = []
|
||||
|
||||
# payload = requests.get(OPNSNSMIRRORURL)
|
||||
# soup = BeautifulSoup(payload.content, "html.parser")
|
||||
|
||||
|
||||
# for data in soup.find_all('div', class_='download_section'):
|
||||
# for a in data.find_all('a', href=True):
|
||||
|
||||
# url = a['href']
|
||||
|
||||
# saniturl = sanitizeURL(url)
|
||||
|
||||
# # print(saniturl)
|
||||
# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||
# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
# # print(IPv4Dict)
|
||||
# try:
|
||||
# for key, ip in IPv4Dict.items():
|
||||
# print(f"Found the ipv4: {ip}")
|
||||
|
||||
# for key, ip in IPv6Dict.items():
|
||||
# print(f"Found the ipv6: {ip}")
|
||||
# # If int returned from WhatDomain then it is error.
|
||||
# # Error type is described via prints from whatdomain functions
|
||||
# except AttributeError:
|
||||
# continue
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
beautifulsoup4==4.13.4
|
||||
requests==2.32.3
|
||||
schedule==1.2.2
|
||||
nslookup==1.8.1
|
||||
@@ -21,7 +21,21 @@ def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Dom
|
||||
try:
|
||||
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
|
||||
except dns.resolver.NoAnswer:
|
||||
return -1
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
@@ -46,7 +60,21 @@ def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "
|
||||
try:
|
||||
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
|
||||
except dns.resolver.NoAnswer:
|
||||
return -1
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
@@ -74,7 +102,21 @@ def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP
|
||||
try:
|
||||
result = dns.resolver.resolve(whatToCheck, 'PTR')
|
||||
except dns.resolver.NoAnswer:
|
||||
return -1
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
|
||||
2
opnsense/.gitignore
vendored
Normal file
2
opnsense/.gitignore
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
__pycache__
|
||||
test
|
||||
52
opnsense/OPNS_MirrorListV4
Normal file
52
opnsense/OPNS_MirrorListV4
Normal file
@@ -0,0 +1,52 @@
|
||||
202.158.214.106/32
|
||||
200.201.197.203/32
|
||||
178.157.61.30/32
|
||||
216.24.182.173/32
|
||||
162.105.120.100/32
|
||||
104.21.112.1/32
|
||||
104.21.16.1/32
|
||||
104.21.32.1/32
|
||||
104.21.48.1/32
|
||||
104.21.64.1/32
|
||||
104.21.80.1/32
|
||||
104.21.96.1/32
|
||||
130.225.254.116/32
|
||||
46.32.144.253/32
|
||||
201.159.221.67/32
|
||||
201.159.221.67/32
|
||||
172.67.206.93/32
|
||||
104.21.22.179/32
|
||||
141.98.136.67/32
|
||||
37.208.104.98/32
|
||||
37.58.58.140/32
|
||||
147.52.159.50/32
|
||||
180.151.236.94/32
|
||||
78.39.136.13/32
|
||||
121.141.119.252/32
|
||||
121.141.34.226/32
|
||||
121.171.109.2/32
|
||||
220.118.162.155/32
|
||||
220.118.162.15/32
|
||||
211.205.104.177/32
|
||||
112.222.241.182/32
|
||||
112.164.39.61/32
|
||||
221.138.103.74/32
|
||||
221.142.142.68/32
|
||||
121.159.44.64/32
|
||||
196.200.160.70/32
|
||||
150.242.40.7/32
|
||||
23.147.168.189/32
|
||||
91.132.103.246/32
|
||||
46.235.145.153/32
|
||||
109.202.202.202/32
|
||||
212.237.209.20/32
|
||||
163.22.168.137/32
|
||||
5.79.108.33/32
|
||||
185.28.39.100/32
|
||||
89.149.222.99/32
|
||||
193.162.43.250/32
|
||||
212.219.56.184/32
|
||||
66.111.2.15/32
|
||||
207.244.94.80/32
|
||||
209.58.135.187/32
|
||||
169.229.200.70/32
|
||||
31
opnsense/OPNS_MirrorListV6
Normal file
31
opnsense/OPNS_MirrorListV6
Normal file
@@ -0,0 +1,31 @@
|
||||
2001:388:30bc:cafe::beef/128
|
||||
2001:da8:201:1120::a269:7864/128
|
||||
2606:4700:3030::6815:7001/128
|
||||
2606:4700:3030::6815:1001/128
|
||||
2606:4700:3030::6815:2001/128
|
||||
2606:4700:3030::6815:3001/128
|
||||
2606:4700:3030::6815:4001/128
|
||||
2606:4700:3030::6815:5001/128
|
||||
2606:4700:3030::6815:6001/128
|
||||
2001:878:346::116/128
|
||||
2001:470:dd11:1337:d0cd:bfff:feaa:fb37/128
|
||||
2800:68:0:bebe::3/128
|
||||
2606:4700:3036::ac43:ce5d/128
|
||||
2606:4700:3034::6815:16b3/128
|
||||
2a09:11c0:1200:210:ffff:ffff:8d62:8843/128
|
||||
2a00:d0c0:d::100/128
|
||||
2a00:c98:2030:a034::21/128
|
||||
2001:648:2c00:6c08::2/128
|
||||
2001:4310:f1::70/128
|
||||
2602:faa5:189::/128
|
||||
2602:faa5:190::/128
|
||||
2a12:d282:102:f6::1/128
|
||||
2a00:d08:1:c::153/128
|
||||
2001:1620::1620/128
|
||||
2a03:da40:2::20/128
|
||||
2001:1af8:4700:b210::33/128
|
||||
2001:1af8:5300:a010:1::1/128
|
||||
2001:630:341:12::184/128
|
||||
2604:9a00:2010:a0b8::5/128
|
||||
2605:fe80:2100:b001::5187/128
|
||||
2607:f140:0:32::70/128
|
||||
187
opnsense/main.log
Normal file
187
opnsense/main.log
Normal file
@@ -0,0 +1,187 @@
|
||||
[<div class="download_section">
|
||||
<h2>Australia</h2>
|
||||
<h3><a href="https://mirror.aarnet.edu.au/pub/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="AARNet Pty Ltd">AARNet Pty Ltd</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Brazil</h2>
|
||||
<h3><a href="https://mirror.cloudfence.com.br/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Cloudfence">Cloudfence</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>China</h2>
|
||||
<h3><a href="https://opnsense.aivian.org/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Aivian">Aivian</a></h3>
|
||||
<h3><a href="https://mirrors.pku.edu.cn/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Peking University open source software mirror">Peking University open source software mirror</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Colombia</h2>
|
||||
<h3><a href="https://mirror.venturasystems.tech/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Ventura Systems">Ventura Systems</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Denmark</h2>
|
||||
<h3><a href="https://mirrors.dotsrc.org/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Aalborg University">Aalborg University</a></h3>
|
||||
<h3><a href="https://opnsense.c0urier.net/releases/mirror/" rel="noopener noreferrer" target="_blank" title="c0urier.net">c0urier.net</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Ecuador</h2>
|
||||
<h3><a href="https://mirror.cedia.org.ec/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="CEDIA">CEDIA</a></h3>
|
||||
<h3><a href="https://mirror.ueb.edu.ec/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Universidad Estatal de Bolívar">Universidad Estatal de Bolívar</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>France</h2>
|
||||
<h3><a href="https://mirror.vraphim.com/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Vraphim">Vraphim</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Germany</h2>
|
||||
<h3><a href="https://mirror.dns-root.de/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="dns-root.de (Cloudflare CDN)">dns-root.de (Cloudflare CDN)</a></h3>
|
||||
<h3><a href="https://mirror.level66.network/opnsense-dist/releases/mirror/" rel="noopener noreferrer" target="_blank" title="level66.network">level66.network</a></h3>
|
||||
<h3><a href="https://mirror.uvensys.de/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="uvensys GmbH">uvensys GmbH</a></h3>
|
||||
<h3><a href="https://mirror.fra10.de.leaseweb.net/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="LeaseWeb">LeaseWeb</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Greece</h2>
|
||||
<h3><a href="https://ftp.cc.uoc.gr/mirrors/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="University of Crete / Department of Physics">University of Crete / Department of Physics</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Hungary</h2>
|
||||
<h3><a href="https://quantum-mirror.hu/mirrors/pub/opnsense" rel="noopener noreferrer" target="_blank" title="Quantum Mirror">Quantum Mirror</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>India</h2>
|
||||
<h3><a href="https://mirrors.hopbox.net/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Hopbox">Hopbox</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Iran</h2>
|
||||
<h3><a href="https://mirror1.isatisidc.ir/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Isatis Data Center (isatisidc.ir)">Isatis Data Center (isatisidc.ir)</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Korea</h2>
|
||||
<h3><a href="https://mirror.amuksa.com/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="DAXNET, ROKFOSS PROJECT">DAXNET, ROKFOSS PROJECT</a></h3>
|
||||
<h3><a href="https://mirror.techlabs.co.kr/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Techlabs, ROKFOSS PROJECT">Techlabs, ROKFOSS PROJECT</a></h3>
|
||||
<h3><a href="https://mirror.zzunipark.com/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Zzunimirror, ROKFOSS PROJECT">Zzunimirror, ROKFOSS PROJECT</a></h3>
|
||||
<h3><a href="https://mirror.hemino.net/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="HMirror, ROKFOSS PROJECT">HMirror, ROKFOSS PROJECT</a></h3>
|
||||
<h3><a href="https://mirror.keiminem.com/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Keiverse (Gumi, South Korea)">Keiverse (Gumi, South Korea)</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Morocco</h2>
|
||||
<h3><a href="https://mirror.marwan.ma/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="MARWAN, Rabat">MARWAN, Rabat</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>New Zealand</h2>
|
||||
<h3><a href="https://mirror.catalyst.net.nz/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="catalyst.net Ltd.">catalyst.net Ltd.</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Portugal</h2>
|
||||
<h3><a href="https://mirror.leitecastro.com/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Community (Lisbon)">Community (Lisbon)</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Spain</h2>
|
||||
<h3><a href="https://mirror.raiolanetworks.com/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Raiola Networks (Madrid)">Raiola Networks (Madrid)</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Switzerland</h2>
|
||||
<h3><a href="https://opnsense-mirror.hiho.ch/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Hiho.ch">Hiho.ch</a></h3>
|
||||
<h3><a href="https://mirror.init7.net/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Init7">Init7</a></h3>
|
||||
<h3><a href="https://mirror-opnsense.serverbase.ch/releases/mirror/" rel="noopener noreferrer" target="_blank" title="ServerBase AG">ServerBase AG</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Taiwan</h2>
|
||||
<h3><a href="https://mirror.ntct.edu.tw/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Nantou County Education Network Center">Nantou County Education Network Center</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>The Netherlands</h2>
|
||||
<h3><a href="https://mirror.ams1.nl.leaseweb.net/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="LeaseWeb">LeaseWeb</a></h3>
|
||||
<h3><a href="https://mirror.serverion.com/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Serverion">Serverion</a></h3>
|
||||
<h3><a href="https://pkg.opnsense.org/releases/mirror/" rel="noopener noreferrer" target="_blank" title="OPNsense">OPNsense</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>Turkey</h2>
|
||||
<h3><a href="https://mirror.verinomi.com/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="Verinomi">Verinomi</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>United Kingdom</h2>
|
||||
<h3><a href="https://www.mirrorservice.org/sites/opnsense.org/releases/mirror/" rel="noopener noreferrer" target="_blank" title="University of Kent">University of Kent</a></h3>
|
||||
</div>, <div class="download_section">
|
||||
<h2>United States</h2>
|
||||
<h3><a href="https://mirrors.nycbug.org/pub/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="NYC*BUG (East Coast)">NYC*BUG (East Coast)</a></h3>
|
||||
<h3><a href="https://mirror.wdc1.us.leaseweb.net/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="LeaseWeb (East Coast)">LeaseWeb (East Coast)</a></h3>
|
||||
<h3><a href="https://mirror.sfo12.us.leaseweb.net/opnsense/releases/mirror/" rel="noopener noreferrer" target="_blank" title="LeaseWeb (West Coast)">LeaseWeb (West Coast)</a></h3>
|
||||
<h3><a href="https://mirrors.ocf.berkeley.edu/opnsense/" rel="noopener noreferrer" target="_blank" title="Open Computing Facility at UC Berkeley (Berkeley, California)">Open Computing Facility at UC Berkeley (Berkeley, California)</a></h3>
|
||||
</div>]
|
||||
Found the URL: https://opnsense.org
|
||||
Found the URL: /
|
||||
Found the URL: #
|
||||
Found the URL: https://opnsense.org/get-started/
|
||||
Found the URL: #
|
||||
Found the URL: https://opnsense.org/blog/
|
||||
Found the URL: https://shop.opnsense.com/
|
||||
Found the URL: https://opnsense.org/download/
|
||||
Found the URL: https://opnsense.org/opnsense/
|
||||
Found the URL: https://opnsense.org/roadmap/
|
||||
Found the URL: https://opnsense.org/features/
|
||||
Found the URL: https://opnsense.org/partners/
|
||||
Found the URL: https://opnsense.org/legal-guidelines/
|
||||
Found the URL: https://opnsense.org/contact/
|
||||
Found the URL: https://opnsense.org/get-started/
|
||||
Found the URL: https://docs.opnsense.org/
|
||||
Found the URL: https://opnsense.org/your-invited/
|
||||
Found the URL: http://forum.opnsense.org/
|
||||
Found the URL: https://docs.opnsense.org/
|
||||
Found the URL: https://opnsense.org/donate/
|
||||
Found the URL: mailto:project@opnsense.org
|
||||
Found the URL: /
|
||||
Found the URL: #
|
||||
Found the URL: https://opnsense.org/opnsense/
|
||||
Found the URL: https://opnsense.org/roadmap/
|
||||
Found the URL: https://opnsense.org/features/
|
||||
Found the URL: https://opnsense.org/partners/
|
||||
Found the URL: https://opnsense.org/legal-guidelines/
|
||||
Found the URL: https://opnsense.org/contact/
|
||||
Found the URL: https://opnsense.org/get-started/
|
||||
Found the URL: https://opnsense.org/get-started/
|
||||
Found the URL: https://docs.opnsense.org/
|
||||
Found the URL: #
|
||||
Found the URL: https://opnsense.org/your-invited/
|
||||
Found the URL: http://forum.opnsense.org/
|
||||
Found the URL: https://docs.opnsense.org/
|
||||
Found the URL: https://opnsense.org/donate/
|
||||
Found the URL: https://opnsense.org/blog/
|
||||
Found the URL: https://shop.opnsense.com/
|
||||
Found the URL: https://opnsense.org/download/
|
||||
Found the URL: https://opnsense.org/
|
||||
Found the URL: https://opnsense.org/donate/
|
||||
Found the URL: https://mirror.ams1.nl.leaseweb.net/opnsense/releases/25.1/OPNsense-25.1-dvd-amd64.iso.bz2
|
||||
Found the URL: #full-mirror-listing
|
||||
Found the URL: https://docs.opnsense.org/manual/install.html#download-and-verification
|
||||
Found the URL: https://shop.opnsense.com/product/opnsense-business-edition/
|
||||
Found the URL: https://www.amazon.de/Practical-OPNsense-Building-Enterprise-Firewalls/dp/3757805364/
|
||||
Found the URL: https://shop.opnsense.com/product/opnsense-business-edition/
|
||||
Found the URL: https://mynetworktraining.com/p/official-opnsense-introduction-course
|
||||
Found the URL: https://mirror.aarnet.edu.au/pub/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.cloudfence.com.br/opnsense/releases/mirror/
|
||||
Found the URL: https://opnsense.aivian.org/releases/mirror/
|
||||
Found the URL: https://mirrors.pku.edu.cn/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.venturasystems.tech/opnsense/releases/mirror/
|
||||
Found the URL: https://mirrors.dotsrc.org/opnsense/releases/mirror/
|
||||
Found the URL: https://opnsense.c0urier.net/releases/mirror/
|
||||
Found the URL: https://mirror.cedia.org.ec/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.ueb.edu.ec/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.vraphim.com/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.dns-root.de/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.level66.network/opnsense-dist/releases/mirror/
|
||||
Found the URL: https://mirror.uvensys.de/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.fra10.de.leaseweb.net/opnsense/releases/mirror/
|
||||
Found the URL: https://ftp.cc.uoc.gr/mirrors/opnsense/releases/mirror/
|
||||
Found the URL: https://quantum-mirror.hu/mirrors/pub/opnsense
|
||||
Found the URL: https://mirrors.hopbox.net/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror1.isatisidc.ir/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.amuksa.com/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.techlabs.co.kr/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.zzunipark.com/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.hemino.net/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.keiminem.com/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.marwan.ma/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.catalyst.net.nz/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.leitecastro.com/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.raiolanetworks.com/opnsense/releases/mirror/
|
||||
Found the URL: https://opnsense-mirror.hiho.ch/releases/mirror/
|
||||
Found the URL: https://mirror.init7.net/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror-opnsense.serverbase.ch/releases/mirror/
|
||||
Found the URL: https://mirror.ntct.edu.tw/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.ams1.nl.leaseweb.net/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.serverion.com/opnsense/releases/mirror/
|
||||
Found the URL: https://pkg.opnsense.org/releases/mirror/
|
||||
Found the URL: https://mirror.verinomi.com/opnsense/releases/mirror/
|
||||
Found the URL: https://www.mirrorservice.org/sites/opnsense.org/releases/mirror/
|
||||
Found the URL: https://mirrors.nycbug.org/pub/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.wdc1.us.leaseweb.net/opnsense/releases/mirror/
|
||||
Found the URL: https://mirror.sfo12.us.leaseweb.net/opnsense/releases/mirror/
|
||||
Found the URL: https://mirrors.ocf.berkeley.edu/opnsense/
|
||||
Found the URL: mailto:project@opnsense.org
|
||||
Found the URL: mailto:sales@opnsense.com
|
||||
Found the URL: https://forum.opnsense.org/
|
||||
Found the URL: https://web-cdn.bsky.app/profile/did:plc:vpna73dxrqinj6ttwh3wryxv
|
||||
Found the URL: https://www.linkedin.com/groups/6958160/
|
||||
Found the URL: https://www.reddit.com/r/opnsense/?rdt=38653
|
||||
Found the URL: https://github.com/opnsense
|
||||
Found the URL: /donate
|
||||
Found the URL: https://deciso.com
|
||||
Found the URL: https://opnsense.org/terms-conditions/
|
||||
Found the URL: https://opnsense.org/privacy-en-cookies/
|
||||
135
opnsense/main.py
Normal file
135
opnsense/main.py
Normal file
@@ -0,0 +1,135 @@
|
||||
import requests, schedule, time
|
||||
from bs4 import BeautifulSoup
|
||||
from whatDomain import ermWhatAAAATheIpFromDomainYaCrazy, ermWhatATheIpFromDomainYaCrazy
|
||||
|
||||
|
||||
|
||||
OPNSNSMIRRORURL = "https://opnsense.org/download/#full-mirror-listing"
|
||||
IPv4FILE = "./OPNS_MirrorListV4"
|
||||
IPv6FILE = "./OPNS_MirrorListV6"
|
||||
|
||||
|
||||
def sanitizeURL(inpurl: str):
|
||||
if not "/" in inpurl[:-1]:
|
||||
inpurl += "/"
|
||||
if "https://" in inpurl:
|
||||
outurl = inpurl[8:]
|
||||
elif "http://" in inpurl:
|
||||
outurl = inpurl[7:]
|
||||
elif "http://" or "https://" not in url:
|
||||
outurl = inpurl
|
||||
else:
|
||||
return -1
|
||||
|
||||
# how the fuck does it work?
|
||||
# I mean I wrote this but I don't know why does it work.
|
||||
i = 0
|
||||
for char in outurl:
|
||||
i += 1
|
||||
if char == "/":
|
||||
outurl = outurl[:i]
|
||||
|
||||
if char == "/":
|
||||
outurl = outurl[:-1]
|
||||
return outurl
|
||||
|
||||
|
||||
def getFreshData():
|
||||
payload = requests.get(OPNSNSMIRRORURL)
|
||||
soup = BeautifulSoup(payload.content, "html.parser")
|
||||
return soup
|
||||
|
||||
def LeJob():
|
||||
|
||||
print("Starting lookup")
|
||||
|
||||
LeSoup = getFreshData()
|
||||
|
||||
# print(LeMirrorDict)
|
||||
|
||||
with open(IPv4FILE, "r",) as fR, open(IPv4FILE, "w",) as fW:
|
||||
|
||||
for data in LeSoup.find_all('div', class_='download_section'):
|
||||
for a in data.find_all('a', href=True):
|
||||
|
||||
url = a['href']
|
||||
|
||||
saniturl = sanitizeURL(url)
|
||||
|
||||
# print(saniturl)
|
||||
IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
# print(IPv4Dict)
|
||||
try:
|
||||
for key, ip in IPv4Dict.items():
|
||||
print(f"Found the ipv4: {ip}")
|
||||
fW.write(ip + "/32" + "\n")
|
||||
|
||||
# If int returned from WhatDomain then it is error.
|
||||
# Error type is described via prints from whatdomain functions
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
with open(IPv6FILE, "r",) as fR, open(IPv6FILE, "w",) as fW:
|
||||
|
||||
for data in LeSoup.find_all('div', class_='download_section'):
|
||||
for a in data.find_all('a', href=True):
|
||||
|
||||
url = a['href']
|
||||
|
||||
saniturl = sanitizeURL(url)
|
||||
|
||||
# print(saniturl)
|
||||
IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
# print(IPv6Dict)
|
||||
try:
|
||||
for key, ip in IPv6Dict.items():
|
||||
print(f"Found the ipv6: {ip}")
|
||||
fW.write(ip + "/128" + "\n")
|
||||
|
||||
# If int returned from WhatDomain then it is error.
|
||||
# Error type is described via prints from whatdomain functions
|
||||
except AttributeError:
|
||||
continue
|
||||
|
||||
# schedule.every().day.at("12:45").do(LeJob)
|
||||
# schedule.every().day.at("17:44").do(LeJob)
|
||||
|
||||
# while True:
|
||||
# schedule.run_pending()
|
||||
# print("Waiting...")
|
||||
# time.sleep(30) #Wait one minute
|
||||
# # LeJob()
|
||||
|
||||
LeJob()
|
||||
|
||||
# gigalist = []
|
||||
|
||||
# payload = requests.get(OPNSNSMIRRORURL)
|
||||
# soup = BeautifulSoup(payload.content, "html.parser")
|
||||
|
||||
|
||||
# for data in soup.find_all('div', class_='download_section'):
|
||||
# for a in data.find_all('a', href=True):
|
||||
|
||||
# url = a['href']
|
||||
|
||||
# saniturl = sanitizeURL(url)
|
||||
|
||||
# # print(saniturl)
|
||||
# IPv4Dict = ermWhatATheIpFromDomainYaCrazy(saniturl)
|
||||
# IPv6Dict = ermWhatAAAATheIpFromDomainYaCrazy(saniturl)
|
||||
|
||||
# # print(IPv4Dict)
|
||||
# try:
|
||||
# for key, ip in IPv4Dict.items():
|
||||
# print(f"Found the ipv4: {ip}")
|
||||
|
||||
# for key, ip in IPv6Dict.items():
|
||||
# print(f"Found the ipv6: {ip}")
|
||||
# # If int returned from WhatDomain then it is error.
|
||||
# # Error type is described via prints from whatdomain functions
|
||||
# except AttributeError:
|
||||
# continue
|
||||
|
||||
4
opnsense/requirements.txt
Normal file
4
opnsense/requirements.txt
Normal file
@@ -0,0 +1,4 @@
|
||||
beautifulsoup4==4.13.4
|
||||
requests==2.32.3
|
||||
schedule==1.2.2
|
||||
nslookup==1.8.1
|
||||
129
opnsense/whatDomain.py
Normal file
129
opnsense/whatDomain.py
Normal file
@@ -0,0 +1,129 @@
|
||||
#from nslookup import Nslookup
|
||||
from typing import Optional, Annotated
|
||||
import dns, dns.resolver
|
||||
|
||||
# https://www.codeunderscored.com/nslookup-python/
|
||||
|
||||
def ermWhatATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
|
||||
#dns_query = Nslookup()
|
||||
"""
|
||||
Tells you what IPv4 address/es a domain point to.
|
||||
Returns:
|
||||
dict: A dictionary with IP addresses associated with that domain.
|
||||
|
||||
"""
|
||||
|
||||
# i = 0
|
||||
outDict = {}
|
||||
|
||||
#result = dns_query.dns_lookup("example.com")
|
||||
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||
try:
|
||||
result = dns.resolver.resolve(inpDomainNameOrSomething, 'A')
|
||||
except dns.resolver.NoAnswer:
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
|
||||
return outDict
|
||||
|
||||
def ermWhatAAAATheIpFromDomainYaCrazy(inpDomainNameOrSomething: Annotated[str, "Domain name to lookup IP for"]):
|
||||
#dns_query = Nslookup()
|
||||
"""
|
||||
Tells you what IPv6 address/es a domain point to.
|
||||
Returns:
|
||||
dict: A dictionary with IP addresses associated with that domain.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
# i = 0
|
||||
outDict = {}
|
||||
|
||||
#result = dns_query.dns_lookup("example.com")
|
||||
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||
try:
|
||||
result = dns.resolver.resolve(inpDomainNameOrSomething, 'AAAA')
|
||||
except dns.resolver.NoAnswer:
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
|
||||
return outDict
|
||||
|
||||
|
||||
def ermWhatPTRTheIpFromDomainYaCrazy(inpIpAddressOrSomething: Annotated[str, "IP address to lookup domain for"]):
|
||||
#dns_query = Nslookup()
|
||||
"""
|
||||
Tells you what IPv6 address/es a domain point to.
|
||||
Returns:
|
||||
dict: A dictionary with IP addresses associated with that domain.
|
||||
|
||||
"""
|
||||
|
||||
whatToCheck = inpIpAddressOrSomething + ".in-addr.arpa"
|
||||
|
||||
|
||||
# i = 0
|
||||
outDict = {}
|
||||
|
||||
#result = dns_query.dns_lookup("example.com")
|
||||
#result = Nslookup.dns_lookup(inpDomainNameOrSomething)
|
||||
try:
|
||||
result = dns.resolver.resolve(whatToCheck, 'PTR')
|
||||
except dns.resolver.NoAnswer:
|
||||
print("\nDNS ERROR")
|
||||
print("No answer from dns server.\n")
|
||||
return 1
|
||||
except dns.resolver.NoNameservers:
|
||||
print("\nDNS ERROR")
|
||||
print("All nameservers failed to answer the query.\n Fix your DNS servers.\n")
|
||||
return 1
|
||||
except dns.resolver.NXDOMAIN:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS query name does not exist.\n")
|
||||
return 1
|
||||
except dns.resolver.LifetimeTimeout:
|
||||
print("\nDNS ERROR")
|
||||
print("The DNS querry got timed out.\nVerify that your FW or PiHole isn't blocking requests for that domain.\n")
|
||||
return 1
|
||||
for i, something in enumerate(result):
|
||||
outDict[i] = something.to_text()
|
||||
# i += 1
|
||||
|
||||
return outDict
|
||||
|
||||
|
||||
#print(ermWhatATheIpFromDomainYaCrazy("fubukus.net"))
|
||||
#print(ermWhatAAAATheIpFromDomainYaCrazy("fubukus.net"))
|
||||
#print(ermWhatPTRTheIpFromDomainYaCrazy("192.168.1.226"))
|
||||
Reference in New Issue
Block a user