First version that is cron-based

This commit is contained in:
2026-08-08 17:45:27 +02:00
parent 7866a764b1
commit fbffec5f91
7 changed files with 781 additions and 0 deletions

495
main.py Normal file
View File

@@ -0,0 +1,495 @@
import os, socket, subprocess, paramiko, io, sys, time
from typing import Final
from datetime import datetime
IDRAC: Final[str] = os.getenv("IDRAC", None)
IDRAC_USR: Final[str] = os.getenv("IDRAC_USR", None)
IDRAC_PWD: Final[str] = os.getenv("IDRAC_PWD", None)
# LAST_VERIFY_DAY: Final[int] = os.getenv("LAST_VERIFY_DAY", 7)
# LAST_GARBAGE_DAY: Final[int] = os.getenv("LAST_GARBAGE_DAY", 7)
HOST: Final[str] = os.getenv("HOST", None)
HOST_SSH_PORT: Final[int] = os.getenv("HOST_SSH_PORT", None)
HOST_USR: Final[str] = os.getenv("HOST_USR", "root")
HOST_USR_PWD: Final[str] = os.getenv("HOST_USR_PWD", None)
HOST_USR_SSHKEY: Final[str] = os.getenv("HOST_USR_SSHKEY", None)
HOST_USR_SSHKEY_PASS: Final[str] = os.getenv("HOST_USR_SSHKEY_PASS", None)
LOCAL_DATASTORE: Final[str] = os.getenv("LOCAL_DATASTORE", None)
REMOTE_DATASTORE: Final[str] = os.getenv("HOST_UREMOTE_DATASTORESR_SSHKEY_PASS", None)
REMOTE_PBS: Final[str] = os.getenv("REMOTE_PBS", None)
SYNC_DIR: Final[str] = os.getenv("SYNC_DIR", "push")
PBS_RATE_OUT: Final[str] = os.getenv("PBS_RATE_OUT", "")
PBS_RATE_IN: Final[str] = os.getenv("PBS_RATE_IN", "")
PBS_TASK_LIMIT: Final[int] = os.getenv("PBS_TASK_LIMIT", 50)
PBS_MAX_RUNTIME_MIN: Final[int] = os.getenv("PBS_MAX_RUNTIME_MIN", 180)
PBS_MAX_SHUT_MIN: Final[int] = os.getenv("PBS_MAX_SHUT_MIN", 10)
AUTO_ADD_NAME: Final[str] = os.getenv("AUTO_ADD_NAME", "autoBackupScript_one-shot")
def exceptionator(exception, message):
"""Return an exception that can be custom-used"""
return exception(message)
def load_private_key(key_data: str, passphrase: str | None = None):
"""Take in SSH key, determine type and return paramiko-compatible element"""
key_classes = [
paramiko.Ed25519Key,
paramiko.ECDSAKey,
paramiko.RSAKey,
# paramiko.DSSKey,
]
for key_class in key_classes:
try:
return key_class.from_private_key(
io.StringIO(key_data),
password=passphrase or None,
)
except (paramiko.SSHException, ValueError):
pass
raise ValueError("Unsupported or invalid SSH private key")
def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
"""Get what tasks are running/ran on PBS
Keyword arguments:
allTasks -- if tasks that are finished should be included. (default False)
limitOfTasks -- how many tasks to ask PBS for (default 50)
"""
(stdin, stdout, stderr) = ssh.exec_command(f'proxmox-backup-manager task list --limit {limitOfTasks} --all {allTasks}')
taskList = stdout.read().decode()
outTaskList = []
for line in taskList.splitlines():
line = line.strip()
# Only process actual table rows
if not line.startswith(""):
continue
columns = [col.strip() for col in line.strip("").split("")]
# Skip the header row
if columns[0] == "starttime":
continue
if len(columns) >= 2:
outTaskList.append(columns[2].split(":")[6])
# exit()
# print(outTaskList)
outTaskDict = {}
for task in outTaskList:
match task:
case "garbage_collection":
outTaskDict["garbage_collection"] = outTaskDict.get("garbage_collection", 0) + 1
case "verificationjob":
outTaskDict["verificationjob"] = outTaskDict.get("verificationjob", 0) + 1
case "syncjob":
outTaskDict["syncjob"] = outTaskDict.get("syncjob", 0) + 1
case "aptupdate":
outTaskDict["aptupdate"] = outTaskDict.get("aptupdate", 0) + 1
case "reader":
outTaskDict["reader"] = outTaskDict.get("reader", 0) + 1
case "logrotate":
outTaskDict["logrotate"] = outTaskDict.get("logrotate", 0) + 1
case "termproxy":
outTaskDict["termproxy"] = outTaskDict.get("termproxy", 0) + 1
case _:
outTaskDict["otherjob"] = outTaskDict.get("otherjob", 0) + 1
return outTaskDict
if len(AUTO_ADD_NAME) > 32:
raise exceptionator(ValueError, "Maxmimum length of AUTO_ADD_NAME is 32")
sys.exit(1)
# dt = datetime.now.timestamp()
# epoch_time = dt.timestamp()
# print(epoch_time)
# Check if power is on
# PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
try:
# power on if needed
if "off" in str(PWR_STATE.stdout.decode()[-4:]):
subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"]) # , capture_output=True
except exception as e:
print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
sys.exit(1)
# else:
# continue
PWR_DATE = datetime.now().timestamp()
pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
if pinging:
print("not reachable")
while datetime.now().timestamp() - PWR_DATE <= 1200:
pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
if not pinging:
print("reachable")
break
else:
raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.")
sys.exit(1)
else:
print("reachable")
# Create object of SSHClient and
# connecting to SSH
ssh = paramiko.SSHClient()
# Adding new host key to the local
# HostKeys object(in case of missing)
# AutoAddPolicy for missing host key to be set before connection setup.
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if not HOST_USR_SSHKEY:
ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, )
else:
key = load_private_key(
HOST_USR_SSHKEY,
HOST_USR_SSHKEY_PASS
)
ssh.connect(
hostname=HOST,
port=HOST_SSH_PORT,
username=HOST_USR,
pkey=key,
timeout=3,
allow_agent=False,
look_for_keys=False,
)
# Execute command on SSH terminal
# using exec_command
# (stdin, stdout, stderr) = ssh.exec_command('hostname')
# # redirecting all the output in cmd_output
# # variable
# cmd_output = stdout.read().decode()
# print(cmd_output)
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
remoteList = stdout.read().decode()
remoteDict = {}
for line in remoteList.splitlines():
line = line.strip()
# Only process actual table rows
if not line.startswith(""):
continue
columns = [col.strip() for col in line.strip("").split("")]
# Skip the header row
if columns[0] == "name":
continue
if len(columns) >= 2:
name = columns[0]
host = columns[1]
remoteDict[host] = name
if REMOTE_PBS in remoteDict:
print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}")
else:
print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr)
REMOTE_PBS.remove(REMOTE_PBS)
sys.exit(1)
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
syncJobList = stdout.read().decode()
for line in syncJobList.splitlines():
line = line.strip()
# Only process actual table rows
if not line.startswith(""):
continue
columns = [col.strip() for col in line.strip("").split("")]
# Skip the header row
if columns[0] == "id":
continue
if len(columns) >= 2:
print("Sync job already on Proxmox Backup Server.\nSkipping sync job creation.")
# print(columns)
break
else:
match SYNC_DIR:
case "push":
if PBS_RATE_OUT:
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --rate-out {PBS_RATE_OUT} --comment 'automatic startupper script'")
else:
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --comment 'automatic startupper script'")
case "pull":
if PBS_RATE_IN:
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --rate-in {PBS_RATE_IN} --comment 'automatic startupper script'")
else:
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --comment 'automatic startupper script'")
case _:
raise exceptionator(SyncDirectionError, f"No such sync direction as {SYNC_DIR}.\n Either use 'push' or 'pull'\n")
# print(stdout.read().decode())
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
# ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
# print("here it runs the sync-job")
# print(stdout.read().decode())
# print(stderr.read().decode())
# Maybe add some checks here if the backup was properly scheduled
# Get tasks
# print(getCurrentTasks(limitOfTasks=100))
# check ongoing tasks
SYNC_JOB_DATE = datetime.now().timestamp()
currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
while currenttasks:
currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
print("Waiting 1 minute before checking tasks again")
time.sleep(60)
if SYNC_JOB_DATE - datetime.now().timestamp() >= (PBS_MAX_RUNTIME_MIN * 60):
print(SYNC_JOB_DATE - datetime.now().timestamp())
# Forcefully skip if these tasks are running. reader task is, I think, PVE backup
# Also otherjob is something else and it is better to wait if there is one otherjob
if (
currenttasks.get("aptupdate", 0)
or currenttasks.get("reader", 0)
or currenttasks.get("otherjob", 0)
):
continue
# No need to do anything here. It's only to check if these keys exist
...
break
(stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
print("Shuting down PBS")
# print("here it runs a shutdown now")
print(stdout.read().decode())
print(stderr.read().decode(), file=sys.stderr)
ssh.close()
# wait for PBS to safely shutdown
time.sleep(PBS_MAX_SHUT_MIN * 60)
if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
print("PBS is powered off")
else:
subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"])
sys.exit(0)
# ▒▒▒▓▓▓▒▒▒▒▓▒▒▒▒▓▓▒▒▓▓▒ ▒ ▒ ▒▒░▒▒▒▒▒▒▒▒▒▒ ▒▒▒▒▒ ░░░ ████████
# ▒▒▒▒▓▓▒▓▒▒▓▒▓▓▒▓▒▒▒▓▓ ▒ ▒░ ▒▒▒▒░▒▒▒▒ ░ ░ ██████
# ▒▒▓░▒▓▓▓▓▒▒▓▓▒▒▓▓▒▓▒ ▒░ ▒░ ▒▒ ▒▒ ░░ ████
# ▓ ▒▒▒▒▒▓▓▒▓▒▓▓▒░▒ ▒ ░░ ██
# ▒▒▓▓▓▒░▒▒▓▒▒▒ ░░ █
# ░ ▓▒▓ ▒ ░░
# ░
# ▒ ▒▓ ░
# ░ ▓▒▒ ░▓▓▓▓▓ ▒▒
# ▒▓▓▓░▓▓░▓▓▓▓▓▓
# ▓▓▓▓▓▓░▓▒░░▒▒ ▒▒▓▓░░░▓▓▓▓▓░▓▓▓▓
# ▓▓▓░▒▓░▓▓▒▒▒ ▒▓▓▓▓░▓▓▓▓▓▓▓▓▓▓▓▓▓
# ░░░░░ ▓▓▓▓▓░▓▒▒▒▒▒▓▓▓▓▒ ▓▓▓▓▓▓░░░▒▓▓▓▓
# ░░░░░░░░░░░░░░ ▓▓▓▓▓░▒▒ ▒▒▓▓▓▓▓▓▓▓░░▓▒▓▓▓▓▓▓▓▓▓
# █ ░░░░░░░░░░░░░░░░ ▓▓▓▓▓▒▒ ▒▒▒░▒▓ ▓▓▓▓▓▓▓▓░▓▓▓▓▓▓▓▓▓
# ███ ░░░░░░░░░░░░░░░░░ ▒░▒▒ ▒▓▓▓▓▒▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░
# ████ ░░░░░░░░░░▓▓▓▓▓▒░░▒▓▓▓▓░▓░▓▓▓▓▓▓▓▓░▒▒░░▓▓▓▓▓
# ██████ ░▓▓▓▓▓▓▓▓▒▒░▓▓▓▓ ▓▓░▓▓▓▓▒░░▓▒▓▓▓▓▓▓▓░▒
# ████████ ▓▓▓▓▒▓▓▓▓▓▓▓▒▒░▓▒░▒ ▓▓▒░░▒▓▓▓▓▓░▓▓▓▓▓▓▒░▒ ▒ ▓░▓▓▒
# ███████████ ▓▓▓▒▓▓▓▓▓▓▓░▒▒▒ ▒▒ ▓▓▓░▓▓▓▓▓▓▓▓▒▓▓▓▓▓▒▒ ▒▓▓░▓▓ ▓▓▓▓▓▓▒▓▓▓▓▓░
# ████████████ ▓▒ ▒▒▒▒▒ ▓▓▓▓▒▓▓▓▓▓▓▓░▓▒▓▓▒░▓░▓▓▓▓▓▓▓▓▓▓░▓▓▓▓▒░░░▓▓▓▓░
# ████████████▓ ▓▒ ▒ ▒ ▓▓▓▓░▓▓▓▓▓▓▓▓░▒▓▓▓▓▓░▒▓▓▓░░▒▓ ▒░▒▓▓▓▓▓▓░▓▓▓▓▓
# ███████████ ▒▒▓▓▓░▓▓▓░▒▓▓▒▓▓▓▓░▒ ▒▒▓▓▓▓▓▒▒▒░▒▓▓▓▓▓▓▒▓▓▓▓
# ████████▓ ▓▓▓▓▓▓▓▓░ ▒▒▓▓▒▓▓░▓▓▓▓▓▓▓░▓▓▓▓
# ██████ ▓▓▓▓▓▓▓▓▓▒░░
# ████ ▓░ ░▓░░░▓▓▓░▓▓▓
# ██ ░░░░░░░░░░░░░░░░░ ░▓▓▓▓▓▓▓▒▓▓▓
# ▓ ░░░░░ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ ░░░░░░░░░ ▓▓▓▓▓▓▓▓▓░▓▓
# ░░ ░ ▓▒▒▒▒▒▒▒▒▒▒▒ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ ░ ▓▓░▓▓▓▒░░▒░▓▓
# ░░░ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ ▒▒▒▒▒▒▒▒▒▒▒▒▒▓▓▒░ ░░ ▓▒ ▓▓▓▓▓▓▓▓▓▒▓
# ▓████▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ ░░░ ░░░ ▒▒▒▒▒▒▒▒ ▓▓▓▓▓▓ ▓
# ███████ ▒▒▒▒▒▒▒▒▒▒▒▓▒▒▒▒▒▒▒▒ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▓░ ░░░░░ ░░░░░ ░ ▒▒▒▒▒▒▒▒▒▒▒▒▓ ▒▒▒▒▒▒▓
# ███████ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒░░▒▒▒▒▒▒▒▒▒▒▓▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ ▒▒▒▒▒▒▒▒▒▒▒▒▓▒▓▒▒▒▒▒▒▒ ░
# ███████ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒░░▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ ▓▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▒▒▒▒▒▒▓ ░▒▒▒▒▒▒▒
# ██████ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▓▒▒▓▒▒▒▓▓▓▓▓▓▓▓▓▒▒▒▓▒░░ ▒▓▓▓▓▓▓▓▓▓▓▓▒▒▒▒▒▒▒▓▒▒▒▓▒▒▒▒▓▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▒▒▒▓ ░▓▒▒▒▓▒▒
# ██████ ▒▒▒▒▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓ █▒░▓▓▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▒▒▒▒▒▓▓▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▒▒▓ ▒ ▒▒▓▒▒
# ██████ ▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ██▒▓▓▓▓▒▓▓██▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▒▓▓▓░▓▓▓▓▓▒░▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒░▒ ░ ▒▓▓▒
# █████ ▒▒▒▒▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓█▓▓▓▓▓████ ▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓ ▒▓▓▓▓
# █████ ▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░▓▓▓▓▓▓▓▓▓▓█████▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓ ▓▓▓
# █████ ▓▓▓▓▓▓▒▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓██████▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓█▓▓▓▓▓▓▓▓▓▓█▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓ ▓▓▓
# █████ ▓▓▓▓▓▓▒▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓█▓▓▓▓▓▓▓██████ ▓▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▒▓▓▓▓▓▓█▓▓▒▓▓▓▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▒▓ ▓▓▓
# ████▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓███████ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓ ▒▓▓▓▓
# ████▓▓▓▓▓▓▓▓▒▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓░▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░███████▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓
# █████▓▓▓▓▒▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓ ▒▓▓▓▓▓▓▓▓ ████████▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒░▓▓▓▓▓▓▒ ▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓
# █████▓▓▓▓▒▓▓▒▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓ ▒ ▓▓▓▓▓▓▓▒█████████▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▒▓▓▓▓▓▓▓▒ ▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓ ░ ▓
# █████▓▓▓▓▒▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▓▓▓▒█▓ ▓▓▓▒▓▓▓▓▓█▓▓▓█▓██ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▒██▓▓▓▓▓██▒▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓ ▒▓
# █████▓▓▓▓▒▓▓▒▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓ ▓ ▓ ▓ ▓ ▒▓█████▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓████▒▓▓▓████ ▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▓
# █████ ▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░ ▓ ▒▒▓▒███▓ ░ ███ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▓▓▓▓ ████░▓▓██▓█▓ ▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓ ░▓
# ██ ██ ▓▓▓▓▒▓▓▓▓▓▓▓▒▓▓▓▓▓▓ ▒▒ ░▓█▓▓▓████▒▓▓▓ ▒▓▓▓▓▓▒ █ █████▓▓▓▓▓▓▓▓▓▓▓▓▓▓███▓▓▓▓▓▓▓█▓ ▓▓▓▓█▓█▓▓ ▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓ ▒▓
# ▓ ███▓▓▓ ░▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▒▓▓▓██░█████▓▓▓▓▓▓▒▒▒▒▒▓▓▓▓▓▓█████▓▓ ▓▓▓▓▓▓▓▓▓██ █▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓ ▓ ▓
# █▒ ██▓▓ ██ ▓▒▓▓▓▓▓▓░▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓████▓█▒▒▓▓ ▓▓ █▒█ ▒▓▓▒███▒▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓ ▒▓▓
# ███ ██ ▓░▓██▓▒▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓██████▓█▓█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓ ▓ ░▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓ ▓ ▒█
# ███ ███▓▓▓██▒▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒ ██████▓▓ ░░░▓▓▓▓▒▓▒▓▓▓▓▓▓▒▓ ▓ ▒ ▒
# ███████ ▓████▓▒▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓░▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓█▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓ ▓ ░ ▒
# ██ ████▒█▓██▓▒▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓█▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▒▓▓▓▓▓▓▒▓ ▓▓ ▒ ▒
# ██ ██████▒░▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓ ▓▓▓▓▓▓▒ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓██▓ ▒
# ██▓▓ ████████▓▓▓▓▒▓▓▓▓▒▓▓▓▓▓▓▒▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓█▒
# ██ ▒██████▒▓█░▓▓▓▓▒▒▒▒▒▓▓▓▓▓▓▓▓▒▒ ░ ░▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒
# ████░█▒▒███████ ▓▓▓ ▒▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▓▓██▓██▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓
# ██▒▒▒▒▒███████ ▓▒▓▓ ▓▓▓▓▓▓█▓▓▓▓▓▓▓ ░▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▓▓▓▓▓▓▓▓▓▓▒▒▒ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▓▓▓▓▓▓▓▓▒▓▓▓▓▓ ▒
# ████░░▓█████ ▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓█████▓████████░▓▓▓▓▓▓▓▓▓▓▓▓▓▒▒█▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▒▓▓▓▓▓ ▓ ▒ ▓▓
# ███████████ ▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒ ██████████████▓▓▓▓▓▓▓▓▓▓▓▓▓▓░███▓▓▓▓▓▓▓▓▓▓▓▓▓ ░ ░▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓ ▒ ▒▓▓
# ██████████ ▓▒▓▓▓ ▓▓▓▓▓▓▓▓▓ ▓▓▓░░░▒▒▒▒█████████████▓▓▓▓▓▓▓▓▓▓▓▓▓█████████▓▓▓▓▓▓ ░░░ ░▓▓▓▓▓▓▓▒▒▓▓▓▓ ▓█▒▓░ ▒ ▓▓▒▒░▓▓
# ███████▒▓▓█ ▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ░░▒▒▒▒▒ ███████████▓▓▓▓▓▓▓▓▓▒██████████████ ░▒ ░░░░ ▓▓▓▓▓▓▓▓▒▓▓▓ ▓ ░ ▒▒▓▓▓▓▓▒▒ ▓ █
# ███▓▒ ░▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓░░ ▒▒▒▒█▒▒ ████████████████████████████ ▒▒▒▒▒▒ ░░░░▓▓ ▓▓▓▓▓▓▓▓▓ ▒▓▓▓▓▓▒▒▓ ▓ █
# ▒ ▓░▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▒▒▓▒▒▒▒▒ ████████████████████▒▒▒▒█ ▒▒▒▒▒ █ ▓▓ ▒▒▓▓▓▓▓▓▓ ▒ ▒ ▓▓▓▒▒▒▒▓ █
# ████████████▓ ██ ▓▓▓▓▓▓ ▒▒▒▒▒▒ ▒▓ ▒▒ ▒▒ ██████████ ▒▒▒ ▒▒▒▒▒▒▒▒▒▒▒▒▓▓ ▒▒▒▒▓▓▓▓▓▒░▒▓ ▓░▓█ ▒░▓▓▓▒▒▓▓▓░▓ █
# ▒ ▒▓▓▓▒ ███████████████████ ▒▓▓▓▓▓▓▓▒▒▒ ▒▒▒▒▒▒▒ ▒▒▒▒▒▒▒░ ░░░░░░▒ ▒▓▒▒▒▒▒▒▒▒▒▒▒ ▒▒▒▒▒▒▒▓▒▒░▒░▒▓▒▓░ ▓ ▓▓▓▓▒▓▓▓▓ ▓▓█
# █▓ ▒▓█▓▓███ ██████▓█████░░▓█████ ░░ ▒▓▓▓▓▓▓▒ ▒▒▒▒▒▒▒▒ ▒ ▒▒▒▒▒▒░░░░░░░░░▒▒░▒▒▒▒▒▒▒▒▒ ▒▒▒▒▒▒▒▒▒ ▒░▒░▒ ░ ▓ ▒▒▓▒▓▒▓▓▓▓▓ █▓█
# ██▓▒██░▓██ █████▒▒░▒░██░░░▒█████ ▒▒▒▒▒ ▒▓▓▓▓▓ ▒▒░▒▒ ▒▒▒▒▒▒▒ ▒▒░░░░░░░▒ ▒ ▒▒▒▒░░░ ▒ ▒▒▒▒▒▒▓▓▓▓░░░░░▒ ▓▒▒▓▓▓▒▓▓ █▓░
# ██░▒█▓ ▒ ██████░░░░▒▒░░░▓█████ ▓▓▓▓▒ ▒ ░░░░░░▒ ▒▒▒▒▒▒░░▒▒ ░░░░░░░▒▒ ░▒▒▒▒▒░░ ▒▒▒▒▓▓▓▓▓▓░░░ ▒ ▓▓▓▓▓▓▒░▓ ▓▓
# ▓▓ ░ ███████░░░░░░░░██████ ▓▓▓▓▓▓ ░░░░░░░▒ ░░░░░░░░ ▒ ░░░▒▒▒▒░▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓▓▓▓▓▓▓▓▓ ▒ ▒▓▓▓▓▓▓▒▓▓ ▓
# ░ ████▒░▒█▒░░░░░▒█░▒▒██ ▓▓▓▓▓▓▓ ░░░░░░░ ░░░░░░░░░░ ▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ░ █ █▒▒▒▒▒▓▓▒▓▓▓▓▓▓▒▓▓ ▓
# ██░░░░▒███░░░░██▒▓██ ▓▓▓▓▓▓▓▓▓ ░░░░░░░░░░▒▒▒▒▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ░ █▒▒ ▓▓▓▒▓▓▓▓▓▓▓▓▓▓ ▓
# ███▒▒█████████████▓ ▓▓▓▓▓▓▓▓▓▓ ░░░▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒░▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ░ ▓ █▒▒▓▒▓▓▒▓▓▓▓▓▓▓▓▓▓ ▓
# ████████████████ ▒▓▓▓▓▓▓▓▓▓▓▓ ░▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓ ▒ █ ▒▒▒▓▒▓▒▓▓▓▓▓▓▓▓▓▓▓ ▓
# ███████████ ▓▓▓▓▓▓▓▓▓▓▓▓▓ ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓ ▓ ▒▒ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓
# █▓ ░ ▓▓▓▓▓▓▓▓▓▓▓▓▓▒▒▒▒▒▒▒▒▒▒▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓ ▓ ░ ▒▒▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒▓▓
# ▓▓▓▓▓▓▓▓▓▓▓▓▓▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ▓▓ ▓ ▓▓ ▒▒▓▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓
# █▒▒▒▒▒ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒ ░ ▒▓ ▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓
# ███████ ▓██▓▒▒▒ ▓▓▓ ▓▓▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ░░░░ ▒▓▓▓▓▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓
# ███████▓ ▒███▒▒▒▒ ▓▓▓░▓▓▓▓▓ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ ░░░░░ ▒▓▓▓▓▒▒▓▓▓▓▓▓▓▓▓▓▓▓▓
# ███████████▓ ▒▓▓▓▓▓▓▓▓▓░ ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▒ ▓▓▓▓▓▓▓▓▓▓ ░░░ ░▓▓▓▓▒▒▒▓▓▓▓▓▓▓▓▓▓▓▓
# ████████▒ ▓▒▓▓▓▓▓▓▓▓▓▓ ░ ▓▓▓▓▓▓▓▓▓▓▒ ▒▓▒▒ ░░ ▓▓▓▒▒▒▒▓▓▒▒▓▓▓▓▓▓▒▓
# █████▒ ▒░ ▒░▓▓▓▓▓▒▓▓▓▓ ░░ ▒▓▓▓▓▓▓ ▓▓▓▓▒▓▒▓▓▓▒▒▓▒▓▓▓▓
# ██▓ ▒▒▒▒░ ▓▓▓▓▓▓▓▓▓▓ ░ ░▓▓▓ ░ ▓ ▓ ▒▓▓▓▓▒▓▓▓▓▓▓▓
# ▒ ▒▒▒▒▒▒░ ▓▒ ▓▓▓▓▓▓▓▓▒▓▒ ░░░░░ ▓▓ ▓ ▒░ ▒▓▓▓▓▓▓▓▓
# ░▒▒▒▒▒▒░ ▓▓ ▓▓▓▓▓▓▓▓▓▒ ░░░ ░ ▒▒▒▒▒ ▒▒ ▒▒▒▒▒▒▓▓▓▓▒
# ▒▒▒▓█▒▒▒░ ▓ ▓▒▓▓▓▓▓▓▓▓▓▒▓░░ ▒ ▒▓ ▓ ▒▒▒▒▒░▒▓▓▓▓▒
# ▒▓██▒▒▒▒ ▓▓▓▓▓▓▓▓▓▒▓▓ ▒▒▒▒▒▒▒▒▒▓▓▓▓▓ ▓▓
# ██▓▒▒▒▒ ░▒▓▓▓▓▓▓ ▓▓▓▓▓▒▒▒ ▓▒
# █▓▒▒▒▒░ ▓▓▓▓▓ ▓▓▓▓▓▒▒▒ ▒▓▓▒
# ▓▒▒▒▒░ ▓▓ ▒▒▒▒▒▒▒░ ▒▒▓
# ▒▒▒▒▒ ░▓▓
# ▒▒▒▒▒ ▓▓
# ▒▒▒▒▒ ▓▓
# ▒▒▒▒▒ ▓▓
# ▒▒▒▒▒ ▒▓
# ▒▒▒▒▒▒ ▓▓
# ▒▒▒▒▒▒ ▒ ▓▓
# ▒▒▒▒▒▒ ▒▓▓
# ▒▒▒▒▒▒░ ▒ ▓▓▓
# ▒▒▒▒▒▒▒ ▒▓▓▓
# ▓▒▒▒▒▒▓▓ ░▓▓▓▒
# ▓▒▒▒▒▓▓▓▒ ▒▓▓▒▒▓