YuruC3/PBSupper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.1 of APscheduler implementation

Browse Source
This commit is contained in:
YuruC3
2026-08-08 19:44:19 +02:00
parent fbffec5f91
commit 8660793ec2
2 changed files with 264 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
main.py
View File

@@ -1,35 +1,37 @@
import os, socket, subprocess, paramiko, io, sys, time
from typing import Final
from datetime import datetime
from apscheduler.schedulers.blocking import BlockingScheduler
from apscheduler.triggers.cron import CronTrigger
IDRAC: Final[str] = os.getenv("IDRAC", None)
IDRAC_USR: Final[str] = os.getenv("IDRAC_USR", None)
IDRAC_PWD: Final[str] = os.getenv("IDRAC_PWD", None)
# LAST_VERIFY_DAY: Final[int] = os.getenv("LAST_VERIFY_DAY", 7)
# LAST_GARBAGE_DAY: Final[int] = os.getenv("LAST_GARBAGE_DAY", 7)
# LAST_VERIFY_DAY: Final[int] = int(os.getenv("LAST_VERIFY_DAY", 7))
# LAST_GARBAGE_DAY: Final[int] = int(os.getenv("LAST_GARBAGE_DAY", 7))
HOST: Final[str] = os.getenv("HOST", None)
HOST_SSH_PORT: Final[int] = os.getenv("HOST_SSH_PORT", None)
HOST_SSH_PORT: Final[int] = int(os.getenv("HOST_SSH_PORT", 22))
HOST_USR: Final[str] = os.getenv("HOST_USR", "root")
HOST_USR_PWD: Final[str] = os.getenv("HOST_USR_PWD", None)
HOST_USR_SSHKEY: Final[str] = os.getenv("HOST_USR_SSHKEY", None)
HOST_USR_SSHKEY_PASS: Final[str] = os.getenv("HOST_USR_SSHKEY_PASS", None)
LOCAL_DATASTORE: Final[str] = os.getenv("LOCAL_DATASTORE", None)
REMOTE_DATASTORE: Final[str] = os.getenv("HOST_UREMOTE_DATASTORESR_SSHKEY_PASS", None)
REMOTE_DATASTORE: Final[str] = os.getenv("REMOTE_DATASTORE", None)
REMOTE_PBS: Final[str] = os.getenv("REMOTE_PBS", None)
SYNC_DIR: Final[str] = os.getenv("SYNC_DIR", "push")
PBS_RATE_OUT: Final[str] = os.getenv("PBS_RATE_OUT", "")
PBS_RATE_IN: Final[str] = os.getenv("PBS_RATE_IN", "")
PBS_TASK_LIMIT: Final[int] = os.getenv("PBS_TASK_LIMIT", 50)
PBS_MAX_RUNTIME_MIN: Final[int] = os.getenv("PBS_MAX_RUNTIME_MIN", 180)
PBS_MAX_SHUT_MIN: Final[int] = os.getenv("PBS_MAX_SHUT_MIN", 10)
PBS_TASK_LIMIT: Final[int] = int(os.getenv("PBS_TASK_LIMIT", 50))
PBS_MAX_RUNTIME_MIN: Final[int] = int(os.getenv("PBS_MAX_RUNTIME_MIN", 180))
PBS_MAX_SHUT_MIN: Final[int] = int(os.getenv("PBS_MAX_SHUT_MIN", 10))
AUTO_ADD_NAME: Final[str] = os.getenv("AUTO_ADD_NAME", "autoBackupScript_one-shot")
RUN_SCHEDULE: Final[str] = os.getenv("RUN_SCHEDULE", "45 17 * * 5")
RUN_SCHEDULE_TZ: Final[str] = os.getenv("RUN_SCHEDULE_TZ", "Europe/Stockholm")
@@ -59,15 +61,16 @@ def load_private_key(key_data: str, passphrase: str | None = None):
raise ValueError("Unsupported or invalid SSH private key")
def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
def getCurrentTasks(sshThingy, allTasks: bool = False, limitOfTasks: int =50) -> dict:
"""Get what tasks are running/ran on PBS
Keyword arguments:
sshThingy -- a paramiko SSH object (no default)
allTasks -- if tasks that are finished should be included. (default False)
limitOfTasks -- how many tasks to ask PBS for (default 50)
"""
(stdin, stdout, stderr) = ssh.exec_command(f'proxmox-backup-manager task list --limit {limitOfTasks} --all {allTasks}')
(stdin, stdout, stderr) = sshThingy.exec_command(f'proxmox-backup-manager task list --limit {limitOfTasks} --all {allTasks}')
taskList = stdout.read().decode()
@@ -84,8 +87,11 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
# Skip the header row
if columns[0] == "starttime":
continue
if len(columns) >= 2:
if len(columns) >= 3:
try:
outTaskList.append(columns[2].split(":")[6])
except IndexError:
raise exceptionator(ConnectionError, "Incorrect message received from PBS")
# exit()
@@ -132,7 +138,7 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
if len(AUTO_ADD_NAME) > 32:
raise exceptionator(ValueError, "Maxmimum length of AUTO_ADD_NAME is 32")
sys.exit(1)
# sys.exit(1)
@@ -143,23 +149,37 @@ if len(AUTO_ADD_NAME) > 32:
# dt = datetime.now.timestamp()
# epoch_time = dt.timestamp()
# print(epoch_time)
def theCodeBasically():
# Check if power is on
# PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
print("\nStarting backup sequence\n")
try:
# power on if needed
if "off" in str(PWR_STATE.stdout.decode()[-4:]):
subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"]) # , capture_output=True
except exception as e:
print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
sys.exit(1)
# else:
# continue
# dt = datetime.now.timestamp()
# epoch_time = dt.timestamp()
# print(epoch_time)
# Check if power is on
PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "status",],capture_output=True,text=True,)
if PWR_STATE.returncode != 0:
raise RuntimeError(f"ipmitool failed: {PWR_STATE.stderr.strip()}")
if "off" in PWR_STATE.stdout.lower():
subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "on",],check=True,)
# PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
# try:
# # power on if needed
# if "off" in str(PWR_STATE.stdout.decode()[-4:]):
# subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"]) # , capture_output=True
# except Exception as e:
# print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
# sys.exit(1)
# else:
# continue
@@ -167,11 +187,11 @@ except exception as e:
PWR_DATE = datetime.now().timestamp()
PWR_DATE = datetime.now().timestamp()
pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
if pinging:
if pinging:
print("not reachable")
while datetime.now().timestamp() - PWR_DATE <= 1200:
@@ -182,9 +202,9 @@ if pinging:
break
else:
raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.")
sys.exit(1)
# sys.exit(1)
else:
else:
print("reachable")
@@ -195,21 +215,21 @@ else:
# Create object of SSHClient and
# connecting to SSH
ssh = paramiko.SSHClient()
# Create object of SSHClient and
# connecting to SSH
ssh = paramiko.SSHClient()
# Adding new host key to the local
# HostKeys object(in case of missing)
# AutoAddPolicy for missing host key to be set before connection setup.
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Adding new host key to the local
# HostKeys object(in case of missing)
# AutoAddPolicy for missing host key to be set before connection setup.
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if not HOST_USR_SSHKEY:
if not HOST_USR_SSHKEY:
ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, )
else:
else:
key = load_private_key(
HOST_USR_SSHKEY,
HOST_USR_SSHKEY_PASS
@@ -229,23 +249,23 @@ else:
# Execute command on SSH terminal
# using exec_command
# (stdin, stdout, stderr) = ssh.exec_command('hostname')
# Execute command on SSH terminal
# using exec_command
# (stdin, stdout, stderr) = ssh.exec_command('hostname')
# # redirecting all the output in cmd_output
# # variable
# cmd_output = stdout.read().decode()
# # redirecting all the output in cmd_output
# # variable
# cmd_output = stdout.read().decode()
# print(cmd_output)
# print(cmd_output)
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
remoteList = stdout.read().decode()
remoteList = stdout.read().decode()
remoteDict = {}
remoteDict = {}
for line in remoteList.splitlines():
for line in remoteList.splitlines():
line = line.strip()
# Only process actual table rows
@@ -266,36 +286,39 @@ for line in remoteList.splitlines():
if REMOTE_PBS in remoteDict:
if REMOTE_PBS in remoteDict:
print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}")
else:
else:
print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr)
REMOTE_PBS.remove(REMOTE_PBS)
sys.exit(1)
raise RuntimeError(f"Remote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.")
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
syncJobList = stdout.read().decode()
syncJobList = stdout.read().decode()
sync_job_exists = False
for line in syncJobList.splitlines():
for line in syncJobList.splitlines():
line = line.strip()
# Only process actual table rows
if not line.startswith("│"):
continue
columns = [col.strip() for col in line.strip("│").split("│")]
# Skip the header row
if columns[0] == "id":
continue
if len(columns) >= 2:
print("Sync job already on Proxmox Backup Server.\nSkipping sync job creation.")
# print(columns)
if columns[0] == AUTO_ADD_NAME:
sync_job_exists = True
break
else:
if sync_job_exists:
print(f"Sync job {AUTO_ADD_NAME} already exists.")
# create it
else:
match SYNC_DIR:
case "push":
@@ -309,47 +332,58 @@ else:
else:
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --comment 'automatic startupper script'")
case _:
raise exceptionator(SyncDirectionError, f"No such sync direction as {SYNC_DIR}.\n Either use 'push' or 'pull'\n")
raise ValueError(f"Invalid sync direction: {SYNC_DIR}. Use 'push' or 'pull'.")
# print(stdout.read().decode())
syncJobCreateExitCode = stdout.channel.recv_exit_status()
if syncJobCreateExitCode != 0:
raise RuntimeError(f"Failed to create sync job: {stderr.read().decode()}")
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
# ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
# print("here it runs the sync-job")
# print(stdout.read().decode())
# print(stderr.read().decode())
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
# ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
# print("here it runs the sync-job")
# print(stdout.read().decode())
# print(stderr.read().decode())
syncJobRunExitCode = stdout.channel.recv_exit_status()
if syncJobRunExitCode != 0:
raise RuntimeError(f"Sync job failed: {stderr.read().decode()}")
# Give some time for PBS to "start" the sync-job task
time.sleep(10)
# Maybe add some checks here if the backup was properly scheduled
# Get tasks
# print(getCurrentTasks(limitOfTasks=100))
# Maybe add some checks here if the backup was properly scheduled
# check ongoing tasks
SYNC_JOB_DATE = time.monotonic() #datetime.now().timestamp()
currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT)
# Get tasks
# print(getCurrentTasks(limitOfTasks=100))
# check ongoing tasks
SYNC_JOB_DATE = datetime.now().timestamp()
currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
while currenttasks:
currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
while currenttasks:
print("Waiting 1 minute before checking tasks again")
time.sleep(60)
if SYNC_JOB_DATE - datetime.now().timestamp() >= (PBS_MAX_RUNTIME_MIN * 60):
print(SYNC_JOB_DATE - datetime.now().timestamp())
currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT)
if time.monotonic() - SYNC_JOB_DATE >= (PBS_MAX_RUNTIME_MIN * 60):
# print(SYNC_JOB_DATE - time.monotonic())
# Forcefully skip if these tasks are running. reader task is, I think, PVE backup
# Also otherjob is something else and it is better to wait if there is one otherjob
if (
@@ -363,24 +397,48 @@ while currenttasks:
break
(stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
print("Shuting down PBS")
# print("here it runs a shutdown now")
print(stdout.read().decode())
print(stderr.read().decode(), file=sys.stderr)
(stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
print("Shuting down PBS")
# print("here it runs a shutdown now")
print(stdout.read().decode())
print(stderr.read().decode(), file=sys.stderr)
ssh.close()
# wait for PBS to safely shutdown
time.sleep(PBS_MAX_SHUT_MIN * 60)
ssh.close()
# wait for PBS to safely shutdown
time.sleep(PBS_MAX_SHUT_MIN * 60)
if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
print("PBS is powered off")
else:
else:
subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"])
sys.exit(0)
# sys.exit(0)
scheduler = BlockingScheduler()
scheduler.add_job(
theCodeBasically,
CronTrigger.from_crontab(
RUN_SCHEDULE,
timezone=RUN_SCHEDULE_TZ,
),
max_instances=1,
coalesce=True,
)
scheduler.start()

1
requirements.txt
View File

@@ -1 +1,2 @@
paramiko==5.0.0
APScheduler==3.11.3