diff --git a/main.py b/main.py index 06eb585..0f20acf 100644 --- a/main.py +++ b/main.py @@ -1,35 +1,37 @@ import os, socket, subprocess, paramiko, io, sys, time from typing import Final from datetime import datetime +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.triggers.cron import CronTrigger IDRAC: Final[str] = os.getenv("IDRAC", None) IDRAC_USR: Final[str] = os.getenv("IDRAC_USR", None) IDRAC_PWD: Final[str] = os.getenv("IDRAC_PWD", None) -# LAST_VERIFY_DAY: Final[int] = os.getenv("LAST_VERIFY_DAY", 7) -# LAST_GARBAGE_DAY: Final[int] = os.getenv("LAST_GARBAGE_DAY", 7) +# LAST_VERIFY_DAY: Final[int] = int(os.getenv("LAST_VERIFY_DAY", 7)) +# LAST_GARBAGE_DAY: Final[int] = int(os.getenv("LAST_GARBAGE_DAY", 7)) HOST: Final[str] = os.getenv("HOST", None) -HOST_SSH_PORT: Final[int] = os.getenv("HOST_SSH_PORT", None) +HOST_SSH_PORT: Final[int] = int(os.getenv("HOST_SSH_PORT", 22)) HOST_USR: Final[str] = os.getenv("HOST_USR", "root") HOST_USR_PWD: Final[str] = os.getenv("HOST_USR_PWD", None) HOST_USR_SSHKEY: Final[str] = os.getenv("HOST_USR_SSHKEY", None) HOST_USR_SSHKEY_PASS: Final[str] = os.getenv("HOST_USR_SSHKEY_PASS", None) LOCAL_DATASTORE: Final[str] = os.getenv("LOCAL_DATASTORE", None) -REMOTE_DATASTORE: Final[str] = os.getenv("HOST_UREMOTE_DATASTORESR_SSHKEY_PASS", None) +REMOTE_DATASTORE: Final[str] = os.getenv("REMOTE_DATASTORE", None) REMOTE_PBS: Final[str] = os.getenv("REMOTE_PBS", None) SYNC_DIR: Final[str] = os.getenv("SYNC_DIR", "push") PBS_RATE_OUT: Final[str] = os.getenv("PBS_RATE_OUT", "") PBS_RATE_IN: Final[str] = os.getenv("PBS_RATE_IN", "") -PBS_TASK_LIMIT: Final[int] = os.getenv("PBS_TASK_LIMIT", 50) -PBS_MAX_RUNTIME_MIN: Final[int] = os.getenv("PBS_MAX_RUNTIME_MIN", 180) -PBS_MAX_SHUT_MIN: Final[int] = os.getenv("PBS_MAX_SHUT_MIN", 10) +PBS_TASK_LIMIT: Final[int] = int(os.getenv("PBS_TASK_LIMIT", 50)) +PBS_MAX_RUNTIME_MIN: Final[int] = int(os.getenv("PBS_MAX_RUNTIME_MIN", 180)) +PBS_MAX_SHUT_MIN: Final[int] = int(os.getenv("PBS_MAX_SHUT_MIN", 10)) AUTO_ADD_NAME: Final[str] = os.getenv("AUTO_ADD_NAME", "autoBackupScript_one-shot") - - +RUN_SCHEDULE: Final[str] = os.getenv("RUN_SCHEDULE", "45 17 * * 5") +RUN_SCHEDULE_TZ: Final[str] = os.getenv("RUN_SCHEDULE_TZ", "Europe/Stockholm") @@ -59,15 +61,16 @@ def load_private_key(key_data: str, passphrase: str | None = None): raise ValueError("Unsupported or invalid SSH private key") -def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict: +def getCurrentTasks(sshThingy, allTasks: bool = False, limitOfTasks: int =50) -> dict: """Get what tasks are running/ran on PBS Keyword arguments: + sshThingy -- a paramiko SSH object (no default) allTasks -- if tasks that are finished should be included. (default False) limitOfTasks -- how many tasks to ask PBS for (default 50) """ - (stdin, stdout, stderr) = ssh.exec_command(f'proxmox-backup-manager task list --limit {limitOfTasks} --all {allTasks}') + (stdin, stdout, stderr) = sshThingy.exec_command(f'proxmox-backup-manager task list --limit {limitOfTasks} --all {allTasks}') taskList = stdout.read().decode() @@ -84,8 +87,11 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict: # Skip the header row if columns[0] == "starttime": continue - if len(columns) >= 2: - outTaskList.append(columns[2].split(":")[6]) + if len(columns) >= 3: + try: + outTaskList.append(columns[2].split(":")[6]) + except IndexError: + raise exceptionator(ConnectionError, "Incorrect message received from PBS") # exit() @@ -132,7 +138,7 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict: if len(AUTO_ADD_NAME) > 32: raise exceptionator(ValueError, "Maxmimum length of AUTO_ADD_NAME is 32") - sys.exit(1) + # sys.exit(1) @@ -143,23 +149,37 @@ if len(AUTO_ADD_NAME) > 32: -# dt = datetime.now.timestamp() -# epoch_time = dt.timestamp() -# print(epoch_time) +def theCodeBasically(): -# Check if power is on -# PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True) + print("\nStarting backup sequence\n") -try: - # power on if needed - if "off" in str(PWR_STATE.stdout.decode()[-4:]): - subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"]) # , capture_output=True -except exception as e: - print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n") - sys.exit(1) -# else: -# continue + # dt = datetime.now.timestamp() + + # epoch_time = dt.timestamp() + # print(epoch_time) + + # Check if power is on + PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "status",],capture_output=True,text=True,) + + if PWR_STATE.returncode != 0: + raise RuntimeError(f"ipmitool failed: {PWR_STATE.stderr.strip()}") + + if "off" in PWR_STATE.stdout.lower(): + subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "on",],check=True,) + + + # PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True) + + # try: + # # power on if needed + # if "off" in str(PWR_STATE.stdout.decode()[-4:]): + # subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"]) # , capture_output=True + # except Exception as e: + # print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n") + # sys.exit(1) + # else: + # continue @@ -167,25 +187,26 @@ except exception as e: -PWR_DATE = datetime.now().timestamp() + PWR_DATE = datetime.now().timestamp() -pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode + pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode -if pinging: - print("not reachable") - - while datetime.now().timestamp() - PWR_DATE <= 1200: - pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode + if pinging: + print("not reachable") + + while datetime.now().timestamp() - PWR_DATE <= 1200: + pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode + + if not pinging: + print("reachable") + break + else: + raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.") + # sys.exit(1) - if not pinging: - print("reachable") - break else: - raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.") - sys.exit(1) + print("reachable") -else: - print("reachable") @@ -194,193 +215,230 @@ else: + # Create object of SSHClient and + # connecting to SSH + ssh = paramiko.SSHClient() -# Create object of SSHClient and -# connecting to SSH -ssh = paramiko.SSHClient() + # Adding new host key to the local + # HostKeys object(in case of missing) + # AutoAddPolicy for missing host key to be set before connection setup. + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) -# Adding new host key to the local -# HostKeys object(in case of missing) -# AutoAddPolicy for missing host key to be set before connection setup. -ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + if not HOST_USR_SSHKEY: -if not HOST_USR_SSHKEY: + ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, ) - ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, ) + else: + key = load_private_key( + HOST_USR_SSHKEY, + HOST_USR_SSHKEY_PASS + ) -else: - key = load_private_key( - HOST_USR_SSHKEY, - HOST_USR_SSHKEY_PASS - ) + ssh.connect( + hostname=HOST, + port=HOST_SSH_PORT, + username=HOST_USR, + pkey=key, + timeout=3, + allow_agent=False, + look_for_keys=False, + + ) - ssh.connect( - hostname=HOST, - port=HOST_SSH_PORT, - username=HOST_USR, - pkey=key, - timeout=3, - allow_agent=False, - look_for_keys=False, - - ) + # Execute command on SSH terminal + # using exec_command + # (stdin, stdout, stderr) = ssh.exec_command('hostname') -# Execute command on SSH terminal -# using exec_command -# (stdin, stdout, stderr) = ssh.exec_command('hostname') + # # redirecting all the output in cmd_output + # # variable + # cmd_output = stdout.read().decode() -# # redirecting all the output in cmd_output -# # variable -# cmd_output = stdout.read().decode() + # print(cmd_output) -# print(cmd_output) + (stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list') -(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list') + remoteList = stdout.read().decode() -remoteList = stdout.read().decode() + remoteDict = {} -remoteDict = {} + for line in remoteList.splitlines(): + line = line.strip() -for line in remoteList.splitlines(): - line = line.strip() - - # Only process actual table rows - if not line.startswith("│"): - continue - - columns = [col.strip() for col in line.strip("│").split("│")] - - # Skip the header row - if columns[0] == "name": - continue - - if len(columns) >= 2: - name = columns[0] - host = columns[1] - - remoteDict[host] = name - - - -if REMOTE_PBS in remoteDict: - print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}") -else: - print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr) - REMOTE_PBS.remove(REMOTE_PBS) - sys.exit(1) - - -(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all') - -syncJobList = stdout.read().decode() - - -for line in syncJobList.splitlines(): - line = line.strip() - - # Only process actual table rows - if not line.startswith("│"): - continue - - columns = [col.strip() for col in line.strip("│").split("│")] - - # Skip the header row - if columns[0] == "id": - continue - if len(columns) >= 2: - print("Sync job already on Proxmox Backup Server.\nSkipping sync job creation.") - # print(columns) - break -else: - - match SYNC_DIR: - case "push": - if PBS_RATE_OUT: - (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --rate-out {PBS_RATE_OUT} --comment 'automatic startupper script'") - else: - (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --comment 'automatic startupper script'") - case "pull": - if PBS_RATE_IN: - (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --rate-in {PBS_RATE_IN} --comment 'automatic startupper script'") - else: - (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --comment 'automatic startupper script'") - case _: - raise exceptionator(SyncDirectionError, f"No such sync direction as {SYNC_DIR}.\n Either use 'push' or 'pull'\n") - - -# print(stdout.read().decode()) - - - - -(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}") -# ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}") -# print("here it runs the sync-job") -# print(stdout.read().decode()) -# print(stderr.read().decode()) - - - - -# Maybe add some checks here if the backup was properly scheduled - - - -# Get tasks -# print(getCurrentTasks(limitOfTasks=100)) - - - - -# check ongoing tasks -SYNC_JOB_DATE = datetime.now().timestamp() - -currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT) - -while currenttasks: - - currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT) - - print("Waiting 1 minute before checking tasks again") - time.sleep(60) - - if SYNC_JOB_DATE - datetime.now().timestamp() >= (PBS_MAX_RUNTIME_MIN * 60): - print(SYNC_JOB_DATE - datetime.now().timestamp()) - # Forcefully skip if these tasks are running. reader task is, I think, PVE backup - # Also otherjob is something else and it is better to wait if there is one otherjob - if ( - currenttasks.get("aptupdate", 0) - or currenttasks.get("reader", 0) - or currenttasks.get("otherjob", 0) - ): + # Only process actual table rows + if not line.startswith("│"): continue - # No need to do anything here. It's only to check if these keys exist - ... - break + + columns = [col.strip() for col in line.strip("│").split("│")] + + # Skip the header row + if columns[0] == "name": + continue + + if len(columns) >= 2: + name = columns[0] + host = columns[1] + + remoteDict[host] = name + + + + if REMOTE_PBS in remoteDict: + print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}") + else: + print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr) + raise RuntimeError(f"Remote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.") + + + (stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all') + + syncJobList = stdout.read().decode() + + sync_job_exists = False + + for line in syncJobList.splitlines(): + line = line.strip() + + if not line.startswith("│"): + continue + + columns = [col.strip() for col in line.strip("│").split("│")] + + if columns[0] == "id": + continue + + if columns[0] == AUTO_ADD_NAME: + sync_job_exists = True + break + + if sync_job_exists: + print(f"Sync job {AUTO_ADD_NAME} already exists.") + + # create it + else: + + match SYNC_DIR: + case "push": + if PBS_RATE_OUT: + (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --rate-out {PBS_RATE_OUT} --comment 'automatic startupper script'") + else: + (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --comment 'automatic startupper script'") + case "pull": + if PBS_RATE_IN: + (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --rate-in {PBS_RATE_IN} --comment 'automatic startupper script'") + else: + (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --comment 'automatic startupper script'") + case _: + raise ValueError(f"Invalid sync direction: {SYNC_DIR}. Use 'push' or 'pull'.") + + + + syncJobCreateExitCode = stdout.channel.recv_exit_status() + + if syncJobCreateExitCode != 0: + raise RuntimeError(f"Failed to create sync job: {stderr.read().decode()}") + + + + + (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}") + # ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}") + # print("here it runs the sync-job") + # print(stdout.read().decode()) + # print(stderr.read().decode()) + + syncJobRunExitCode = stdout.channel.recv_exit_status() + + if syncJobRunExitCode != 0: + raise RuntimeError(f"Sync job failed: {stderr.read().decode()}") + + # Give some time for PBS to "start" the sync-job task + time.sleep(10) + + + # Maybe add some checks here if the backup was properly scheduled + + + + # Get tasks + # print(getCurrentTasks(limitOfTasks=100)) + + + + + # check ongoing tasks + SYNC_JOB_DATE = time.monotonic() #datetime.now().timestamp() + + currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT) + + while currenttasks: + + print("Waiting 1 minute before checking tasks again") + time.sleep(60) + + currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT) + + + if time.monotonic() - SYNC_JOB_DATE >= (PBS_MAX_RUNTIME_MIN * 60): + # print(SYNC_JOB_DATE - time.monotonic()) + # Forcefully skip if these tasks are running. reader task is, I think, PVE backup + # Also otherjob is something else and it is better to wait if there is one otherjob + if ( + currenttasks.get("aptupdate", 0) + or currenttasks.get("reader", 0) + or currenttasks.get("otherjob", 0) + ): + continue + # No need to do anything here. It's only to check if these keys exist + ... + break + + + (stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown") + print("Shuting down PBS") + # print("here it runs a shutdown now") + print(stdout.read().decode()) + print(stderr.read().decode(), file=sys.stderr) + + ssh.close() + # wait for PBS to safely shutdown + time.sleep(PBS_MAX_SHUT_MIN * 60) + + + if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode: + print("PBS is powered off") + + else: + subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"]) + + # sys.exit(0) + + + + + +scheduler = BlockingScheduler() +scheduler.add_job( + theCodeBasically, + CronTrigger.from_crontab( + RUN_SCHEDULE, + timezone=RUN_SCHEDULE_TZ, + ), + max_instances=1, + coalesce=True, +) + +scheduler.start() -(stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown") -print("Shuting down PBS") -# print("here it runs a shutdown now") -print(stdout.read().decode()) -print(stderr.read().decode(), file=sys.stderr) -ssh.close() -# wait for PBS to safely shutdown -time.sleep(PBS_MAX_SHUT_MIN * 60) -if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode: - print("PBS is powered off") -else: - subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"]) -sys.exit(0) diff --git a/requirements.txt b/requirements.txt index 80db7d1..2517d38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -paramiko==5.0.0 \ No newline at end of file +paramiko==5.0.0 +APScheduler==3.11.3 \ No newline at end of file