v0.1 of APscheduler implementation

main.py

@@ -1,35 +1,37 @@
 import os, socket, subprocess, paramiko, io, sys, time
 from typing import Final
 from datetime import datetime
+from apscheduler.schedulers.blocking import BlockingScheduler
+from apscheduler.triggers.cron import CronTrigger
 
 
 IDRAC: Final[str] = os.getenv("IDRAC", None)
 IDRAC_USR: Final[str] = os.getenv("IDRAC_USR", None)
 IDRAC_PWD: Final[str] = os.getenv("IDRAC_PWD", None)
 
-# LAST_VERIFY_DAY: Final[int] = os.getenv("LAST_VERIFY_DAY", 7)
+# LAST_VERIFY_DAY: Final[int] = int(os.getenv("LAST_VERIFY_DAY", 7))
-# LAST_GARBAGE_DAY: Final[int] = os.getenv("LAST_GARBAGE_DAY", 7)
+# LAST_GARBAGE_DAY: Final[int] = int(os.getenv("LAST_GARBAGE_DAY", 7))
 
 HOST: Final[str] = os.getenv("HOST", None)
-HOST_SSH_PORT: Final[int] = os.getenv("HOST_SSH_PORT", None)
+HOST_SSH_PORT: Final[int] = int(os.getenv("HOST_SSH_PORT", 22))
 HOST_USR: Final[str] = os.getenv("HOST_USR", "root")
 HOST_USR_PWD: Final[str] = os.getenv("HOST_USR_PWD", None)
 HOST_USR_SSHKEY: Final[str] = os.getenv("HOST_USR_SSHKEY", None)
 HOST_USR_SSHKEY_PASS: Final[str] = os.getenv("HOST_USR_SSHKEY_PASS", None)
 
 LOCAL_DATASTORE: Final[str] = os.getenv("LOCAL_DATASTORE", None)
-REMOTE_DATASTORE: Final[str] = os.getenv("HOST_UREMOTE_DATASTORESR_SSHKEY_PASS", None)
+REMOTE_DATASTORE: Final[str] = os.getenv("REMOTE_DATASTORE", None)
 REMOTE_PBS: Final[str] = os.getenv("REMOTE_PBS", None)
 SYNC_DIR: Final[str] = os.getenv("SYNC_DIR", "push")
 PBS_RATE_OUT: Final[str] = os.getenv("PBS_RATE_OUT", "")
 PBS_RATE_IN: Final[str] = os.getenv("PBS_RATE_IN", "")
-PBS_TASK_LIMIT: Final[int] = os.getenv("PBS_TASK_LIMIT", 50)
+PBS_TASK_LIMIT: Final[int] = int(os.getenv("PBS_TASK_LIMIT", 50))
-PBS_MAX_RUNTIME_MIN: Final[int] = os.getenv("PBS_MAX_RUNTIME_MIN", 180)
+PBS_MAX_RUNTIME_MIN: Final[int] = int(os.getenv("PBS_MAX_RUNTIME_MIN", 180))
-PBS_MAX_SHUT_MIN: Final[int] = os.getenv("PBS_MAX_SHUT_MIN", 10)
+PBS_MAX_SHUT_MIN: Final[int] = int(os.getenv("PBS_MAX_SHUT_MIN", 10))
 AUTO_ADD_NAME: Final[str] = os.getenv("AUTO_ADD_NAME", "autoBackupScript_one-shot")
 
-
+RUN_SCHEDULE: Final[str] = os.getenv("RUN_SCHEDULE", "45 17 * * 5")
-
+RUN_SCHEDULE_TZ: Final[str] = os.getenv("RUN_SCHEDULE_TZ", "Europe/Stockholm")
 
 
 
@@ -59,15 +61,16 @@ def load_private_key(key_data: str, passphrase: str | None = None):
     raise ValueError("Unsupported or invalid SSH private key")
 
 
-def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
+def getCurrentTasks(sshThingy, allTasks: bool = False, limitOfTasks: int =50) -> dict:
     """Get what tasks are running/ran on PBS
     
     Keyword arguments:
+    sshThingy -- a paramiko SSH object (no default)
     allTasks -- if tasks that are finished should be included. (default False)
     limitOfTasks -- how many tasks to ask PBS for (default 50)
     """
 
-    (stdin, stdout, stderr) = ssh.exec_command(f'proxmox-backup-manager task list  --limit {limitOfTasks} --all {allTasks}')
+    (stdin, stdout, stderr) = sshThingy.exec_command(f'proxmox-backup-manager task list  --limit {limitOfTasks} --all {allTasks}')
 
     taskList = stdout.read().decode()
 
@@ -84,8 +87,11 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
         # Skip the header row
         if columns[0] == "starttime":
             continue
-        if len(columns) >= 2:
+        if len(columns) >= 3:
+            try:
                 outTaskList.append(columns[2].split(":")[6])
+            except IndexError:
+                raise exceptionator(ConnectionError, "Incorrect message received from PBS")
             # exit()
 
 
@@ -132,7 +138,7 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
 
 if len(AUTO_ADD_NAME) > 32:
     raise exceptionator(ValueError, "Maxmimum length of AUTO_ADD_NAME is 32")
-    sys.exit(1)
+    # sys.exit(1)
 
 
 
@@ -143,23 +149,37 @@ if len(AUTO_ADD_NAME) > 32:
 
 
 
-# dt = datetime.now.timestamp()
 
-# epoch_time = dt.timestamp()
+def theCodeBasically():
-# print(epoch_time)
 
-# Check if power is on
+    print("\nStarting backup sequence\n")
-# PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
 
-try: 
+    # dt = datetime.now.timestamp()
-    # power on if needed
+
-    if "off" in str(PWR_STATE.stdout.decode()[-4:]):
+    # epoch_time = dt.timestamp()
-        subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"])  # , capture_output=True
+    # print(epoch_time)
-except exception as e:
+
-    print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
+    # Check if power is on
-    sys.exit(1)
+    PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "status",],capture_output=True,text=True,)
-# else:
+
-#     continue
+    if PWR_STATE.returncode != 0:
+        raise RuntimeError(f"ipmitool failed: {PWR_STATE.stderr.strip()}")
+
+    if "off" in PWR_STATE.stdout.lower():
+        subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "on",],check=True,)
+
+
+    # PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
+
+    # try: 
+    #     # power on if needed
+    #     if "off" in str(PWR_STATE.stdout.decode()[-4:]):
+    #         subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"])  # , capture_output=True
+    # except Exception as e:
+    #     print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
+    #     sys.exit(1)
+    # else:
+    #     continue
 
 
 
@@ -167,11 +187,11 @@ except exception as e:
 
 
 
-PWR_DATE = datetime.now().timestamp()
+    PWR_DATE = datetime.now().timestamp()
 
-pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
+    pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
 
-if pinging:
+    if pinging:
         print("not reachable")
         
         while datetime.now().timestamp() - PWR_DATE <= 1200:
@@ -182,9 +202,9 @@ if pinging:
                 break
         else:
             raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.")
-        sys.exit(1)
+            # sys.exit(1)
 
-else:
+    else:
         print("reachable")
 
 
@@ -195,21 +215,21 @@ else:
 
 
 
-# Create object of SSHClient and 
+    # Create object of SSHClient and 
-# connecting to SSH
+    # connecting to SSH
-ssh = paramiko.SSHClient()
+    ssh = paramiko.SSHClient()
 
-# Adding new host key to the local 
+    # Adding new host key to the local 
-# HostKeys object(in case of missing)
+    # HostKeys object(in case of missing)
-# AutoAddPolicy for missing host key to be set before connection setup.
+    # AutoAddPolicy for missing host key to be set before connection setup.
-ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
 
-if not HOST_USR_SSHKEY:
+    if not HOST_USR_SSHKEY:
 
         ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, )
 
-else:
+    else:
         key = load_private_key(
             HOST_USR_SSHKEY,
             HOST_USR_SSHKEY_PASS
@@ -229,23 +249,23 @@ else:
 
 
 
-# Execute command on SSH terminal 
+    # Execute command on SSH terminal 
-# using exec_command
+    # using exec_command
-# (stdin, stdout, stderr) = ssh.exec_command('hostname')
+    # (stdin, stdout, stderr) = ssh.exec_command('hostname')
 
-# # redirecting all the output in cmd_output 
+    # # redirecting all the output in cmd_output 
-# # variable
+    # # variable
-# cmd_output = stdout.read().decode()
+    # cmd_output = stdout.read().decode()
 
-# print(cmd_output)
+    # print(cmd_output)
 
-(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
+    (stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
 
-remoteList = stdout.read().decode()
+    remoteList = stdout.read().decode()
 
-remoteDict = {}
+    remoteDict = {}
 
-for line in remoteList.splitlines():
+    for line in remoteList.splitlines():
         line = line.strip()
 
         # Only process actual table rows
@@ -266,36 +286,39 @@ for line in remoteList.splitlines():
 
 
 
-if REMOTE_PBS in remoteDict:
+    if REMOTE_PBS in remoteDict:
         print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}")
-else:
+    else:
         print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr)
-    REMOTE_PBS.remove(REMOTE_PBS)
+        raise RuntimeError(f"Remote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.")
-    sys.exit(1)
 
 
-(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
+    (stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
 
-syncJobList = stdout.read().decode()
+    syncJobList = stdout.read().decode()
 
+    sync_job_exists = False
 
-for line in syncJobList.splitlines():
+    for line in syncJobList.splitlines():
         line = line.strip()
 
-    # Only process actual table rows
         if not line.startswith("│"):
             continue
 
         columns = [col.strip() for col in line.strip("│").split("│")]
 
-    # Skip the header row
         if columns[0] == "id":
             continue
-    if len(columns) >= 2:
+
-        print("Sync job already on Proxmox Backup Server.\nSkipping sync job creation.")
+        if columns[0] == AUTO_ADD_NAME:
-        # print(columns)
+            sync_job_exists = True
             break
-else:
+
+    if sync_job_exists:
+        print(f"Sync job {AUTO_ADD_NAME} already exists.")
+    
+    # create it
+    else:
 
         match SYNC_DIR:
             case "push":
@@ -309,47 +332,58 @@ else:
                 else:
                     (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE}    --sync-direction pull --comment 'automatic startupper script'")
             case _:
-            raise exceptionator(SyncDirectionError, f"No such sync direction as {SYNC_DIR}.\n Either use 'push' or 'pull'\n")
+                raise ValueError(f"Invalid sync direction: {SYNC_DIR}. Use 'push' or 'pull'.")
 
 
-# print(stdout.read().decode())
+        
+        syncJobCreateExitCode = stdout.channel.recv_exit_status()
+
+        if syncJobCreateExitCode != 0:
+            raise RuntimeError(f"Failed to create sync job: {stderr.read().decode()}")
 
 
 
 
-(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
+    (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
-# ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
+    # ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
-# print("here it runs the sync-job")
+    # print("here it runs the sync-job")
-# print(stdout.read().decode())
+    # print(stdout.read().decode())
-# print(stderr.read().decode())
+    # print(stderr.read().decode())
+
+    syncJobRunExitCode = stdout.channel.recv_exit_status()
+
+    if syncJobRunExitCode != 0:
+        raise RuntimeError(f"Sync job failed: {stderr.read().decode()}")
+
+    # Give some time for PBS to "start" the sync-job task
+    time.sleep(10)
+
+
+    # Maybe add some checks here if the backup was properly scheduled
+
+
+
+    # Get tasks
+    # print(getCurrentTasks(limitOfTasks=100))
 
 
 
 
-# Maybe add some checks here if the backup was properly scheduled
+    # check ongoing tasks
+    SYNC_JOB_DATE = time.monotonic()    #datetime.now().timestamp()
 
+    currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT)
 
-
+    while currenttasks:
-# Get tasks
-# print(getCurrentTasks(limitOfTasks=100))
-
-
-
-
-# check ongoing tasks
-SYNC_JOB_DATE = datetime.now().timestamp()
-
-currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
-
-while currenttasks:
-
-    currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
 
         print("Waiting 1 minute before checking tasks again")
         time.sleep(60)
         
-    if SYNC_JOB_DATE - datetime.now().timestamp() >= (PBS_MAX_RUNTIME_MIN * 60):
+        currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT)
-        print(SYNC_JOB_DATE - datetime.now().timestamp())
+
+
+        if time.monotonic() - SYNC_JOB_DATE >= (PBS_MAX_RUNTIME_MIN * 60):
+            # print(SYNC_JOB_DATE - time.monotonic())
             # Forcefully skip if these tasks are running. reader task is, I think, PVE backup
             # Also otherjob is something else and it is better to wait if there is one otherjob
             if (
@@ -363,24 +397,48 @@ while currenttasks:
             break
 
 
-(stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
+    (stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
-print("Shuting down PBS")
+    print("Shuting down PBS")
-# print("here it runs a shutdown now")
+    # print("here it runs a shutdown now")
-print(stdout.read().decode())
+    print(stdout.read().decode())
-print(stderr.read().decode(), file=sys.stderr)
+    print(stderr.read().decode(), file=sys.stderr)
 
-ssh.close()
+    ssh.close()
-# wait for PBS to safely shutdown
+    # wait for PBS to safely shutdown
-time.sleep(PBS_MAX_SHUT_MIN * 60)
+    time.sleep(PBS_MAX_SHUT_MIN * 60)
 
 
-if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
+    if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
         print("PBS is powered off")
 
-else:
+    else:
         subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"])
 
-sys.exit(0)
+    # sys.exit(0)
+
+
+
+
+
+scheduler = BlockingScheduler()
+scheduler.add_job(
+    theCodeBasically,
+    CronTrigger.from_crontab(
+        RUN_SCHEDULE,
+        timezone=RUN_SCHEDULE_TZ,
+    ),
+    max_instances=1,
+    coalesce=True,
+)
+
+scheduler.start()
+
+
+
+
+
+
+
 
 
 

requirements.txt

@@ -1 +1,2 @@
 paramiko==5.0.0
+APScheduler==3.11.3