v0.1 of APscheduler implementation

main.py

@@ -1,35 +1,37 @@
 import os, socket, subprocess, paramiko, io, sys, time
 from typing import Final
 from datetime import datetime
+from apscheduler.schedulers.blocking import BlockingScheduler
+from apscheduler.triggers.cron import CronTrigger
 
 
 IDRAC: Final[str] = os.getenv("IDRAC", None)
 IDRAC_USR: Final[str] = os.getenv("IDRAC_USR", None)
 IDRAC_PWD: Final[str] = os.getenv("IDRAC_PWD", None)
 
-# LAST_VERIFY_DAY: Final[int] = os.getenv("LAST_VERIFY_DAY", 7)
+# LAST_VERIFY_DAY: Final[int] = int(os.getenv("LAST_VERIFY_DAY", 7))
-# LAST_GARBAGE_DAY: Final[int] = os.getenv("LAST_GARBAGE_DAY", 7)
+# LAST_GARBAGE_DAY: Final[int] = int(os.getenv("LAST_GARBAGE_DAY", 7))
 
 HOST: Final[str] = os.getenv("HOST", None)
-HOST_SSH_PORT: Final[int] = os.getenv("HOST_SSH_PORT", None)
+HOST_SSH_PORT: Final[int] = int(os.getenv("HOST_SSH_PORT", 22))
 HOST_USR: Final[str] = os.getenv("HOST_USR", "root")
 HOST_USR_PWD: Final[str] = os.getenv("HOST_USR_PWD", None)
 HOST_USR_SSHKEY: Final[str] = os.getenv("HOST_USR_SSHKEY", None)
 HOST_USR_SSHKEY_PASS: Final[str] = os.getenv("HOST_USR_SSHKEY_PASS", None)
 
 LOCAL_DATASTORE: Final[str] = os.getenv("LOCAL_DATASTORE", None)
-REMOTE_DATASTORE: Final[str] = os.getenv("HOST_UREMOTE_DATASTORESR_SSHKEY_PASS", None)
+REMOTE_DATASTORE: Final[str] = os.getenv("REMOTE_DATASTORE", None)
 REMOTE_PBS: Final[str] = os.getenv("REMOTE_PBS", None)
 SYNC_DIR: Final[str] = os.getenv("SYNC_DIR", "push")
 PBS_RATE_OUT: Final[str] = os.getenv("PBS_RATE_OUT", "")
 PBS_RATE_IN: Final[str] = os.getenv("PBS_RATE_IN", "")
-PBS_TASK_LIMIT: Final[int] = os.getenv("PBS_TASK_LIMIT", 50)
+PBS_TASK_LIMIT: Final[int] = int(os.getenv("PBS_TASK_LIMIT", 50))
-PBS_MAX_RUNTIME_MIN: Final[int] = os.getenv("PBS_MAX_RUNTIME_MIN", 180)
+PBS_MAX_RUNTIME_MIN: Final[int] = int(os.getenv("PBS_MAX_RUNTIME_MIN", 180))
-PBS_MAX_SHUT_MIN: Final[int] = os.getenv("PBS_MAX_SHUT_MIN", 10)
+PBS_MAX_SHUT_MIN: Final[int] = int(os.getenv("PBS_MAX_SHUT_MIN", 10))
 AUTO_ADD_NAME: Final[str] = os.getenv("AUTO_ADD_NAME", "autoBackupScript_one-shot")
 
-
+RUN_SCHEDULE: Final[str] = os.getenv("RUN_SCHEDULE", "45 17 * * 5")
-
+RUN_SCHEDULE_TZ: Final[str] = os.getenv("RUN_SCHEDULE_TZ", "Europe/Stockholm")
 
 
 
@@ -59,15 +61,16 @@ def load_private_key(key_data: str, passphrase: str | None = None):
     raise ValueError("Unsupported or invalid SSH private key")
 
 
-def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
+def getCurrentTasks(sshThingy, allTasks: bool = False, limitOfTasks: int =50) -> dict:
     """Get what tasks are running/ran on PBS
     
     Keyword arguments:
+    sshThingy -- a paramiko SSH object (no default)
     allTasks -- if tasks that are finished should be included. (default False)
     limitOfTasks -- how many tasks to ask PBS for (default 50)
     """
 
-    (stdin, stdout, stderr) = ssh.exec_command(f'proxmox-backup-manager task list  --limit {limitOfTasks} --all {allTasks}')
+    (stdin, stdout, stderr) = sshThingy.exec_command(f'proxmox-backup-manager task list  --limit {limitOfTasks} --all {allTasks}')
 
     taskList = stdout.read().decode()
 
@@ -84,8 +87,11 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
         # Skip the header row
         if columns[0] == "starttime":
             continue
-        if len(columns) >= 2:
+        if len(columns) >= 3:
-            outTaskList.append(columns[2].split(":")[6])
+            try:
+                outTaskList.append(columns[2].split(":")[6])
+            except IndexError:
+                raise exceptionator(ConnectionError, "Incorrect message received from PBS")
             # exit()
 
 
@@ -132,7 +138,7 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
 
 if len(AUTO_ADD_NAME) > 32:
     raise exceptionator(ValueError, "Maxmimum length of AUTO_ADD_NAME is 32")
-    sys.exit(1)
+    # sys.exit(1)
 
 
 
@@ -143,23 +149,37 @@ if len(AUTO_ADD_NAME) > 32:
 
 
 
-# dt = datetime.now.timestamp()
 
-# epoch_time = dt.timestamp()
+def theCodeBasically():
-# print(epoch_time)
 
-# Check if power is on
+    print("\nStarting backup sequence\n")
-# PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
 
-try: 
+    # dt = datetime.now.timestamp()
-    # power on if needed
+
-    if "off" in str(PWR_STATE.stdout.decode()[-4:]):
+    # epoch_time = dt.timestamp()
-        subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"])  # , capture_output=True
+    # print(epoch_time)
-except exception as e:
+
-    print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
+    # Check if power is on
-    sys.exit(1)
+    PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "status",],capture_output=True,text=True,)
-# else:
+
-#     continue
+    if PWR_STATE.returncode != 0:
+        raise RuntimeError(f"ipmitool failed: {PWR_STATE.stderr.strip()}")
+
+    if "off" in PWR_STATE.stdout.lower():
+        subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "on",],check=True,)
+
+
+    # PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
+
+    # try: 
+    #     # power on if needed
+    #     if "off" in str(PWR_STATE.stdout.decode()[-4:]):
+    #         subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"])  # , capture_output=True
+    # except Exception as e:
+    #     print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
+    #     sys.exit(1)
+    # else:
+    #     continue
 
 
 
@@ -167,25 +187,26 @@ except exception as e:
 
 
 
-PWR_DATE = datetime.now().timestamp()
+    PWR_DATE = datetime.now().timestamp()
 
-pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
+    pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
 
-if pinging:
+    if pinging:
-    print("not reachable")
+        print("not reachable")
-    
+        
-    while datetime.now().timestamp() - PWR_DATE <= 1200:
+        while datetime.now().timestamp() - PWR_DATE <= 1200:
-        pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
+            pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
+
+            if not pinging:
+                print("reachable")
+                break
+        else:
+            raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.")
+            # sys.exit(1)
 
-        if not pinging:
-            print("reachable")
-            break
     else:
-        raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.")
+        print("reachable")
-        sys.exit(1)
 
-else:
-    print("reachable")
 
 
 
@@ -194,193 +215,230 @@ else:
 
 
 
+    # Create object of SSHClient and 
+    # connecting to SSH
+    ssh = paramiko.SSHClient()
 
-# Create object of SSHClient and 
+    # Adding new host key to the local 
-# connecting to SSH
+    # HostKeys object(in case of missing)
-ssh = paramiko.SSHClient()
+    # AutoAddPolicy for missing host key to be set before connection setup.
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-# Adding new host key to the local 
-# HostKeys object(in case of missing)
-# AutoAddPolicy for missing host key to be set before connection setup.
-ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
+    if not HOST_USR_SSHKEY:
 
-if not HOST_USR_SSHKEY:
+        ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, )
 
-    ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, )
+    else:
+        key = load_private_key(
+            HOST_USR_SSHKEY,
+            HOST_USR_SSHKEY_PASS
+        )
 
-else:
+        ssh.connect(
-    key = load_private_key(
+            hostname=HOST,
-        HOST_USR_SSHKEY,
+            port=HOST_SSH_PORT,
-        HOST_USR_SSHKEY_PASS
+            username=HOST_USR,
-    )
+            pkey=key,
+            timeout=3,
+            allow_agent=False,
+            look_for_keys=False,
+                
+        )
 
-    ssh.connect(
-        hostname=HOST,
-        port=HOST_SSH_PORT,
-        username=HOST_USR,
-        pkey=key,
-        timeout=3,
-        allow_agent=False,
-        look_for_keys=False,
-            
-    )
 
 
 
+    # Execute command on SSH terminal 
+    # using exec_command
+    # (stdin, stdout, stderr) = ssh.exec_command('hostname')
 
-# Execute command on SSH terminal 
+    # # redirecting all the output in cmd_output 
-# using exec_command
+    # # variable
-# (stdin, stdout, stderr) = ssh.exec_command('hostname')
+    # cmd_output = stdout.read().decode()
 
-# # redirecting all the output in cmd_output 
+    # print(cmd_output)
-# # variable
-# cmd_output = stdout.read().decode()
 
-# print(cmd_output)
+    (stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
 
-(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
+    remoteList = stdout.read().decode()
 
-remoteList = stdout.read().decode()
+    remoteDict = {}
 
-remoteDict = {}
+    for line in remoteList.splitlines():
+        line = line.strip()
 
-for line in remoteList.splitlines():
+        # Only process actual table rows
-    line = line.strip()
+        if not line.startswith("│"):
-
-    # Only process actual table rows
-    if not line.startswith("│"):
-        continue
-
-    columns = [col.strip() for col in line.strip("│").split("│")]
-
-    # Skip the header row
-    if columns[0] == "name":
-        continue
-
-    if len(columns) >= 2:
-        name = columns[0]
-        host = columns[1]
-
-        remoteDict[host] = name
-
-
-
-if REMOTE_PBS in remoteDict:
-    print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}")
-else:
-    print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr)
-    REMOTE_PBS.remove(REMOTE_PBS)
-    sys.exit(1)
-
-
-(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
-
-syncJobList = stdout.read().decode()
-
-
-for line in syncJobList.splitlines():
-    line = line.strip()
-
-    # Only process actual table rows
-    if not line.startswith("│"):
-        continue
-
-    columns = [col.strip() for col in line.strip("│").split("│")]
-
-    # Skip the header row
-    if columns[0] == "id":
-        continue
-    if len(columns) >= 2:
-        print("Sync job already on Proxmox Backup Server.\nSkipping sync job creation.")
-        # print(columns)
-        break
-else:
-
-    match SYNC_DIR:
-        case "push":
-            if PBS_RATE_OUT:
-                (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --rate-out {PBS_RATE_OUT} --comment 'automatic startupper script'")
-            else:
-                (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --comment 'automatic startupper script'")
-        case "pull":
-            if PBS_RATE_IN:
-                (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --rate-in {PBS_RATE_IN} --comment 'automatic startupper script'")
-            else:
-                (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE}    --sync-direction pull --comment 'automatic startupper script'")
-        case _:
-            raise exceptionator(SyncDirectionError, f"No such sync direction as {SYNC_DIR}.\n Either use 'push' or 'pull'\n")
-
-
-# print(stdout.read().decode())
-
-
-
-
-(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
-# ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
-# print("here it runs the sync-job")
-# print(stdout.read().decode())
-# print(stderr.read().decode())
-
-
-
-
-# Maybe add some checks here if the backup was properly scheduled
-
-
-
-# Get tasks
-# print(getCurrentTasks(limitOfTasks=100))
-
-
-
-
-# check ongoing tasks
-SYNC_JOB_DATE = datetime.now().timestamp()
-
-currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
-
-while currenttasks:
-
-    currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
-
-    print("Waiting 1 minute before checking tasks again")
-    time.sleep(60)
-
-    if SYNC_JOB_DATE - datetime.now().timestamp() >= (PBS_MAX_RUNTIME_MIN * 60):
-        print(SYNC_JOB_DATE - datetime.now().timestamp())
-        # Forcefully skip if these tasks are running. reader task is, I think, PVE backup
-        # Also otherjob is something else and it is better to wait if there is one otherjob
-        if (
-            currenttasks.get("aptupdate", 0)
-            or currenttasks.get("reader", 0)
-            or currenttasks.get("otherjob", 0)
-        ):
             continue
-            # No need to do anything here. It's only to check if these keys exist
+
-            ...
+        columns = [col.strip() for col in line.strip("│").split("│")]
-        break
+
+        # Skip the header row
+        if columns[0] == "name":
+            continue
+
+        if len(columns) >= 2:
+            name = columns[0]
+            host = columns[1]
+
+            remoteDict[host] = name
+
+
+
+    if REMOTE_PBS in remoteDict:
+        print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}")
+    else:
+        print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr)
+        raise RuntimeError(f"Remote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.")
+
+
+    (stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
+
+    syncJobList = stdout.read().decode()
+
+    sync_job_exists = False
+
+    for line in syncJobList.splitlines():
+        line = line.strip()
+
+        if not line.startswith("│"):
+            continue
+
+        columns = [col.strip() for col in line.strip("│").split("│")]
+
+        if columns[0] == "id":
+            continue
+
+        if columns[0] == AUTO_ADD_NAME:
+            sync_job_exists = True
+            break
+
+    if sync_job_exists:
+        print(f"Sync job {AUTO_ADD_NAME} already exists.")
+    
+    # create it
+    else:
+
+        match SYNC_DIR:
+            case "push":
+                if PBS_RATE_OUT:
+                    (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --rate-out {PBS_RATE_OUT} --comment 'automatic startupper script'")
+                else:
+                    (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --comment 'automatic startupper script'")
+            case "pull":
+                if PBS_RATE_IN:
+                    (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --rate-in {PBS_RATE_IN} --comment 'automatic startupper script'")
+                else:
+                    (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE}    --sync-direction pull --comment 'automatic startupper script'")
+            case _:
+                raise ValueError(f"Invalid sync direction: {SYNC_DIR}. Use 'push' or 'pull'.")
+
+
+        
+        syncJobCreateExitCode = stdout.channel.recv_exit_status()
+
+        if syncJobCreateExitCode != 0:
+            raise RuntimeError(f"Failed to create sync job: {stderr.read().decode()}")
+
+
+
+
+    (stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
+    # ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
+    # print("here it runs the sync-job")
+    # print(stdout.read().decode())
+    # print(stderr.read().decode())
+
+    syncJobRunExitCode = stdout.channel.recv_exit_status()
+
+    if syncJobRunExitCode != 0:
+        raise RuntimeError(f"Sync job failed: {stderr.read().decode()}")
+
+    # Give some time for PBS to "start" the sync-job task
+    time.sleep(10)
+
+
+    # Maybe add some checks here if the backup was properly scheduled
+
+
+
+    # Get tasks
+    # print(getCurrentTasks(limitOfTasks=100))
+
+
+
+
+    # check ongoing tasks
+    SYNC_JOB_DATE = time.monotonic()    #datetime.now().timestamp()
+
+    currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT)
+
+    while currenttasks:
+
+        print("Waiting 1 minute before checking tasks again")
+        time.sleep(60)
+        
+        currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT)
+
+
+        if time.monotonic() - SYNC_JOB_DATE >= (PBS_MAX_RUNTIME_MIN * 60):
+            # print(SYNC_JOB_DATE - time.monotonic())
+            # Forcefully skip if these tasks are running. reader task is, I think, PVE backup
+            # Also otherjob is something else and it is better to wait if there is one otherjob
+            if (
+                currenttasks.get("aptupdate", 0)
+                or currenttasks.get("reader", 0)
+                or currenttasks.get("otherjob", 0)
+            ):
+                continue
+                # No need to do anything here. It's only to check if these keys exist
+                ...
+            break
+
+
+    (stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
+    print("Shuting down PBS")
+    # print("here it runs a shutdown now")
+    print(stdout.read().decode())
+    print(stderr.read().decode(), file=sys.stderr)
+
+    ssh.close()
+    # wait for PBS to safely shutdown
+    time.sleep(PBS_MAX_SHUT_MIN * 60)
+
+
+    if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
+        print("PBS is powered off")
+
+    else:
+        subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"])
+
+    # sys.exit(0)
+
+
+
+
+
+scheduler = BlockingScheduler()
+scheduler.add_job(
+    theCodeBasically,
+    CronTrigger.from_crontab(
+        RUN_SCHEDULE,
+        timezone=RUN_SCHEDULE_TZ,
+    ),
+    max_instances=1,
+    coalesce=True,
+)
+
+scheduler.start()
 
 
-(stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
-print("Shuting down PBS")
-# print("here it runs a shutdown now")
-print(stdout.read().decode())
-print(stderr.read().decode(), file=sys.stderr)
 
-ssh.close()
-# wait for PBS to safely shutdown
-time.sleep(PBS_MAX_SHUT_MIN * 60)
 
 
-if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
-    print("PBS is powered off")
 
-else:
-    subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"])
 
-sys.exit(0)
 
 
 

requirements.txt

@@ -1 +1,2 @@
-paramiko==5.0.0
+paramiko==5.0.0
+APScheduler==3.11.3