v0.1 of APscheduler implementation
This commit is contained in:
466
main.py
466
main.py
@@ -1,35 +1,37 @@
|
||||
import os, socket, subprocess, paramiko, io, sys, time
|
||||
from typing import Final
|
||||
from datetime import datetime
|
||||
from apscheduler.schedulers.blocking import BlockingScheduler
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
|
||||
IDRAC: Final[str] = os.getenv("IDRAC", None)
|
||||
IDRAC_USR: Final[str] = os.getenv("IDRAC_USR", None)
|
||||
IDRAC_PWD: Final[str] = os.getenv("IDRAC_PWD", None)
|
||||
|
||||
# LAST_VERIFY_DAY: Final[int] = os.getenv("LAST_VERIFY_DAY", 7)
|
||||
# LAST_GARBAGE_DAY: Final[int] = os.getenv("LAST_GARBAGE_DAY", 7)
|
||||
# LAST_VERIFY_DAY: Final[int] = int(os.getenv("LAST_VERIFY_DAY", 7))
|
||||
# LAST_GARBAGE_DAY: Final[int] = int(os.getenv("LAST_GARBAGE_DAY", 7))
|
||||
|
||||
HOST: Final[str] = os.getenv("HOST", None)
|
||||
HOST_SSH_PORT: Final[int] = os.getenv("HOST_SSH_PORT", None)
|
||||
HOST_SSH_PORT: Final[int] = int(os.getenv("HOST_SSH_PORT", 22))
|
||||
HOST_USR: Final[str] = os.getenv("HOST_USR", "root")
|
||||
HOST_USR_PWD: Final[str] = os.getenv("HOST_USR_PWD", None)
|
||||
HOST_USR_SSHKEY: Final[str] = os.getenv("HOST_USR_SSHKEY", None)
|
||||
HOST_USR_SSHKEY_PASS: Final[str] = os.getenv("HOST_USR_SSHKEY_PASS", None)
|
||||
|
||||
LOCAL_DATASTORE: Final[str] = os.getenv("LOCAL_DATASTORE", None)
|
||||
REMOTE_DATASTORE: Final[str] = os.getenv("HOST_UREMOTE_DATASTORESR_SSHKEY_PASS", None)
|
||||
REMOTE_DATASTORE: Final[str] = os.getenv("REMOTE_DATASTORE", None)
|
||||
REMOTE_PBS: Final[str] = os.getenv("REMOTE_PBS", None)
|
||||
SYNC_DIR: Final[str] = os.getenv("SYNC_DIR", "push")
|
||||
PBS_RATE_OUT: Final[str] = os.getenv("PBS_RATE_OUT", "")
|
||||
PBS_RATE_IN: Final[str] = os.getenv("PBS_RATE_IN", "")
|
||||
PBS_TASK_LIMIT: Final[int] = os.getenv("PBS_TASK_LIMIT", 50)
|
||||
PBS_MAX_RUNTIME_MIN: Final[int] = os.getenv("PBS_MAX_RUNTIME_MIN", 180)
|
||||
PBS_MAX_SHUT_MIN: Final[int] = os.getenv("PBS_MAX_SHUT_MIN", 10)
|
||||
PBS_TASK_LIMIT: Final[int] = int(os.getenv("PBS_TASK_LIMIT", 50))
|
||||
PBS_MAX_RUNTIME_MIN: Final[int] = int(os.getenv("PBS_MAX_RUNTIME_MIN", 180))
|
||||
PBS_MAX_SHUT_MIN: Final[int] = int(os.getenv("PBS_MAX_SHUT_MIN", 10))
|
||||
AUTO_ADD_NAME: Final[str] = os.getenv("AUTO_ADD_NAME", "autoBackupScript_one-shot")
|
||||
|
||||
|
||||
|
||||
RUN_SCHEDULE: Final[str] = os.getenv("RUN_SCHEDULE", "45 17 * * 5")
|
||||
RUN_SCHEDULE_TZ: Final[str] = os.getenv("RUN_SCHEDULE_TZ", "Europe/Stockholm")
|
||||
|
||||
|
||||
|
||||
@@ -59,15 +61,16 @@ def load_private_key(key_data: str, passphrase: str | None = None):
|
||||
raise ValueError("Unsupported or invalid SSH private key")
|
||||
|
||||
|
||||
def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
|
||||
def getCurrentTasks(sshThingy, allTasks: bool = False, limitOfTasks: int =50) -> dict:
|
||||
"""Get what tasks are running/ran on PBS
|
||||
|
||||
Keyword arguments:
|
||||
sshThingy -- a paramiko SSH object (no default)
|
||||
allTasks -- if tasks that are finished should be included. (default False)
|
||||
limitOfTasks -- how many tasks to ask PBS for (default 50)
|
||||
"""
|
||||
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f'proxmox-backup-manager task list --limit {limitOfTasks} --all {allTasks}')
|
||||
(stdin, stdout, stderr) = sshThingy.exec_command(f'proxmox-backup-manager task list --limit {limitOfTasks} --all {allTasks}')
|
||||
|
||||
taskList = stdout.read().decode()
|
||||
|
||||
@@ -84,8 +87,11 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
|
||||
# Skip the header row
|
||||
if columns[0] == "starttime":
|
||||
continue
|
||||
if len(columns) >= 2:
|
||||
outTaskList.append(columns[2].split(":")[6])
|
||||
if len(columns) >= 3:
|
||||
try:
|
||||
outTaskList.append(columns[2].split(":")[6])
|
||||
except IndexError:
|
||||
raise exceptionator(ConnectionError, "Incorrect message received from PBS")
|
||||
# exit()
|
||||
|
||||
|
||||
@@ -132,7 +138,7 @@ def getCurrentTasks(allTasks: bool = False, limitOfTasks: int =50) -> dict:
|
||||
|
||||
if len(AUTO_ADD_NAME) > 32:
|
||||
raise exceptionator(ValueError, "Maxmimum length of AUTO_ADD_NAME is 32")
|
||||
sys.exit(1)
|
||||
# sys.exit(1)
|
||||
|
||||
|
||||
|
||||
@@ -143,23 +149,37 @@ if len(AUTO_ADD_NAME) > 32:
|
||||
|
||||
|
||||
|
||||
# dt = datetime.now.timestamp()
|
||||
|
||||
# epoch_time = dt.timestamp()
|
||||
# print(epoch_time)
|
||||
def theCodeBasically():
|
||||
|
||||
# Check if power is on
|
||||
# PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
|
||||
print("\nStarting backup sequence\n")
|
||||
|
||||
try:
|
||||
# power on if needed
|
||||
if "off" in str(PWR_STATE.stdout.decode()[-4:]):
|
||||
subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"]) # , capture_output=True
|
||||
except exception as e:
|
||||
print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
|
||||
sys.exit(1)
|
||||
# else:
|
||||
# continue
|
||||
# dt = datetime.now.timestamp()
|
||||
|
||||
# epoch_time = dt.timestamp()
|
||||
# print(epoch_time)
|
||||
|
||||
# Check if power is on
|
||||
PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "status",],capture_output=True,text=True,)
|
||||
|
||||
if PWR_STATE.returncode != 0:
|
||||
raise RuntimeError(f"ipmitool failed: {PWR_STATE.stderr.strip()}")
|
||||
|
||||
if "off" in PWR_STATE.stdout.lower():
|
||||
subprocess.run(["ipmitool", "-I", "lanplus","-H", IDRAC,"-U", IDRAC_USR,"-P", IDRAC_PWD,"chassis", "power", "on",],check=True,)
|
||||
|
||||
|
||||
# PWR_STATE = subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "status"], capture_output=True)
|
||||
|
||||
# try:
|
||||
# # power on if needed
|
||||
# if "off" in str(PWR_STATE.stdout.decode()[-4:]):
|
||||
# subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "on"]) # , capture_output=True
|
||||
# except Exception as e:
|
||||
# print("Most likely wrong password, username or hostname has been passed for idrac.\nExiting\n")
|
||||
# sys.exit(1)
|
||||
# else:
|
||||
# continue
|
||||
|
||||
|
||||
|
||||
@@ -167,25 +187,26 @@ except exception as e:
|
||||
|
||||
|
||||
|
||||
PWR_DATE = datetime.now().timestamp()
|
||||
PWR_DATE = datetime.now().timestamp()
|
||||
|
||||
pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
|
||||
pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
|
||||
|
||||
if pinging:
|
||||
print("not reachable")
|
||||
|
||||
while datetime.now().timestamp() - PWR_DATE <= 1200:
|
||||
pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
|
||||
if pinging:
|
||||
print("not reachable")
|
||||
|
||||
while datetime.now().timestamp() - PWR_DATE <= 1200:
|
||||
pinging = subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode
|
||||
|
||||
if not pinging:
|
||||
print("reachable")
|
||||
break
|
||||
else:
|
||||
raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.")
|
||||
# sys.exit(1)
|
||||
|
||||
if not pinging:
|
||||
print("reachable")
|
||||
break
|
||||
else:
|
||||
raise exceptionator(TimeoutError, "Host didn't start after 20 minutes.\nCheck the server.")
|
||||
sys.exit(1)
|
||||
print("reachable")
|
||||
|
||||
else:
|
||||
print("reachable")
|
||||
|
||||
|
||||
|
||||
@@ -194,193 +215,230 @@ else:
|
||||
|
||||
|
||||
|
||||
# Create object of SSHClient and
|
||||
# connecting to SSH
|
||||
ssh = paramiko.SSHClient()
|
||||
|
||||
# Create object of SSHClient and
|
||||
# connecting to SSH
|
||||
ssh = paramiko.SSHClient()
|
||||
# Adding new host key to the local
|
||||
# HostKeys object(in case of missing)
|
||||
# AutoAddPolicy for missing host key to be set before connection setup.
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
# Adding new host key to the local
|
||||
# HostKeys object(in case of missing)
|
||||
# AutoAddPolicy for missing host key to be set before connection setup.
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
|
||||
if not HOST_USR_SSHKEY:
|
||||
|
||||
if not HOST_USR_SSHKEY:
|
||||
ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, )
|
||||
|
||||
ssh.connect(hostname=HOST, port=HOST_SSH_PORT, username=HOST_USR, password=HOST_USR_PWD, timeout=3, )
|
||||
else:
|
||||
key = load_private_key(
|
||||
HOST_USR_SSHKEY,
|
||||
HOST_USR_SSHKEY_PASS
|
||||
)
|
||||
|
||||
else:
|
||||
key = load_private_key(
|
||||
HOST_USR_SSHKEY,
|
||||
HOST_USR_SSHKEY_PASS
|
||||
)
|
||||
ssh.connect(
|
||||
hostname=HOST,
|
||||
port=HOST_SSH_PORT,
|
||||
username=HOST_USR,
|
||||
pkey=key,
|
||||
timeout=3,
|
||||
allow_agent=False,
|
||||
look_for_keys=False,
|
||||
|
||||
)
|
||||
|
||||
ssh.connect(
|
||||
hostname=HOST,
|
||||
port=HOST_SSH_PORT,
|
||||
username=HOST_USR,
|
||||
pkey=key,
|
||||
timeout=3,
|
||||
allow_agent=False,
|
||||
look_for_keys=False,
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
# Execute command on SSH terminal
|
||||
# using exec_command
|
||||
# (stdin, stdout, stderr) = ssh.exec_command('hostname')
|
||||
|
||||
# Execute command on SSH terminal
|
||||
# using exec_command
|
||||
# (stdin, stdout, stderr) = ssh.exec_command('hostname')
|
||||
# # redirecting all the output in cmd_output
|
||||
# # variable
|
||||
# cmd_output = stdout.read().decode()
|
||||
|
||||
# # redirecting all the output in cmd_output
|
||||
# # variable
|
||||
# cmd_output = stdout.read().decode()
|
||||
# print(cmd_output)
|
||||
|
||||
# print(cmd_output)
|
||||
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
|
||||
|
||||
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager remote list')
|
||||
remoteList = stdout.read().decode()
|
||||
|
||||
remoteList = stdout.read().decode()
|
||||
remoteDict = {}
|
||||
|
||||
remoteDict = {}
|
||||
for line in remoteList.splitlines():
|
||||
line = line.strip()
|
||||
|
||||
for line in remoteList.splitlines():
|
||||
line = line.strip()
|
||||
|
||||
# Only process actual table rows
|
||||
if not line.startswith("│"):
|
||||
continue
|
||||
|
||||
columns = [col.strip() for col in line.strip("│").split("│")]
|
||||
|
||||
# Skip the header row
|
||||
if columns[0] == "name":
|
||||
continue
|
||||
|
||||
if len(columns) >= 2:
|
||||
name = columns[0]
|
||||
host = columns[1]
|
||||
|
||||
remoteDict[host] = name
|
||||
|
||||
|
||||
|
||||
if REMOTE_PBS in remoteDict:
|
||||
print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}")
|
||||
else:
|
||||
print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr)
|
||||
REMOTE_PBS.remove(REMOTE_PBS)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
|
||||
|
||||
syncJobList = stdout.read().decode()
|
||||
|
||||
|
||||
for line in syncJobList.splitlines():
|
||||
line = line.strip()
|
||||
|
||||
# Only process actual table rows
|
||||
if not line.startswith("│"):
|
||||
continue
|
||||
|
||||
columns = [col.strip() for col in line.strip("│").split("│")]
|
||||
|
||||
# Skip the header row
|
||||
if columns[0] == "id":
|
||||
continue
|
||||
if len(columns) >= 2:
|
||||
print("Sync job already on Proxmox Backup Server.\nSkipping sync job creation.")
|
||||
# print(columns)
|
||||
break
|
||||
else:
|
||||
|
||||
match SYNC_DIR:
|
||||
case "push":
|
||||
if PBS_RATE_OUT:
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --rate-out {PBS_RATE_OUT} --comment 'automatic startupper script'")
|
||||
else:
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --comment 'automatic startupper script'")
|
||||
case "pull":
|
||||
if PBS_RATE_IN:
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --rate-in {PBS_RATE_IN} --comment 'automatic startupper script'")
|
||||
else:
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --comment 'automatic startupper script'")
|
||||
case _:
|
||||
raise exceptionator(SyncDirectionError, f"No such sync direction as {SYNC_DIR}.\n Either use 'push' or 'pull'\n")
|
||||
|
||||
|
||||
# print(stdout.read().decode())
|
||||
|
||||
|
||||
|
||||
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
|
||||
# ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
|
||||
# print("here it runs the sync-job")
|
||||
# print(stdout.read().decode())
|
||||
# print(stderr.read().decode())
|
||||
|
||||
|
||||
|
||||
|
||||
# Maybe add some checks here if the backup was properly scheduled
|
||||
|
||||
|
||||
|
||||
# Get tasks
|
||||
# print(getCurrentTasks(limitOfTasks=100))
|
||||
|
||||
|
||||
|
||||
|
||||
# check ongoing tasks
|
||||
SYNC_JOB_DATE = datetime.now().timestamp()
|
||||
|
||||
currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
|
||||
|
||||
while currenttasks:
|
||||
|
||||
currenttasks = getCurrentTasks(limitOfTasks=PBS_TASK_LIMIT)
|
||||
|
||||
print("Waiting 1 minute before checking tasks again")
|
||||
time.sleep(60)
|
||||
|
||||
if SYNC_JOB_DATE - datetime.now().timestamp() >= (PBS_MAX_RUNTIME_MIN * 60):
|
||||
print(SYNC_JOB_DATE - datetime.now().timestamp())
|
||||
# Forcefully skip if these tasks are running. reader task is, I think, PVE backup
|
||||
# Also otherjob is something else and it is better to wait if there is one otherjob
|
||||
if (
|
||||
currenttasks.get("aptupdate", 0)
|
||||
or currenttasks.get("reader", 0)
|
||||
or currenttasks.get("otherjob", 0)
|
||||
):
|
||||
# Only process actual table rows
|
||||
if not line.startswith("│"):
|
||||
continue
|
||||
# No need to do anything here. It's only to check if these keys exist
|
||||
...
|
||||
break
|
||||
|
||||
columns = [col.strip() for col in line.strip("│").split("│")]
|
||||
|
||||
# Skip the header row
|
||||
if columns[0] == "name":
|
||||
continue
|
||||
|
||||
if len(columns) >= 2:
|
||||
name = columns[0]
|
||||
host = columns[1]
|
||||
|
||||
remoteDict[host] = name
|
||||
|
||||
|
||||
|
||||
if REMOTE_PBS in remoteDict:
|
||||
print(f"Found {REMOTE_PBS} as {remoteDict[REMOTE_PBS]}")
|
||||
else:
|
||||
print(f"\n!!!\nRemote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.\n!!!\n", file=sys.stderr)
|
||||
raise RuntimeError(f"Remote {REMOTE_PBS} not found.\nAdd remote {REMOTE_PBS} manually.")
|
||||
|
||||
|
||||
(stdin, stdout, stderr) = ssh.exec_command('proxmox-backup-manager sync-job list --sync-direction all')
|
||||
|
||||
syncJobList = stdout.read().decode()
|
||||
|
||||
sync_job_exists = False
|
||||
|
||||
for line in syncJobList.splitlines():
|
||||
line = line.strip()
|
||||
|
||||
if not line.startswith("│"):
|
||||
continue
|
||||
|
||||
columns = [col.strip() for col in line.strip("│").split("│")]
|
||||
|
||||
if columns[0] == "id":
|
||||
continue
|
||||
|
||||
if columns[0] == AUTO_ADD_NAME:
|
||||
sync_job_exists = True
|
||||
break
|
||||
|
||||
if sync_job_exists:
|
||||
print(f"Sync job {AUTO_ADD_NAME} already exists.")
|
||||
|
||||
# create it
|
||||
else:
|
||||
|
||||
match SYNC_DIR:
|
||||
case "push":
|
||||
if PBS_RATE_OUT:
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --rate-out {PBS_RATE_OUT} --comment 'automatic startupper script'")
|
||||
else:
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction push --comment 'automatic startupper script'")
|
||||
case "pull":
|
||||
if PBS_RATE_IN:
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --rate-in {PBS_RATE_IN} --comment 'automatic startupper script'")
|
||||
else:
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job create {AUTO_ADD_NAME} --remote {remoteDict[REMOTE_PBS]} --remote-store {REMOTE_DATASTORE} --store {LOCAL_DATASTORE} --sync-direction pull --comment 'automatic startupper script'")
|
||||
case _:
|
||||
raise ValueError(f"Invalid sync direction: {SYNC_DIR}. Use 'push' or 'pull'.")
|
||||
|
||||
|
||||
|
||||
syncJobCreateExitCode = stdout.channel.recv_exit_status()
|
||||
|
||||
if syncJobCreateExitCode != 0:
|
||||
raise RuntimeError(f"Failed to create sync job: {stderr.read().decode()}")
|
||||
|
||||
|
||||
|
||||
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
|
||||
# ssh.exec_command(f"proxmox-backup-manager sync-job run {AUTO_ADD_NAME}")
|
||||
# print("here it runs the sync-job")
|
||||
# print(stdout.read().decode())
|
||||
# print(stderr.read().decode())
|
||||
|
||||
syncJobRunExitCode = stdout.channel.recv_exit_status()
|
||||
|
||||
if syncJobRunExitCode != 0:
|
||||
raise RuntimeError(f"Sync job failed: {stderr.read().decode()}")
|
||||
|
||||
# Give some time for PBS to "start" the sync-job task
|
||||
time.sleep(10)
|
||||
|
||||
|
||||
# Maybe add some checks here if the backup was properly scheduled
|
||||
|
||||
|
||||
|
||||
# Get tasks
|
||||
# print(getCurrentTasks(limitOfTasks=100))
|
||||
|
||||
|
||||
|
||||
|
||||
# check ongoing tasks
|
||||
SYNC_JOB_DATE = time.monotonic() #datetime.now().timestamp()
|
||||
|
||||
currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT)
|
||||
|
||||
while currenttasks:
|
||||
|
||||
print("Waiting 1 minute before checking tasks again")
|
||||
time.sleep(60)
|
||||
|
||||
currenttasks = getCurrentTasks(ssh, limitOfTasks=PBS_TASK_LIMIT)
|
||||
|
||||
|
||||
if time.monotonic() - SYNC_JOB_DATE >= (PBS_MAX_RUNTIME_MIN * 60):
|
||||
# print(SYNC_JOB_DATE - time.monotonic())
|
||||
# Forcefully skip if these tasks are running. reader task is, I think, PVE backup
|
||||
# Also otherjob is something else and it is better to wait if there is one otherjob
|
||||
if (
|
||||
currenttasks.get("aptupdate", 0)
|
||||
or currenttasks.get("reader", 0)
|
||||
or currenttasks.get("otherjob", 0)
|
||||
):
|
||||
continue
|
||||
# No need to do anything here. It's only to check if these keys exist
|
||||
...
|
||||
break
|
||||
|
||||
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
|
||||
print("Shuting down PBS")
|
||||
# print("here it runs a shutdown now")
|
||||
print(stdout.read().decode())
|
||||
print(stderr.read().decode(), file=sys.stderr)
|
||||
|
||||
ssh.close()
|
||||
# wait for PBS to safely shutdown
|
||||
time.sleep(PBS_MAX_SHUT_MIN * 60)
|
||||
|
||||
|
||||
if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
|
||||
print("PBS is powered off")
|
||||
|
||||
else:
|
||||
subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"])
|
||||
|
||||
# sys.exit(0)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
scheduler = BlockingScheduler()
|
||||
scheduler.add_job(
|
||||
theCodeBasically,
|
||||
CronTrigger.from_crontab(
|
||||
RUN_SCHEDULE,
|
||||
timezone=RUN_SCHEDULE_TZ,
|
||||
),
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
)
|
||||
|
||||
scheduler.start()
|
||||
|
||||
|
||||
(stdin, stdout, stderr) = ssh.exec_command(f"sudo shutdown")
|
||||
print("Shuting down PBS")
|
||||
# print("here it runs a shutdown now")
|
||||
print(stdout.read().decode())
|
||||
print(stderr.read().decode(), file=sys.stderr)
|
||||
|
||||
ssh.close()
|
||||
# wait for PBS to safely shutdown
|
||||
time.sleep(PBS_MAX_SHUT_MIN * 60)
|
||||
|
||||
|
||||
if subprocess.run(["ping", "-c", "3", "-W", "5", HOST], capture_output=True).returncode:
|
||||
print("PBS is powered off")
|
||||
|
||||
else:
|
||||
subprocess.run(["ipmitool", "-I", "lanplus", "-H", IDRAC, "-U", IDRAC_USR, "-P", IDRAC_PWD, "chassis", "power", "off"])
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1 +1,2 @@
|
||||
paramiko==5.0.0
|
||||
paramiko==5.0.0
|
||||
APScheduler==3.11.3
|
||||
Reference in New Issue
Block a user