From de97d2574a68ed1e28810ff07dcd85104e1644d8 Mon Sep 17 00:00:00 2001 From: jaydee Date: Tue, 26 Aug 2025 12:12:17 +0200 Subject: [PATCH] build --- omv_backup.py | 576 ++++++++++++++++++++++++-------------------------- 1 file changed, 275 insertions(+), 301 deletions(-) diff --git a/omv_backup.py b/omv_backup.py index 8bd0f9d..14b3c46 100755 --- a/omv_backup.py +++ b/omv_backup.py @@ -145,15 +145,6 @@ TOPIC = 'sectorq/backups/start' USERNAME = 'jaydee' PASSWORD = 'jaydee1' USE_TLS = False # Set to False if not using TLS - - - - - - - - - backups = { "nas": { "login": "admin@nas.home.lan", @@ -482,305 +473,288 @@ def restore_job(): cmnd = "ssh root@amd.home.lan 'systemctl suspend &'" status, output = subprocess.getstatusoutput(cmnd) -def backup_job(): +def backup_job(server): logging.info(f'starting backup job') message = "start" topic = "sectorq/backups/start" + finished = [] + now = datetime.datetime.now() + STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + topic = "sectorq/amd/restore" + msg = {"mode":"restore", "status":"restore","bak_name":"s","host":0,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":0,"used_space":0} + send_mqtt_message(msg) + topic = "sectorq/amd/backups" + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + send_mqtt_message(msg) + + # iterate over files in + # that directory + + + host = server + logging.info("Backup") + for b in backups[host]["jobs"]: + + if not backups[host]["jobs"][b]["active"]: + logging.info("Backup {} is not active!".format(b)) + msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0} + send_mqtt_message(msg) + continue + + SOURCE_DIR = backups[host]["jobs"][b]["source"] + now = datetime.datetime.now() + BACKUP_HOST = backups[host]["login"] + BACKUP_DEVICE = "/mnt/raid" + BACKUP_DIR = f"{BACKUP_HOST}:{SOURCE_DIR}" + BACKUP_ROOT = f"{BACKUP_DEVICE}/backup/{host}/{b}" + DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") + + if _FIRST: + NEW_BACKUP_DIR = f"{BACKUP_ROOT}/initial" + else: + NEW_BACKUP_DIR = f"{BACKUP_ROOT}/{DATETIME}_running" + + FULL_BACKUP_LATEST = f"{BACKUP_ROOT}/latest" + + # msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0} + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + client.connect(broker,1883,60) + client.publish(topic, json.dumps(msg),qos=0, retain=True) + client.disconnect() + + + cmnd = "mkdir -p " + NEW_BACKUP_DIR + logging.info(cmnd) + + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + logging.info(output) + logging.info(status) + logging.info("Create backup dir") + + cmnd = f"ssh {BACKUP_HOST} 'ls {SOURCE_DIR}'" + logger.debug(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + logger.debug(output) + apps = output.splitlines() + c = len(apps) + print(apps) + print(len(apps)) + + step = round(100 / c,1) + progress = 0 + #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" + #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR] + + #cmd = ['rsync', '-avz', '--delete', BACKUP_DIR, '--link-dest', FULL_BACKUP_LATEST, '--exclude="home-assistant_v2.db"', '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', '--exclude="/.esphome/build"', '--exclude="nextcloud\mariadb\*"', NEW_BACKUP_DIR] + + cmd = ['rsync', '-avz', '--delete', BACKUP_DIR, '--link-dest', FULL_BACKUP_LATEST, '--exclude-from=/myapps/exclude.txt', NEW_BACKUP_DIR] + logging.info(" ".join(cmd)) + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE) + while process.poll() is None: + line = process.stdout.readline().decode("utf-8").split("/") + print(line[0]) + if line[0] in apps: + logging.info(f"Working on app {line[0]}") + while True: + if line[0] != apps[0]: + del apps[0] + progress = progress + step + else: + break + apps.remove(line[0]) + #print(len(apps)) + topic = "sectorq/amd/backups" + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)} + send_mqtt_message(msg) + progress = progress + step + # input(apps) + + # for a in apps: + # logging.info(f"App {a}") + # topic = "sectorq/amd/backups" + # msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} + # send_mqtt_message(msg) + # logger.debug(cmnd) + # if _FIRST: + # cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" + # else: + # cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" + + # ans = "y" + # logging.info(cmnd) + # logging.info("Sync files1") + # #input("??????") + # if _TEST: + # ans = input("continue?") or "n" + # if ans == "y" and _EXECUTE: + # status, output = subprocess.getstatusoutput(cmnd) + # #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True) + # progress = progress + step + # topic = "sectorq/amd/backups" + # msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} + # send_mqtt_message(msg) + + + cmnd = f"rm -rf {FULL_BACKUP_LATEST}" + + logging.info(cmnd) + logging.info("Removing latest link") + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + if _FIRST: + cmnd = f"cd {BACKUP_ROOT}; ln -s initial latest" + else: + cmnd = f"cd {BACKUP_ROOT}; mv {DATETIME}_running {DATETIME};ln -s {DATETIME} latest" + logging.info("Creating new latest link") + #print(cmnd) + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + + + #Remove old + logging.info("Removing old dirs") + # input("????") + #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR) + #cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;" + + cmnd = f"ls {BACKUP_ROOT}" + + + #print(cmnd) + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + for f in output.splitlines(): + pattern = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}$" # regex pattern: string starts with 'abc' + logging.info(f"Checking {f}") + if re.match(pattern, f): + logging.info("Match!") + dt = datetime.datetime.strptime(f, "%Y-%m-%d_%H-%M-%S") + epoch_time = int(dt.timestamp()) + now_epoch = int(datetime.datetime.now().timestamp()) + x = now_epoch - epoch_time + # logging.info(epoch_time) # Output: 45 + if x > 2592000: + dir_path = f"{BACKUP_ROOT}/{f}" + logging.info(f"removing {dir_path}") + shutil.rmtree(dir_path) + else: + print("No match.") + + cmnd = f"ls {BACKUP_ROOT}|grep _running" + #print(cmnd) + # input("????") + logging.info(f"removing obsolete dirs") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + for f in output.splitlines(): + dir_path = f"{BACKUP_ROOT}/{f}" + logging.info(f"removing {dir_path}") + shutil.rmtree(dir_path) + + now = datetime.datetime.now() + ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0} + finished.append(b) + msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + send_mqtt_message(msg) + + logging.info("Getting size of FS") + cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1" + logging.info(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + used_space = (output.split())[0] + now = datetime.datetime.now() + ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") + logging.info("Size : {}".format(used_space)) + logging.info("Sending finished status") + #msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space} + msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space} + logging.info(msg) + + send_mqtt_message(msg) + + + topic = "sectorq/amd/restore" + for s in servers: + #if s != "rack.home.lan": + if s == "m-server.home.lan": + continue + elif s == "nas.home.lan": + user = "admin" + cmnd = "/share/Data/python/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all" + else: + user = "jd" + cmnd = "sudo /myapps/omv_backup.py -r all" + msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1} + logging.info(msg) + + send_mqtt_message(msg) + if is_port_open(s,22): + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + # Add SSH host key automatically if needed. + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + # Connect to router using username/password authentication. + logger.info(f"Sync {s}") + print(f"Sync {s}") + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa") + ssh.connect(s, + username=user, + look_for_keys=False, + allow_agent=False, + pkey=pkey) + print(cmnd) + ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd) + for line in iter(ssh_stdout.readline, ""): + logger.info(line) + print(line, end="") + for line in iter(ssh_stderr.readline, ""): + logger.info(line) + ssh.close() + try: + os.remove("/backups/restore") + except: + pass + # if _MODE == "auto": + # hostup = True + # cmnd = "ssh root@omv.home.lan 'systemctl suspend &'" + # status, output = subprocess.getstatusoutput(cmnd) + # while hostup: + # #HOST_UP = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0 + # cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}" + # status, output = subprocess.getstatusoutput(cmnd) + # # print(status) + # # print(output) + + + # if status == 0: + # print(f"Backup host up, waiting - {n}\r", end="") + # time.sleep(5) + # n += 1 + # else: + # print("Backup host down " ) + # hostup = False + + + + # try: + # url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off" + + # x = requests.post(url) + + # print(x.text) + # except: + # pass message= "finished1" client.publish(topic, message,qos=2,retain=True) return 0 - last = 1 - while True: - directory = '/backups/' - count = len(fnmatch.filter(os.listdir(directory), '*')) - if last != count: - logging.info(f'File Count: {count}') - last = count - if count == 0: - time.sleep(10) - continue - else: - finished = [] - now = datetime.datetime.now() - STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") - topic = "sectorq/amd/restore" - msg = {"mode":"restore", "status":"restore","bak_name":"s","host":0,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":0,"used_space":0} - send_mqtt_message(msg) - topic = "sectorq/amd/backups" - msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} - send_mqtt_message(msg) - - # iterate over files in - # that directory - - for filename in os.scandir(directory): - if filename.is_file(): - logging.info(filename.path) - logging.info(filename.name) - if filename.name == "restore": - break - host = filename.name - logging.info("Backup") - for b in backups[host]["jobs"]: - - if not backups[host]["jobs"][b]["active"]: - logging.info("Backup {} is not active!".format(b)) - msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0} - send_mqtt_message(msg) - continue - SOURCE_DIR = backups[host]["jobs"][b]["source"] - now = datetime.datetime.now() - BACKUP_HOST = backups[host]["login"] - BACKUP_DEVICE = "/mnt/raid" - BACKUP_DIR = f"{BACKUP_HOST}:{SOURCE_DIR}" - BACKUP_ROOT = f"{BACKUP_DEVICE}/backup/{host}/{b}" - DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") - - if _FIRST: - NEW_BACKUP_DIR = f"{BACKUP_ROOT}/initial" - else: - NEW_BACKUP_DIR = f"{BACKUP_ROOT}/{DATETIME}_running" - - FULL_BACKUP_LATEST = f"{BACKUP_ROOT}/latest" - - # msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0} - msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} - client.connect(broker,1883,60) - client.publish(topic, json.dumps(msg),qos=0, retain=True) - client.disconnect() - - - cmnd = "mkdir -p " + NEW_BACKUP_DIR - logging.info(cmnd) - - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - logging.info(output) - logging.info(status) - logging.info("Create backup dir") - - cmnd = f"ssh {BACKUP_HOST} 'ls {SOURCE_DIR}'" - logger.debug(cmnd) - status, output = subprocess.getstatusoutput(cmnd) - logger.debug(output) - apps = output.splitlines() - c = len(apps) - print(apps) - print(len(apps)) - - step = round(100 / c,1) - progress = 0 - #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" - #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR] - - #cmd = ['rsync', '-avz', '--delete', BACKUP_DIR, '--link-dest', FULL_BACKUP_LATEST, '--exclude="home-assistant_v2.db"', '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', '--exclude="/.esphome/build"', '--exclude="nextcloud\mariadb\*"', NEW_BACKUP_DIR] - - cmd = ['rsync', '-avz', '--delete', BACKUP_DIR, '--link-dest', FULL_BACKUP_LATEST, '--exclude-from=/myapps/exclude.txt', NEW_BACKUP_DIR] - logging.info(" ".join(cmd)) - process = subprocess.Popen(cmd, - stdout=subprocess.PIPE) - while process.poll() is None: - line = process.stdout.readline().decode("utf-8").split("/") - print(line[0]) - if line[0] in apps: - logging.info(f"Working on app {line[0]}") - while True: - if line[0] != apps[0]: - del apps[0] - progress = progress + step - else: - break - apps.remove(line[0]) - #print(len(apps)) - topic = "sectorq/amd/backups" - msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)} - send_mqtt_message(msg) - progress = progress + step - # input(apps) - - # for a in apps: - # logging.info(f"App {a}") - # topic = "sectorq/amd/backups" - # msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} - # send_mqtt_message(msg) - # logger.debug(cmnd) - # if _FIRST: - # cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" - # else: - # cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" - - # ans = "y" - # logging.info(cmnd) - # logging.info("Sync files1") - # #input("??????") - # if _TEST: - # ans = input("continue?") or "n" - # if ans == "y" and _EXECUTE: - # status, output = subprocess.getstatusoutput(cmnd) - # #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True) - # progress = progress + step - # topic = "sectorq/amd/backups" - # msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} - # send_mqtt_message(msg) - - - cmnd = f"rm -rf {FULL_BACKUP_LATEST}" - - logging.info(cmnd) - logging.info("Removing latest link") - # input("????") - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - if _FIRST: - cmnd = f"cd {BACKUP_ROOT}; ln -s initial latest" - else: - cmnd = f"cd {BACKUP_ROOT}; mv {DATETIME}_running {DATETIME};ln -s {DATETIME} latest" - logging.info("Creating new latest link") - #print(cmnd) - # input("????") - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - - - #Remove old - logging.info("Removing old dirs") - # input("????") - #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR) - #cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;" - - cmnd = f"ls {BACKUP_ROOT}" - - - #print(cmnd) - # input("????") - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - for f in output.splitlines(): - pattern = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}$" # regex pattern: string starts with 'abc' - logging.info(f"Checking {f}") - if re.match(pattern, f): - logging.info("Match!") - dt = datetime.datetime.strptime(f, "%Y-%m-%d_%H-%M-%S") - epoch_time = int(dt.timestamp()) - now_epoch = int(datetime.datetime.now().timestamp()) - x = now_epoch - epoch_time - # logging.info(epoch_time) # Output: 45 - if x > 2592000: - dir_path = f"{BACKUP_ROOT}/{f}" - logging.info(f"removing {dir_path}") - shutil.rmtree(dir_path) - else: - print("No match.") - - cmnd = f"ls {BACKUP_ROOT}|grep _running" - #print(cmnd) - # input("????") - logging.info(f"removing obsolete dirs") - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - for f in output.splitlines(): - dir_path = f"{BACKUP_ROOT}/{f}" - logging.info(f"removing {dir_path}") - shutil.rmtree(dir_path) - - now = datetime.datetime.now() - ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S") - #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0} - finished.append(b) - msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} - send_mqtt_message(msg) - - logging.info("Getting size of FS") - cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1" - logging.info(cmnd) - status, output = subprocess.getstatusoutput(cmnd) - used_space = (output.split())[0] - now = datetime.datetime.now() - ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") - logging.info("Size : {}".format(used_space)) - logging.info("Sending finished status") - #msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space} - msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space} - logging.info(msg) - - send_mqtt_message(msg) - os.remove(filename.path) - - - topic = "sectorq/amd/restore" - for s in servers: - #if s != "rack.home.lan": - if s == "m-server.home.lan": - continue - elif s == "nas.home.lan": - user = "admin" - cmnd = "/share/Data/python/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all" - else: - user = "jd" - cmnd = "sudo /myapps/omv_backup.py -r all" - msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1} - logging.info(msg) - - send_mqtt_message(msg) - if is_port_open(s,22): - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - # Add SSH host key automatically if needed. - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - # Connect to router using username/password authentication. - logger.info(f"Sync {s}") - print(f"Sync {s}") - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa") - ssh.connect(s, - username=user, - look_for_keys=False, - allow_agent=False, - pkey=pkey) - print(cmnd) - ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd) - for line in iter(ssh_stdout.readline, ""): - logger.info(line) - print(line, end="") - for line in iter(ssh_stderr.readline, ""): - logger.info(line) - ssh.close() - try: - os.remove("/backups/restore") - except: - pass - # if _MODE == "auto": - # hostup = True - # cmnd = "ssh root@omv.home.lan 'systemctl suspend &'" - # status, output = subprocess.getstatusoutput(cmnd) - # while hostup: - # #HOST_UP = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0 - # cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}" - # status, output = subprocess.getstatusoutput(cmnd) - # # print(status) - # # print(output) - - - # if status == 0: - # print(f"Backup host up, waiting - {n}\r", end="") - # time.sleep(5) - # n += 1 - # else: - # print("Backup host down " ) - # hostup = False - - - - # try: - # url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off" - - # x = requests.post(url) - - # print(x.text) - # except: - # pass - if _SSH_TEST: user = "root" cmnd = "ls -la"