#!/myapps/venv/bin/python3 import datetime import logging from paho.mqtt import client as mqtt_client import getopt import json import time import socket import subprocess from subprocess import Popen, PIPE, CalledProcessError import sys import os import re import platform import requests import fnmatch import yaml import paramiko import numpy as np file_path = os.path.realpath(__file__) dir_path = os.path.dirname(file_path) VERSION="1.0.6" # print(file_path) # print(dir_path) os.chdir(dir_path) from wakeonlan import send_magic_packet pid = os.getpid() def is_port_open(host, port): try: sock = socket.create_connection((host, port)) sock.close() return True except socket.error: return False servers = ["rpi5.home.lan","nas.home.lan","rack.home.lan","m-server.home.lan"] host = platform.node().lower() #input(host) cmnd = "ps -ef|grep omv_backups.py|grep -v grep |grep -v {}|wc -l".format(pid) status, output = subprocess.getstatusoutput(cmnd) if int(output) > 0: print("Running already!") sys.exit() def is_port_open(host, port): try: sock = socket.create_connection((host, port)) sock.close() return True except socket.error: return False s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # doesn't even have to be reachable conn = False while not conn: try: s.connect(('192.168.77.1', 1)) IP = s.getsockname()[0] conn = True except: time.sleep(5) broker = 'mqtt.home.lan' port = 1883 topic_sum = "sectorq/amd/backups" mqtt_username = 'jaydee' mqtt_password = 'jaydee1' try: opts, args = getopt.getopt(sys.argv[1:], "hTamftDr:bd:sSOl:", ["command=", "help", "output="]) except getopt.GetoptError as err: #usage() sys.exit(2) output = None # QJ : getopts _MODE = "manual" _FIRST = _TEST = _RESTORE = _BACKUP = _SYNC = _START = _STOP = _SSH_TEST = False _EXECUTE = True _DATE = "pick" _LOG_LEVEL = "" for o, a in opts: if o == "-a": _MODE = "auto" elif o in ("-m", "--manual"): _MODE = "manual" elif o in ("-l", "--level"): _LOG_LEVEL = a.upper() elif o in ("-f", "--first"): _FIRST = True elif o in ("-d", "--date"): _DATE = a elif o in ("-t", "--test"): _TEST = True elif o in ("-s", "--sync"): _SYNC = True elif o in ("-S", "--start"): _START = True elif o in ("-O", "--stop"): _STOP = True elif o in ("-r", "--restore"): _RESTORE = True _APP = a print("RESTORE") elif o in ("-b", "--backup"): _BACKUP = True elif o in ("-D", "--dry"): _EXECUTE = False elif o in ("-T", "--dry"): _SSH_TEST = True elif o in ("-h", "--help"): print(VERSION) sys.exit() LOG_FILE = "omv_backup.log" if _LOG_LEVEL == "DEBUG": logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') logging.debug('using debug loging') elif _LOG_LEVEL == "ERROR": logging.basicConfig(filename=LOG_FILE, level=logging.ERROR, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') logging.info('using error loging') elif _LOG_LEVEL == "SCAN": logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') logging.info('using error loging') else: logging.basicConfig(filename=LOG_FILE, level=logging.INFO, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') logging.info("script started") logger = logging.getLogger(__name__) client_id = "dasdasdasd333" try: client = mqtt_client.Client(mqtt_client.CallbackAPIVersion.VERSION1, client_id) except: client = mqtt_client.Client() client.username_pw_set(mqtt_username, mqtt_password) backups = { "nas": { "login": "admin@nas.home.lan", "jobs": { "github": {"source":"/share/Data/__GITHUB", "exclude":"", "active": True }, "photo": { "source":"/share/Photo/Years", "exclude":"", "active":True } } }, "m-server":{ "login": "root@m-server.home.lan", "jobs": { "docker_data":{ "source":"/share/docker_data/", "exclude":"", "active":True }, "fail2ban":{ "source":"/etc/fail2ban/", "exclude":"", "active":True } } }, "rpi5.home.lan":{ "docker_data":{ "source":"/share/docker_data/", "exclude":"", "active":True }, "fail2ban":{ "source":"/etc/fail2ban/", "exclude":"", "active":True } } } BACKUP_FS = "/media/backup/" BACKUP_HOST = "amd.home.lan" #BACKUP_HOST = "morefine.home.lan" logging.info("Test connection") hm = socket.gethostbyaddr(BACKUP_HOST) logging.info(_RESTORE) def send_mqtt_message(msg): try: client.connect(broker,1883,60) client.publish(topic, json.dumps(msg)) client.disconnect() except ValueError as e: logging.error("Failed to send") print("Failed to send") print(e) if _SYNC: containers = ["HomeAssistant","webhub-web-1","heimdall","pihole","mosquitto-mosquitto-1","mailu3-redis-1","mailu3-webmail-1","mailu3-resolver-1","mailu3-antispam-1","mailu3-webdav-1","mailu3-smtp-1","mailu3-oletools-1","mailu3-front-1","mailu3-fetchmail-1","mailu3-imap-1","matter-server","piper-en","openwakeword","whisper-en","auth-worker-1","auth-server-1","auth-authentik_ldap-1","auth-redis-1","auth-postgresql-1","nginx-app-1"] cmnd = f"curl -H 'Authorization: Bearer l4c1j4yd33Du5lo' 192.168.77.238:8094/v1/update" logging.info(cmnd) status, output = subprocess.getstatusoutput(cmnd) if _START: for c in containers: cmnd = f"docker start {c}" print(cmnd) status, output = subprocess.getstatusoutput(cmnd) if _STOP: cmnd = "docker ps" status, running_containers = subprocess.getstatusoutput(cmnd) logging.info(running_containers) for c in running_containers.splitlines(): print(c.split()[-1]) if c.split()[-1] == "watchtower-watchtower-1": continue cmnd = f"docker stop {c.split()[-1]}" status, running_containers = subprocess.getstatusoutput(cmnd) if _RESTORE: logging.info("Starting Restore") print("Starting Restore") now = datetime.datetime.now() STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") if _APP == "all": _DATE = "latest" if host == "rpi5.home.lan" or host == "rpi5": _APP = ["nginx","ha","gitea","gitlab","mailu","bitwarden","esphome","grafana","ingluxdb","kestra","matter-server","mosquitto","octoprint","octoprint2","pihole","unify_block","webhub","homepage","watchtower"] else: cmnd = "ssh root@amd.home.lan 'ls /mnt/raid/backup/m-server/docker_data/latest'" status, output = subprocess.getstatusoutput(cmnd) _APP = output.splitlines() logging.info(_APP) #input("????") else: _APP = _APP.split(",") PROGRESS = 0 topic = "sectorq/amd/restore" step = 100 / len(_APP) for app in _APP: msg = {"mode":_MODE, "status":"restore","bak_name":"Restore","host":host,"cur_job":app,"start_time":STARTTIME,"end_time":"","progress":str(round(np.ceil(PROGRESS))) + "%","finished":1,"used_space":1} logging.info(msg) send_mqtt_message(msg) PROGRESS = PROGRESS + step now = datetime.datetime.now() DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") BACKUP_HOST = f"root@amd.home.lan" BACKUP_DEVICE = "/mnt/raid" BACKUP_DIR = f"/backup/{host}" if _DATE == "pick": cmnd = f"ssh root@amd.home.lan 'ls {BACKUP_DEVICE}/backup/m-server/docker_data'" status, output = subprocess.getstatusoutput(cmnd) #print(output) dates = output.splitlines() n = 1 for i in dates: logging.info(f"{n} - {i}" ) n += 1 ans = input("Pick a backup to restore : ") _DATE = dates[int(ans) - 1] if app == "fail2ban": logging.info("?>?????") NEW_BACKUP_DIR = f"/backup/m-server/fail2ban/{_DATE}/" SOURCE_DIR = f"/etc/fail2ban" else: NEW_BACKUP_DIR = f"/backup/m-server/docker_data/{_DATE}/{app}" SOURCE_DIR = f"/share/docker_data/" if _FIRST: BACKUP_PATH="{}/initial".format(BACKUP_DIR) else: BACKUP_PATH="{}/{}".format(BACKUP_DIR, DATETIME) LATEST_LINK="{}/{}".format(BACKUP_DIR,_DATE) FULL_BACKUP_LATEST = f"{NEW_BACKUP_DIR}/{_DATE}" LATEST_LINK = f"/{host}/{app}/{_DATE}" logging.info("Create backup dir") logging.info(cmnd) #cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH) if app == "heimdall": logging.info("Stopping docker") cmnd = "docker stop heimdall" status, output = subprocess.getstatusoutput(cmnd) cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" logging.info(cmnd) logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) entries = ["Home Assistant","Nginx Proxy Manager","Portainer","Roundcube","Authentik","Kestra"] for e in entries: cmnd = f"sqlite3 /share/docker_data/heimdall/config/www/app.sqlite \"SELECT url FROM items WHERE title = '{e}'\"" logging.info(cmnd) status, output = subprocess.getstatusoutput(cmnd) regex = re.compile(r'[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}') contents = re.sub(regex, IP , output) cmnd = f"sqlite3 /share/docker_data/heimdall/config/www/app.sqlite \"UPDATE items SET url = '{contents}' WHERE title = '{e}'\"" logging.info(cmnd) status, output = subprocess.getstatusoutput(cmnd) # cmnd = "docker start heimdall" # status, output = subprocess.getstatusoutput(cmnd) if app == "ha": logging.info("Stopping docker") cmnd = "docker stop heimdall" status, output = subprocess.getstatusoutput(cmnd) cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" logging.info(cmnd) logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) logging.info("Start docker") # cmnd = "docker start heimdall" # status, output = subprocess.getstatusoutput(cmnd) elif app == "fail2ban": logging.info("Stopping docker") cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" logging.info(cmnd) logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) logging.info("Start docker") # cmnd = "docker start heimdall" # status, output = subprocess.getstatusoutput(cmnd) elif app == "homepage": logging.info("Stopping docker") cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" logging.info(cmnd) if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) file = "/share/docker_data/homepage/config/widgets.yaml" with open(file, 'r') as stream: try: loaded = yaml.load(stream, Loader=yaml.FullLoader) except yaml.YAMLError as exc: logging.info(exc) # Modify the fields from the dict #loaded['logo']['icon'] = "/images/morefine2.png" logging.info(json.dumps(loaded, indent=2)) i = 0 for y in loaded: logging.info(i) logging.info(y) if "logo" in y: if host == "rpi5.home.lan" or host == "rpi5": loaded[i]['logo']['icon'] = "/images/rpi5.png" elif host == "nas.home.lan": loaded[i]['logo']['icon'] = "/images/qnap_nas.png" elif host == "rack.home.lan": loaded[i]['logo']['icon'] = "/images/rack.png" else: loaded[i]['logo']['icon'] = "/images/morefine2.png" i+=1 # Save it again logging.info(f"writing to file {file}") with open(file, 'w') as stream: try: yaml.dump(loaded, stream, default_flow_style=False) except yaml.YAMLError as exc: print("failed") print(exc) logging.info("Start docker") # cmnd = "docker start heimdall" # status, output = subprocess.getstatusoutput(cmnd) elif app == "nginx1": logging.info("Stopping docker") cmnd = "docker stop nginx-app-1" status, output = subprocess.getstatusoutput(cmnd) cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" logging.info(cmnd) logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) domains = ["sectorq.eu","gitlab.sectorq.eu","ha.sectorq.eu","mail.sectorq.eu","pw.sectorq.eu","semaphore.sectorq.eu","kestra.sectorq.eu","auth.sectorq.eu"] for d in domains: cmnd = f'sqlite3 /share/docker_data/nginx/data/database.sqlite "UPDATE proxy_host SET forward_host = \'{IP}\' WHERE domain_names = \'[\\"{d}\\"]\'"' logging.info(cmnd) status, output = subprocess.getstatusoutput(cmnd) cmnd = 'egrep -l "# bazarr.sectorq.eu|# gitea.sectorq.eu|# jf.sectorq.eu|# kestra.sectorq.eu|# auth.sectorq.eu|# ha.sectorq.eu|# pw.sectorq.eu|# semaphore.sectorq.eu|# sectorq.eu|# gitlab.sectorq.eu|# ha.sectorq.eu" /share/docker_data/nginx/data/nginx/proxy_host/*' status, output = subprocess.getstatusoutput(cmnd) logging.info(output.splitlines()) for file in output.splitlines(): logging.info(file) f = open(file) contents = f.read() f.close() regex = re.compile(r'\n\s+set \$server\s+\"\w+.\w+.\w+.\w+\";') contents = re.sub(regex, f'\n set $server \"{IP}\";', contents) #print(contents) logging.info(regex) f = open(file, "w") contents = f.write(contents) f.close() status, output = subprocess.getstatusoutput(cmnd) logging.info("Starting docker") # cmnd = "docker start nginx-app-1" # status, output = subprocess.getstatusoutput(cmnd) else: cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" logging.info(cmnd) logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) now = datetime.datetime.now() ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") logging.info("Sending finished status") msg = {"mode":_MODE, "status":"restore","bak_name":"Restore","host":host,"cur_job":app,"start_time":STARTTIME,"end_time":"","progress":100,"finished":ENDJOB,"used_space":1} logging.info(msg) send_mqtt_message(msg) if _MODE == "auto": cmnd = "ssh root@amd.home.lan 'systemctl suspend &'" status, output = subprocess.getstatusoutput(cmnd) if _BACKUP: last = 1 while True: directory = '/backups/' count = len(fnmatch.filter(os.listdir(directory), '*')) if last != count: logging.info(f'File Count: {count}') last = count if count == 0: time.sleep(10) continue else: finished = [] now = datetime.datetime.now() STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") topic = "sectorq/amd/backups" msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} send_mqtt_message(msg) # iterate over files in # that directory for filename in os.scandir(directory): if filename.is_file(): logging.info(filename.path) logging.info(filename.name) if filename.name == "restore": break host = filename.name logging.info("Backup") for b in backups[host]["jobs"]: if not backups[host]["jobs"][b]["active"]: logging.info("Backup {} is not active!".format(b)) msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0} send_mqtt_message(msg) continue SOURCE_DIR = backups[host]["jobs"][b]["source"] now = datetime.datetime.now() BACKUP_HOST = backups[host]["login"] BACKUP_DEVICE = "/mnt/raid" BACKUP_DIR = f"{BACKUP_HOST}:{SOURCE_DIR}" BACKUP_ROOT = f"{BACKUP_DEVICE}/backup/{host}/{b}" DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") if _FIRST: NEW_BACKUP_DIR = f"{BACKUP_ROOT}/initial" else: NEW_BACKUP_DIR = f"{BACKUP_ROOT}/{DATETIME}_running" FULL_BACKUP_LATEST = f"{BACKUP_ROOT}/latest" # msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0} msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} client.connect(broker,1883,60) client.publish(topic, json.dumps(msg),qos=0, retain=True) client.disconnect() cmnd = "mkdir -p " + NEW_BACKUP_DIR logging.info(cmnd) if _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) logging.info(output) logging.info(status) logging.info("Create backup dir") cmnd = f"ssh {BACKUP_HOST} 'ls {SOURCE_DIR}'" logger.debug(cmnd) status, output = subprocess.getstatusoutput(cmnd) logger.debug(output) apps = output.splitlines() c = len(apps) print(apps) print(len(apps)) step = round(100 / c,1) progress = 0 #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR] cmd = ['rsync', '-avz', '--delete', BACKUP_DIR, '--link-dest', FULL_BACKUP_LATEST, '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', NEW_BACKUP_DIR] logging.info(" ".join(cmd)) process = subprocess.Popen(cmd, stdout=subprocess.PIPE) while process.poll() is None: line = process.stdout.readline().decode("utf-8").split("/") print(line[0]) if line[0] in apps: logging.info(f"Working on app {line[0]}") while True: if line[0] != apps[0]: del apps[0] progress = progress + step else: break apps.remove(line[0]) #print(len(apps)) topic = "sectorq/amd/backups" msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)} send_mqtt_message(msg) progress = progress + step # input(apps) # for a in apps: # logging.info(f"App {a}") # topic = "sectorq/amd/backups" # msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} # send_mqtt_message(msg) # logger.debug(cmnd) # if _FIRST: # cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" # else: # cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" # ans = "y" # logging.info(cmnd) # logging.info("Sync files1") # #input("??????") # if _TEST: # ans = input("continue?") or "n" # if ans == "y" and _EXECUTE: # status, output = subprocess.getstatusoutput(cmnd) # #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True) # progress = progress + step # topic = "sectorq/amd/backups" # msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} # send_mqtt_message(msg) cmnd = f"rm -rf {FULL_BACKUP_LATEST}" logging.info(cmnd) logging.info("Removing latest link") # input("????") if _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) if _FIRST: cmnd = f"cd {BACKUP_ROOT}; ln -s initial latest" else: cmnd = f"cd {BACKUP_ROOT}; mv {DATETIME}_running {DATETIME};ln -s {DATETIME} latest" logging.info("Creating new latest link") #print(cmnd) # input("????") if _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) #Remove old logging.info("Removing old dirs") # input("????") #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR) cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;" #print(cmnd) # input("????") if _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) now = datetime.datetime.now() ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S") #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0} finished.append(b) msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} send_mqtt_message(msg) logging.info("Getting size of FS") cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1" logging.info(cmnd) status, output = subprocess.getstatusoutput(cmnd) used_space = (output.split())[0] now = datetime.datetime.now() ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") logging.info("Size : {}".format(used_space)) logging.info("Sending finished status") #msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space} msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space} logging.info(msg) send_mqtt_message(msg) os.remove(filename.path) topic = "sectorq/amd/restore" for s in servers: #if s != "rack.home.lan": if s == "m-server.home.lan": continue elif s == "nas.home.lan": user = "admin" cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all" else: user = "jd" cmnd = "sudo /myapps/omv_backup.py -r all" msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1} logging.info(msg) send_mqtt_message(msg) if is_port_open(s,22): ssh = paramiko.SSHClient() ssh.load_system_host_keys() # Add SSH host key automatically if needed. ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Connect to router using username/password authentication. logger.info(f"Sync {s}") print(f"Sync {s}") ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa") ssh.connect(s, username=user, look_for_keys=False, allow_agent=False, pkey=pkey) print(cmnd) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd) for line in iter(ssh_stdout.readline, ""): logger.info(line) print(line, end="") for line in iter(ssh_stderr.readline, ""): logger.info(line) ssh.close() try: os.remove("/backups/restore") except: pass # if _MODE == "auto": # hostup = True # cmnd = "ssh root@omv.home.lan 'systemctl suspend &'" # status, output = subprocess.getstatusoutput(cmnd) # while hostup: # #HOST_UP = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0 # cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}" # status, output = subprocess.getstatusoutput(cmnd) # # print(status) # # print(output) # if status == 0: # print(f"Backup host up, waiting - {n}\r", end="") # time.sleep(5) # n += 1 # else: # print("Backup host down " ) # hostup = False # try: # url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off" # x = requests.post(url) # print(x.text) # except: # pass if _SSH_TEST: user = "root" cmnd = "ls -la" topic = "sectorq/amd/backups" for s in servers: # if s == "m-server.home.lan": # continue # elif s == "nas.home.lan": # user = "admin" # cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all" msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1} logging.info(msg) send_mqtt_message(msg) if s != "rack.home.lan": continue if is_port_open(s,22): ssh = paramiko.SSHClient() ssh.load_system_host_keys() # Add SSH host key automatically if needed. ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Connect to router using username/password authentication. logger.info(f"Sync {s}") print(f"Sync {s}") ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa") ssh.connect(s, username=user, look_for_keys=False, allow_agent=False, pkey=pkey) print(cmnd) ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd) for line in iter(ssh_stdout.readline, ""): logger.info(line) print(line, end="") for line in iter(ssh_stderr.readline, ""): logger.info(line) ssh.close()