build

omv_backup.py

@@ -145,15 +145,6 @@ TOPIC = 'sectorq/backups/start'
 USERNAME = 'jaydee'
 PASSWORD = 'jaydee1'
 USE_TLS = False                       # Set to False if not using TLS
-
-
-
-
-
-
-
-
-
 backups = {
     "nas": {
         "login": "admin@nas.home.lan",
@@ -482,304 +473,287 @@ def restore_job():
         cmnd = "ssh root@amd.home.lan 'systemctl suspend &'"
         status, output = subprocess.getstatusoutput(cmnd)
 
-def backup_job():
+def backup_job(server):
     logging.info(f'starting backup job')
     message = "start"
     topic = "sectorq/backups/start"
+    finished = []
+    now = datetime.datetime.now()
+    STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
+    topic = "sectorq/amd/restore"
+    msg = {"mode":"restore", "status":"restore","bak_name":"s","host":0,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":0,"used_space":0}
+    send_mqtt_message(msg)
+    topic = "sectorq/amd/backups"
+    msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
+    send_mqtt_message(msg)
+    
+    # iterate over files in 
+    # that directory
+    
+    
+    host = server
+    logging.info("Backup")
+    for b in backups[host]["jobs"]:
+        
+        if not backups[host]["jobs"][b]["active"]:
+            logging.info("Backup {} is not active!".format(b))
+            msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0}
+            send_mqtt_message(msg)
+            continue
+
+        SOURCE_DIR = backups[host]["jobs"][b]["source"]
+        now = datetime.datetime.now()
+        BACKUP_HOST = backups[host]["login"]
+        BACKUP_DEVICE = "/mnt/raid"
+        BACKUP_DIR = f"{BACKUP_HOST}:{SOURCE_DIR}"
+        BACKUP_ROOT = f"{BACKUP_DEVICE}/backup/{host}/{b}"
+        DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S")
+        
+        if _FIRST:
+            NEW_BACKUP_DIR = f"{BACKUP_ROOT}/initial"
+        else:
+            NEW_BACKUP_DIR = f"{BACKUP_ROOT}/{DATETIME}_running"
+
+        FULL_BACKUP_LATEST = f"{BACKUP_ROOT}/latest"
+
+        # msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0}
+        msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
+        client.connect(broker,1883,60)
+        client.publish(topic, json.dumps(msg),qos=0, retain=True)
+        client.disconnect()
+
+
+        cmnd = "mkdir -p " + NEW_BACKUP_DIR
+        logging.info(cmnd)
+        
+        if _EXECUTE:
+            status, output = subprocess.getstatusoutput(cmnd)
+            logging.info(output)
+            logging.info(status)
+        logging.info("Create backup dir")
+        
+        cmnd = f"ssh {BACKUP_HOST} 'ls  {SOURCE_DIR}'"
+        logger.debug(cmnd)
+        status, output = subprocess.getstatusoutput(cmnd)
+        logger.debug(output)
+        apps = output.splitlines()
+        c = len(apps)
+        print(apps)
+        print(len(apps))
+        
+        step = round(100 / c,1)
+        progress = 0
+        #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
+        #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR]
+        
+        #cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude="home-assistant_v2.db"', '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', '--exclude="/.esphome/build"', '--exclude="nextcloud\mariadb\*"', NEW_BACKUP_DIR]
+        
+        cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude-from=/myapps/exclude.txt', NEW_BACKUP_DIR]
+        logging.info(" ".join(cmd))
+        process = subprocess.Popen(cmd,
+                stdout=subprocess.PIPE)
+        while process.poll() is None:
+            line = process.stdout.readline().decode("utf-8").split("/")
+            print(line[0])
+            if line[0] in apps:
+                logging.info(f"Working on app {line[0]}")
+                while True:
+                    if line[0] != apps[0]:
+                        del apps[0]
+                        progress = progress + step
+                    else:
+                        break
+                apps.remove(line[0])
+                #print(len(apps))
+                topic = "sectorq/amd/backups"
+                msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)}
+                send_mqtt_message(msg)
+                progress = progress + step
+        # input(apps)
+
+        # for a in apps:
+        #     logging.info(f"App {a}")
+        #     topic = "sectorq/amd/backups"
+        #     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
+        #     send_mqtt_message(msg) 
+        #     logger.debug(cmnd)
+        #     if _FIRST:
+        #         cmnd = f"rsync -avz --delete {SOURCE_DIR}  --exclude=\"jellyfin/cache/transcodes\"  --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}"     
+        #     else:
+        #         cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
+
+        #     ans = "y"
+        #     logging.info(cmnd)
+        #     logging.info("Sync files1")
+        #     #input("??????")
+        #     if _TEST:
+        #         ans = input("continue?") or "n"
+        #     if ans == "y" and _EXECUTE:
+        #         status, output = subprocess.getstatusoutput(cmnd)
+        #         #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True)
+        #     progress = progress + step
+        #     topic = "sectorq/amd/backups"
+        #     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
+        #     send_mqtt_message(msg) 
+                
+            
+        cmnd = f"rm -rf {FULL_BACKUP_LATEST}"
+        
+        logging.info(cmnd)
+        logging.info("Removing latest link")
+    #    input("????")
+        if _EXECUTE:
+            status, output = subprocess.getstatusoutput(cmnd)
+        if _FIRST:
+            cmnd = f"cd {BACKUP_ROOT}; ln -s initial latest"
+        else:
+            cmnd = f"cd {BACKUP_ROOT}; mv {DATETIME}_running {DATETIME};ln -s {DATETIME} latest"
+        logging.info("Creating new latest link")
+        #print(cmnd)
+    #    input("????")
+        if _EXECUTE:
+            status, output = subprocess.getstatusoutput(cmnd)
+
+
+        #Remove old
+        logging.info("Removing old dirs")
+    #    input("????")
+        #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR)
+        #cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;"
+        
+        cmnd = f"ls {BACKUP_ROOT}"
+        
+        
+        #print(cmnd)
+    #   input("????")
+        if _EXECUTE:
+            status, output = subprocess.getstatusoutput(cmnd)
+            for f in output.splitlines():
+                pattern = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}$"  # regex pattern: string starts with 'abc'
+                logging.info(f"Checking {f}")
+                if re.match(pattern, f):
+                    logging.info("Match!")
+                    dt = datetime.datetime.strptime(f, "%Y-%m-%d_%H-%M-%S")
+                    epoch_time = int(dt.timestamp())
+                    now_epoch = int(datetime.datetime.now().timestamp())
+                    x = now_epoch - epoch_time
+                    # logging.info(epoch_time)  # Output: 45
+                    if x > 2592000:
+                        dir_path = f"{BACKUP_ROOT}/{f}"
+                        logging.info(f"removing {dir_path}")
+                        shutil.rmtree(dir_path)
+                else:
+                    print("No match.")
+                    
+        cmnd = f"ls {BACKUP_ROOT}|grep _running"
+        #print(cmnd)
+    #   input("????")
+        logging.info(f"removing obsolete dirs")
+        if _EXECUTE:
+            status, output = subprocess.getstatusoutput(cmnd)
+            for f in output.splitlines():
+                dir_path = f"{BACKUP_ROOT}/{f}"
+                logging.info(f"removing {dir_path}")
+                shutil.rmtree(dir_path)
+
+        now = datetime.datetime.now()
+        ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
+        #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0}
+        finished.append(b)
+        msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
+        send_mqtt_message(msg)
+
+    logging.info("Getting size of FS")
+    cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1"
+    logging.info(cmnd)
+    status, output = subprocess.getstatusoutput(cmnd)
+    used_space = (output.split())[0]
+    now = datetime.datetime.now()
+    ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S")
+    logging.info("Size : {}".format(used_space))
+    logging.info("Sending finished status")
+    #msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space}
+    msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space}
+    logging.info(msg)
+    
+    send_mqtt_message(msg)
+
+    
+    topic = "sectorq/amd/restore"
+    for s in servers:
+        #if s != "rack.home.lan":
+        if s == "m-server.home.lan":
+            continue
+        elif s == "nas.home.lan":
+            user = "admin"
+            cmnd = "/share/Data/python/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
+        else:
+            user = "jd"
+            cmnd = "sudo /myapps/omv_backup.py -r all"
+        msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
+        logging.info(msg)
+        
+        send_mqtt_message(msg)
+        if is_port_open(s,22):
+            ssh = paramiko.SSHClient()
+            ssh.load_system_host_keys()
+            # Add SSH host key automatically if needed.
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            # Connect to router using username/password authentication.
+            logger.info(f"Sync {s}")
+            print(f"Sync {s}")
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa")
+            ssh.connect(s, 
+                username=user, 
+                look_for_keys=False,
+                allow_agent=False,
+                pkey=pkey)
+            print(cmnd)
+            ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd)
+            for line in iter(ssh_stdout.readline, ""):
+                logger.info(line)
+                print(line, end="")
+            for line in iter(ssh_stderr.readline, ""):
+                logger.info(line)
+            ssh.close()
+    try:
+        os.remove("/backups/restore")
+    except:
+        pass
+        # if _MODE == "auto":
+        #     hostup = True
+        #     cmnd = "ssh root@omv.home.lan 'systemctl suspend &'"
+        #     status, output = subprocess.getstatusoutput(cmnd)
+        #     while hostup:
+        #         #HOST_UP  = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0
+        #         cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}"
+        #         status, output = subprocess.getstatusoutput(cmnd)
+        #         # print(status)
+        #         # print(output)
+                
+                
+        #         if status == 0:
+        #             print(f"Backup host up, waiting - {n}\r", end="")
+        #             time.sleep(5)
+        #             n += 1
+        #         else:
+        #             print("Backup host down                         " )
+        #             hostup = False
+            
+            
+            
+        #     try:
+        #         url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off"
+
+        #         x = requests.post(url)
+
+        #         print(x.text)  
+        #     except:
+        #         pass
     message= "finished1"
     client.publish(topic, message,qos=2,retain=True)
     return 0
-    last = 1
-    while True:
-        directory = '/backups/'
-        count = len(fnmatch.filter(os.listdir(directory), '*'))
-        if last != count:
-            logging.info(f'File Count: {count}')
-            last = count
-        if count == 0:
-            time.sleep(10)
-            continue
-        else:
-            finished = []
-            now = datetime.datetime.now()
-            STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
-            topic = "sectorq/amd/restore"
-            msg = {"mode":"restore", "status":"restore","bak_name":"s","host":0,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":0,"used_space":0}
-            send_mqtt_message(msg)
-            topic = "sectorq/amd/backups"
-            msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
-            send_mqtt_message(msg)
-            
-            # iterate over files in 
-            # that directory
-            
-            for filename in os.scandir(directory):
-                if filename.is_file():
-                    logging.info(filename.path)
-                    logging.info(filename.name)
-                if filename.name == "restore":
-                    break
-                host = filename.name
-                logging.info("Backup")
-                for b in backups[host]["jobs"]:
-                    
-                    if not backups[host]["jobs"][b]["active"]:
-                        logging.info("Backup {} is not active!".format(b))
-                        msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0}
-                        send_mqtt_message(msg)
-                        continue
-
-                    SOURCE_DIR = backups[host]["jobs"][b]["source"]
-                    now = datetime.datetime.now()
-                    BACKUP_HOST = backups[host]["login"]
-                    BACKUP_DEVICE = "/mnt/raid"
-                    BACKUP_DIR = f"{BACKUP_HOST}:{SOURCE_DIR}"
-                    BACKUP_ROOT = f"{BACKUP_DEVICE}/backup/{host}/{b}"
-                    DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S")
-                    
-                    if _FIRST:
-                        NEW_BACKUP_DIR = f"{BACKUP_ROOT}/initial"
-                    else:
-                        NEW_BACKUP_DIR = f"{BACKUP_ROOT}/{DATETIME}_running"
-  
-                    FULL_BACKUP_LATEST = f"{BACKUP_ROOT}/latest"
-
-                   # msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0}
-                    msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
-                    client.connect(broker,1883,60)
-                    client.publish(topic, json.dumps(msg),qos=0, retain=True)
-                    client.disconnect()
-
-
-                    cmnd = "mkdir -p " + NEW_BACKUP_DIR
-                    logging.info(cmnd)
-                    
-                    if _EXECUTE:
-                        status, output = subprocess.getstatusoutput(cmnd)
-                        logging.info(output)
-                        logging.info(status)
-                    logging.info("Create backup dir")
-                    
-                    cmnd = f"ssh {BACKUP_HOST} 'ls  {SOURCE_DIR}'"
-                    logger.debug(cmnd)
-                    status, output = subprocess.getstatusoutput(cmnd)
-                    logger.debug(output)
-                    apps = output.splitlines()
-                    c = len(apps)
-                    print(apps)
-                    print(len(apps))
-                    
-                    step = round(100 / c,1)
-                    progress = 0
-                    #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
-                    #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR]
-                    
-                    #cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude="home-assistant_v2.db"', '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', '--exclude="/.esphome/build"', '--exclude="nextcloud\mariadb\*"', NEW_BACKUP_DIR]
-                    
-                    cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude-from=/myapps/exclude.txt', NEW_BACKUP_DIR]
-                    logging.info(" ".join(cmd))
-                    process = subprocess.Popen(cmd,
-                           stdout=subprocess.PIPE)
-                    while process.poll() is None:
-                        line = process.stdout.readline().decode("utf-8").split("/")
-                        print(line[0])
-                        if line[0] in apps:
-                            logging.info(f"Working on app {line[0]}")
-                            while True:
-                                if line[0] != apps[0]:
-                                    del apps[0]
-                                    progress = progress + step
-                                else:
-                                    break
-                            apps.remove(line[0])
-                            #print(len(apps))
-                            topic = "sectorq/amd/backups"
-                            msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)}
-                            send_mqtt_message(msg)
-                            progress = progress + step
-                   # input(apps)
- 
-                    # for a in apps:
-                    #     logging.info(f"App {a}")
-                    #     topic = "sectorq/amd/backups"
-                    #     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
-                    #     send_mqtt_message(msg) 
-                    #     logger.debug(cmnd)
-                    #     if _FIRST:
-                    #         cmnd = f"rsync -avz --delete {SOURCE_DIR}  --exclude=\"jellyfin/cache/transcodes\"  --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}"     
-                    #     else:
-                    #         cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
-
-                    #     ans = "y"
-                    #     logging.info(cmnd)
-                    #     logging.info("Sync files1")
-                    #     #input("??????")
-                    #     if _TEST:
-                    #         ans = input("continue?") or "n"
-                    #     if ans == "y" and _EXECUTE:
-                    #         status, output = subprocess.getstatusoutput(cmnd)
-                    #         #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True)
-                    #     progress = progress + step
-                    #     topic = "sectorq/amd/backups"
-                    #     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
-                    #     send_mqtt_message(msg) 
-                            
-                        
-                    cmnd = f"rm -rf {FULL_BACKUP_LATEST}"
-                    
-                    logging.info(cmnd)
-                    logging.info("Removing latest link")
-                #    input("????")
-                    if _EXECUTE:
-                        status, output = subprocess.getstatusoutput(cmnd)
-                    if _FIRST:
-                        cmnd = f"cd {BACKUP_ROOT}; ln -s initial latest"
-                    else:
-                        cmnd = f"cd {BACKUP_ROOT}; mv {DATETIME}_running {DATETIME};ln -s {DATETIME} latest"
-                    logging.info("Creating new latest link")
-                    #print(cmnd)
-                #    input("????")
-                    if _EXECUTE:
-                        status, output = subprocess.getstatusoutput(cmnd)
-
-
-                    #Remove old
-                    logging.info("Removing old dirs")
-                #    input("????")
-                    #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR)
-                    #cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;"
-                    
-                    cmnd = f"ls {BACKUP_ROOT}"
-                    
-                    
-                    #print(cmnd)
-                #   input("????")
-                    if _EXECUTE:
-                        status, output = subprocess.getstatusoutput(cmnd)
-                        for f in output.splitlines():
-                            pattern = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}$"  # regex pattern: string starts with 'abc'
-                            logging.info(f"Checking {f}")
-                            if re.match(pattern, f):
-                                logging.info("Match!")
-                                dt = datetime.datetime.strptime(f, "%Y-%m-%d_%H-%M-%S")
-                                epoch_time = int(dt.timestamp())
-                                now_epoch = int(datetime.datetime.now().timestamp())
-                                x = now_epoch - epoch_time
-                                # logging.info(epoch_time)  # Output: 45
-                                if x > 2592000:
-                                    dir_path = f"{BACKUP_ROOT}/{f}"
-                                    logging.info(f"removing {dir_path}")
-                                    shutil.rmtree(dir_path)
-                            else:
-                                print("No match.")
-                                
-                    cmnd = f"ls {BACKUP_ROOT}|grep _running"
-                    #print(cmnd)
-                #   input("????")
-                    logging.info(f"removing obsolete dirs")
-                    if _EXECUTE:
-                        status, output = subprocess.getstatusoutput(cmnd)
-                        for f in output.splitlines():
-                            dir_path = f"{BACKUP_ROOT}/{f}"
-                            logging.info(f"removing {dir_path}")
-                            shutil.rmtree(dir_path)
-  
-                    now = datetime.datetime.now()
-                    ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
-                    #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0}
-                    finished.append(b)
-                    msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
-                    send_mqtt_message(msg)
-
-                logging.info("Getting size of FS")
-                cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1"
-                logging.info(cmnd)
-                status, output = subprocess.getstatusoutput(cmnd)
-                used_space = (output.split())[0]
-                now = datetime.datetime.now()
-                ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S")
-                logging.info("Size : {}".format(used_space))
-                logging.info("Sending finished status")
-                #msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space}
-                msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space}
-                logging.info(msg)
-                
-                send_mqtt_message(msg)
-                os.remove(filename.path)
-    
-            
-            topic = "sectorq/amd/restore"
-            for s in servers:
-                #if s != "rack.home.lan":
-                if s == "m-server.home.lan":
-                    continue
-                elif s == "nas.home.lan":
-                    user = "admin"
-                    cmnd = "/share/Data/python/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
-                else:
-                    user = "jd"
-                    cmnd = "sudo /myapps/omv_backup.py -r all"
-                msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
-                logging.info(msg)
-                
-                send_mqtt_message(msg)
-                if is_port_open(s,22):
-                    ssh = paramiko.SSHClient()
-                    ssh.load_system_host_keys()
-                    # Add SSH host key automatically if needed.
-                    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-                    # Connect to router using username/password authentication.
-                    logger.info(f"Sync {s}")
-                    print(f"Sync {s}")
-                    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-                    pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa")
-                    ssh.connect(s, 
-                        username=user, 
-                        look_for_keys=False,
-                        allow_agent=False,
-                        pkey=pkey)
-                    print(cmnd)
-                    ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd)
-                    for line in iter(ssh_stdout.readline, ""):
-                        logger.info(line)
-                        print(line, end="")
-                    for line in iter(ssh_stderr.readline, ""):
-                        logger.info(line)
-                    ssh.close()
-            try:
-                os.remove("/backups/restore")
-            except:
-                pass
-                # if _MODE == "auto":
-                #     hostup = True
-                #     cmnd = "ssh root@omv.home.lan 'systemctl suspend &'"
-                #     status, output = subprocess.getstatusoutput(cmnd)
-                #     while hostup:
-                #         #HOST_UP  = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0
-                #         cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}"
-                #         status, output = subprocess.getstatusoutput(cmnd)
-                #         # print(status)
-                #         # print(output)
-                        
-                        
-                #         if status == 0:
-                #             print(f"Backup host up, waiting - {n}\r", end="")
-                #             time.sleep(5)
-                #             n += 1
-                #         else:
-                #             print("Backup host down                         " )
-                #             hostup = False
-                    
-                    
-                    
-                #     try:
-                #         url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off"
-
-                #         x = requests.post(url)
-
-                #         print(x.text)  
-                #     except:
-                #         pass
 
 if _SSH_TEST:                
     user = "root"