added v3

omv_backup_v4.py

@@ -533,13 +533,26 @@ if _BACKUP:
                     
                     step = round(100 / c,1)
                     progress = 0
-                    cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
-                    with Popen(cmd, stdout=PIPE, bufsize=1, universal_newlines=True) as p:
-                        for line in p.stdout:
-                            print(line, end='') # process line here
-
-                    if p.returncode != 0:
-                        raise CalledProcessError(p.returncode, p.args)
+                    #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
+                    #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR]
+                    
+                    cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude="jellyfin/cache/transcodes"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', NEW_BACKUP_DIR]
+                    #print(cmd)
+                    process = subprocess.Popen(cmd,
+                           stdout=subprocess.PIPE)
+                    while process.poll() is None:
+                        line = process.stdout.readline().decode("utf-8").split("/")
+                        print(line[0])
+                        if line[0] in apps:
+                            logging.info(f"Working on app {line[0]}")
+                            apps.remove(line[0])
+                            #print(len(apps))
+                            topic = "sectorq/amd/backups"
+                            msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
+                            send_mqtt_message(msg)
+                            progress = progress + step
+                   # input(apps)
+                    
                     
                     # for a in apps:
                     #     logging.info(f"App {a}")
@@ -621,6 +634,7 @@ if _BACKUP:
             cmnd = "/myapps/venv/bin/python3 /myapps/omv_backup.py -r all"
             topic = "sectorq/amd/backups"
             for s in servers:
+                #if s != "rack.home.lan":
                 if s == "m-server.home.lan":
                     continue
                 elif s == "nas.home.lan":
@@ -638,11 +652,13 @@ if _BACKUP:
                     # Connect to router using username/password authentication.
                     logger.info(f"Sync {s}")
                     print(f"Sync {s}")
+                    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+                    pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa")
                     ssh.connect(s, 
                         username=user, 
-                        look_for_keys=True,
+                        look_for_keys=False,
                         allow_agent=False,
-                        key_filename="/root/.ssh/id_rsa")
+                        pkey=pkey)
                     print(cmnd)
                     ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd)
                     for line in iter(ssh_stdout.readline, ""):
@@ -683,18 +699,20 @@ if _BACKUP:
                 #         pass
 if _SSH_TEST:                
     user = "root"
-    cmnd = "/myapps/venv/bin/python3 /myapps/omv_backup.py -r all"
+    cmnd = "ls -la"
     topic = "sectorq/amd/backups"
     for s in servers:
-        if s == "m-server.home.lan":
-            continue
-        elif s == "nas.home.lan":
-            user = "admin"
-            cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
+        # if s == "m-server.home.lan":
+        #     continue
+        # elif s == "nas.home.lan":
+        #     user = "admin"
+        #     cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
         msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
         logging.info(msg)
         
         send_mqtt_message(msg)
+        if s != "rack.home.lan":
+            continue
         if is_port_open(s,22):
             ssh = paramiko.SSHClient()
             ssh.load_system_host_keys()
@@ -703,11 +721,13 @@ if _SSH_TEST:
             # Connect to router using username/password authentication.
             logger.info(f"Sync {s}")
             print(f"Sync {s}")
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+            pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa")
             ssh.connect(s, 
                 username=user, 
-                look_for_keys=True,
+                look_for_keys=False,
                 allow_agent=False,
-                key_filename="/root/.ssh/id_rsa")
+                pkey=pkey)
             print(cmnd)
             ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd)
             for line in iter(ssh_stdout.readline, ""):
@@ -715,4 +735,4 @@ if _SSH_TEST:
                 print(line, end="")
             for line in iter(ssh_stderr.readline, ""):
                 logger.info(line)
-            ssh.close()
+            ssh.close()