build

omv_backup.py

@@ -17,11 +17,18 @@ import fnmatch
 import yaml
 import paramiko
 import shutil
+import signal
 import paho.mqtt.client as mqtt
 #import numpy as np
 
+def signal_handler(sig, frame):
+    print('You pressed Ctrl+C!')
+    conn.close()
+    sys.exit(0)
 
 
+
+signal.signal(signal.SIGINT, signal_handler)
 file_path = os.path.realpath(__file__)
 dir_path = os.path.dirname(file_path)
 VERSION="1.0.9"
@@ -45,6 +52,7 @@ status, output = subprocess.getstatusoutput(cmnd)
 if int(output) > 0:
     print("Running already!")
     sys.exit()
+    
 def is_port_open(host, port):
     try:
         sock = socket.create_connection((host, port))
@@ -103,8 +111,6 @@ for o, a in opts:
        _RESTORE = True
        _APP =  a
        print("RESTORE")
-    elif o in ("-b", "--backup"):
-       _BACKUP = True
     elif o in ("-D", "--dry"):
        _EXECUTE = False
     elif o in ("-T", "--dry"):
@@ -198,14 +204,14 @@ logging.info("Test connection")
 hm = socket.gethostbyaddr(BACKUP_HOST)
 
 logging.info(_RESTORE)
-def send_mqtt_message(topic,msg):
+def send_mqtt_message(topic,msg,qos=0,retain=False):
     client2 = mqtt.Client()
-    client2.username_pw_set(USERNAME, PASSWORD)
+    client2.username_pw_set("jaydee", "jaydee1")
     try:
-        client2.connect(broker,1883,60)
+        client2.connect("mqtt.home.lan",1883,60)
-        client2.publish(topic, json.dumps(msg))
+        client2.publish(topic, json.dumps(msg), qos=qos, retain=retain)
         client2.disconnect()
-        logging.info(f"Message sent {topic}, {msg}")
+        logging.info(f"Message1 sent {topic}, {msg}")
     except ValueError as e:
         logging.error("Failed to send")
         print("Failed to send")
@@ -235,10 +241,7 @@ if _STOP:
         cmnd = f"docker stop {c.split()[-1]}"
         status, running_containers = subprocess.getstatusoutput(cmnd)
 def restore_job():
-    logging.info(f'starting restore job')
-    return 0
     logging.info("Starting Restore")
-    print("Starting Restore")
     now = datetime.datetime.now()
     STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
     if _APP == "all":
@@ -470,26 +473,28 @@ def restore_job():
         
     msg = {"mode":_MODE, "status":"restore","bak_name":"Restore","host":host,"cur_job":app,"start_time":STARTTIME,"end_time":"","progress":100,"finished":ENDJOB,"used_space":1}
     logging.info(msg)
-    
     send_mqtt_message(topic,msg)
     if _MODE == "auto":
         cmnd = "ssh root@amd.home.lan 'systemctl suspend &'"
         status, output = subprocess.getstatusoutput(cmnd)
 
 def backup_job(server):
+    client = mqtt.Client()
+    client.username_pw_set("jaydee", "jaydee1")
+    client.connect("mqtt.home.lan",1883,60)
     logging.info(f'starting backup job')
-    message = "start"
+    
-    topic = "sectorq/backups/start"
     finished = []
     now = datetime.datetime.now()
     STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
     topic = "sectorq/amd/restore"
     msg = {"mode":"restore", "status":"restore","bak_name":"s","host":0,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":0,"used_space":0}
-    send_mqtt_message(topic,msg)
+
+    client.publish(topic, json.dumps(msg),qos=0, retain=True)
     #client.publish(topic, msg)
     topic = "sectorq/amd/backups"
     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
-    send_mqtt_message(topic,msg)
+    client.publish(topic, json.dumps(msg),qos=0, retain=True)
     
     # iterate over files in 
     # that directory
@@ -502,7 +507,7 @@ def backup_job(server):
         if not backups[host]["jobs"][b]["active"]:
             logging.info("Backup {} is not active!".format(b))
             msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0}
-            send_mqtt_message(topic,msg)
+            client.publish(topic, json.dumps(msg),qos=0, retain=True)
             continue
 
         SOURCE_DIR = backups[host]["jobs"][b]["source"]
@@ -522,9 +527,9 @@ def backup_job(server):
 
         # msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0}
         msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
-        client.connect(broker,1883,60)
+        
         client.publish(topic, json.dumps(msg),qos=0, retain=True)
-        client.disconnect()
+        
 
 
         cmnd = "mkdir -p " + NEW_BACKUP_DIR
@@ -547,15 +552,11 @@ def backup_job(server):
         
         step = round(100 / c,1)
         progress = 0
-        #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
-        #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR]
-        
-        #cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude="home-assistant_v2.db"', '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', '--exclude="/.esphome/build"', '--exclude="nextcloud\mariadb\*"', NEW_BACKUP_DIR]
-        
         cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude-from=/myapps/exclude.txt', NEW_BACKUP_DIR]
         logging.info(" ".join(cmd))
         process = subprocess.Popen(cmd,
                 stdout=subprocess.PIPE)
+        
         while process.poll() is None:
             line = process.stdout.readline().decode("utf-8").split("/")
             print(line[0])
@@ -571,39 +572,12 @@ def backup_job(server):
                 #print(len(apps))
                 topic = "sectorq/amd/backups"
                 msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)}
-                send_mqtt_message(topic,msg)
+                client.publish(topic, json.dumps(msg),qos=0, retain=False)
                 progress = progress + step
-        # input(apps)
-
-        # for a in apps:
-        #     logging.info(f"App {a}")
-        #     topic = "sectorq/amd/backups"
-        #     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
-        #     send_mqtt_message(topic,msg) 
-        #     logger.debug(cmnd)
-        #     if _FIRST:
-        #         cmnd = f"rsync -avz --delete {SOURCE_DIR}  --exclude=\"jellyfin/cache/transcodes\"  --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}"     
-        #     else:
-        #         cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
-
-        #     ans = "y"
-        #     logging.info(cmnd)
-        #     logging.info("Sync files1")
-        #     #input("??????")
-        #     if _TEST:
-        #         ans = input("continue?") or "n"
-        #     if ans == "y" and _EXECUTE:
-        #         status, output = subprocess.getstatusoutput(cmnd)
-        #         #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True)
-        #     progress = progress + step
-        #     topic = "sectorq/amd/backups"
-        #     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
-        #     send_mqtt_message(topic,msg) 
-                
             
         cmnd = f"rm -rf {FULL_BACKUP_LATEST}"
         
-        logging.info(cmnd)
+        #logging.info(cmnd)
         logging.info("Removing latest link")
     #    input("????")
         if _EXECUTE:
@@ -618,25 +592,18 @@ def backup_job(server):
         if _EXECUTE:
             status, output = subprocess.getstatusoutput(cmnd)
 
-
         #Remove old
         logging.info("Removing old dirs")
-    #    input("????")
-        #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR)
-        #cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;"
         
         cmnd = f"ls {BACKUP_ROOT}"
-        
+
-        
-        #print(cmnd)
-    #   input("????")
         if _EXECUTE:
             status, output = subprocess.getstatusoutput(cmnd)
             for f in output.splitlines():
                 pattern = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}$"  # regex pattern: string starts with 'abc'
-                logging.info(f"Checking {f}")
+                # logging.info(f"Checking {f}")
                 if re.match(pattern, f):
-                    logging.info("Match!")
+                    # logging.info("Match!")
                     dt = datetime.datetime.strptime(f, "%Y-%m-%d_%H-%M-%S")
                     epoch_time = int(dt.timestamp())
                     
@@ -651,8 +618,6 @@ def backup_job(server):
                     print("No match.")
                     
         cmnd = f"ls {BACKUP_ROOT}|grep _running"
-        #print(cmnd)
-    #   input("????")
         logging.info(f"removing obsolete dirs")
         if _EXECUTE:
             status, output = subprocess.getstatusoutput(cmnd)
@@ -666,7 +631,7 @@ def backup_job(server):
         #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0}
         finished.append(b)
         msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
-        send_mqtt_message(topic,msg)
+        client.publish(topic, json.dumps(msg),qos=0, retain=True)
 
     logging.info("Getting size of FS")
     cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1"
@@ -681,11 +646,17 @@ def backup_job(server):
     msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space}
     logging.info(msg)
     
-    send_mqtt_message(topic,msg)
+    client.publish(topic, json.dumps(msg),qos=0, retain=True)
-
+    topic = "sectorq/backups/start"
+    logging.info(f"LALA : {topic}")
+    client.publish(topic, "finished",qos=0, retain=True)
+    client.disconnect()
+    return "finished"
+    
     
     topic = "sectorq/amd/restore"
     for s in servers:
+        logging.info(f"Restoring {s}")
         #if s != "rack.home.lan":
         if s == "m-server.home.lan":
             continue
@@ -696,9 +667,10 @@ def backup_job(server):
             user = "jd"
             cmnd = "sudo /myapps/omv_backup.py -r all"
         msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
-        logging.info(msg)
+        #logging.info(msg)
         
         send_mqtt_message(topic,msg)
+        continue
         if is_port_open(s,22):
             ssh = paramiko.SSHClient()
             ssh.load_system_host_keys()
@@ -726,40 +698,13 @@ def backup_job(server):
         os.remove("/backups/restore")
     except:
         pass
-        # if _MODE == "auto":
-        #     hostup = True
-        #     cmnd = "ssh root@omv.home.lan 'systemctl suspend &'"
-        #     status, output = subprocess.getstatusoutput(cmnd)
-        #     while hostup:
-        #         #HOST_UP  = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0
-        #         cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}"
-        #         status, output = subprocess.getstatusoutput(cmnd)
-        #         # print(status)
-        #         # print(output)
-                
-                
-        #         if status == 0:
-        #             print(f"Backup host up, waiting - {n}\r", end="")
-        #             time.sleep(5)
-        #             n += 1
-        #         else:
-        #             print("Backup host down                         " )
-        #             hostup = False
-            
-            
-            
-        #     try:
-        #         url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off"
 
-        #         x = requests.post(url)
+    return "finished1"
 
-        #         print(x.text)  
-        #     except:
-        #         pass
-    message= "finished1"
-    client.publish(topic, message,qos=2,retain=True)
-    return 0
 
+if _RESTORE:
+    restore_job()
+    sys.exit()
 if _SSH_TEST:                
     user = "root"
     cmnd = "ls -la"
@@ -771,7 +716,7 @@ if _SSH_TEST:
         #     user = "admin"
         #     cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
         msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
-        logging.info(msg)
+        #logging.info(msg)
         
         send_mqtt_message(topic,msg)
         if s != "rack.home.lan":
@@ -806,6 +751,8 @@ def handle_payload(payload):
     if payload == 'm-server':
         logging.info("💡 Starting backup job")
         backup_job(payload)
+        logging.info(f"💡 Finished backup job")
+
     else:
         logging.error(f"⚠️ Unknown command: {payload}")
 
@@ -836,4 +783,5 @@ if USE_TLS:
 
 # Connect and loop forever
 client.connect(BROKER, PORT, keepalive=60)
+client.publish("sectorq/backups/start", "finished", qos=0, retain=False)
 client.loop_forever()