build

docker_backups.py

@@ -47,6 +47,13 @@ cmnd = f"zip -P {PASSWORD} authentik_{DATETIME}.sql.zip authentik_{DATETIME}.sql
 status, output = subprocess.getstatusoutput(cmnd)
 os.remove(f"authentik_{DATETIME}.sql")
 
+print("Backup Zabbix")
+cmnd = f"docker exec zabbix-server-db-server-1 sh -c 'pg_dump -h localhost -p 5432 -U zabbix -d zabbix' > /share/docker_data/__backups/zabbix_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} zabbix_{DATETIME}.sql.zip zabbix_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"zabbix_{DATETIME}.sql")
+
 
 print("Backup portainer")
 headers = {

exclude.txt

@@ -0,0 +1,13 @@
+
+ha/home-assistant_v2.db
+jellyfin/cache/transcodes
+.@__thumb/*
+ha/home-assistant.log
+gitlab/logs/*
+esphome/config/.esphome
+.cache
+.git
+var_lib_motioneye/*
+*/.esphome/build/*
+nextcloud/mariadb/*
+zabbix-server/postgres-data/*

omv_backup.py

@@ -17,8 +17,11 @@ import fnmatch
 import yaml
 import paramiko
 import shutil
+import paho.mqtt.client as mqtt
 #import numpy as np
 
+
+
 file_path = os.path.realpath(__file__)
 dir_path = os.path.dirname(file_path)
 VERSION="1.0.9"
@@ -125,13 +128,23 @@ logging.info("script started")
       
 logger = logging.getLogger(__name__) 
 client_id = "dasdasdasd333"
-try:
+# try:
-    client = mqtt_client.Client(mqtt_client.CallbackAPIVersion.VERSION1, client_id)
+#     client = mqtt_client.Client(mqtt_client.CallbackAPIVersion.VERSION1, client_id)
-except:
+# except:
-    client = mqtt_client.Client()
+#     client = mqtt_client.Client()
-client.username_pw_set(mqtt_username, mqtt_password)
+# client.username_pw_set(mqtt_username, mqtt_password)
 
 
+
+
+
+logging.info(f"Start OMV Backup")
+BROKER = 'mqtt.home.lan'       # e.g., 'mqtt.example.com'
+PORT = 1883                          # Typically 8883 for secure MQTT
+TOPIC = 'sectorq/backups/start'
+USERNAME = 'jaydee'
+PASSWORD = 'jaydee1'
+USE_TLS = False                       # Set to False if not using TLS
 backups = {
     "nas": {
         "login": "admin@nas.home.lan",
@@ -185,11 +198,14 @@ logging.info("Test connection")
 hm = socket.gethostbyaddr(BACKUP_HOST)
 
 logging.info(_RESTORE)
-def send_mqtt_message(msg):
+def send_mqtt_message(topic,msg):
+    client2 = mqtt.Client()
+    client2.username_pw_set(USERNAME, PASSWORD)
     try:
-        client.connect(broker,1883,60)
+        client2.connect(broker,1883,60)
-        client.publish(topic, json.dumps(msg))
+        client2.publish(topic, json.dumps(msg))
-        client.disconnect()
+        client2.disconnect()
+        logging.info(f"Message sent {topic}, {msg}")
     except ValueError as e:
         logging.error("Failed to send")
         print("Failed to send")
@@ -218,7 +234,9 @@ if _STOP:
             continue
         cmnd = f"docker stop {c.split()[-1]}"
         status, running_containers = subprocess.getstatusoutput(cmnd)
-if _RESTORE:
+def restore_job():
+    logging.info(f'starting restore job')
+    return 0
     logging.info("Starting Restore")
     print("Starting Restore")
     now = datetime.datetime.now()
@@ -244,7 +262,7 @@ if _RESTORE:
         logging.info(msg)
         
         
-        send_mqtt_message(msg)
+        send_mqtt_message(topic,msg)
         PROGRESS = PROGRESS + step
         now = datetime.datetime.now()
         DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S")
@@ -453,50 +471,38 @@ if _RESTORE:
     msg = {"mode":_MODE, "status":"restore","bak_name":"Restore","host":host,"cur_job":app,"start_time":STARTTIME,"end_time":"","progress":100,"finished":ENDJOB,"used_space":1}
     logging.info(msg)
     
-    send_mqtt_message(msg)
+    send_mqtt_message(topic,msg)
     if _MODE == "auto":
         cmnd = "ssh root@amd.home.lan 'systemctl suspend &'"
         status, output = subprocess.getstatusoutput(cmnd)
 
-if _BACKUP:
+def backup_job(server):
-    last = 1
+    logging.info(f'starting backup job')
-    while True:
+    message = "start"
-        directory = '/backups/'
+    topic = "sectorq/backups/start"
-        count = len(fnmatch.filter(os.listdir(directory), '*'))
-        if last != count:
-            logging.info(f'File Count: {count}')
-            last = count
-        if count == 0:
-            time.sleep(10)
-            continue
-        else:
     finished = []
     now = datetime.datetime.now()
     STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
     topic = "sectorq/amd/restore"
     msg = {"mode":"restore", "status":"restore","bak_name":"s","host":0,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":0,"used_space":0}
-            send_mqtt_message(msg)
+    send_mqtt_message(topic,msg)
+    #client.publish(topic, msg)
     topic = "sectorq/amd/backups"
     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
-            send_mqtt_message(msg)
+    send_mqtt_message(topic,msg)
     
     # iterate over files in 
     # that directory
     
-            for filename in os.scandir(directory):
+    
-                if filename.is_file():
+    host = server
-                    logging.info(filename.path)
-                    logging.info(filename.name)
-                if filename.name == "restore":
-                    break
-                host = filename.name
     logging.info("Backup")
     for b in backups[host]["jobs"]:
         
         if not backups[host]["jobs"][b]["active"]:
             logging.info("Backup {} is not active!".format(b))
             msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0}
-                        send_mqtt_message(msg)
+            send_mqtt_message(topic,msg)
             continue
 
         SOURCE_DIR = backups[host]["jobs"][b]["source"]
@@ -544,7 +550,9 @@ if _BACKUP:
         #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
         #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR]
         
-                    cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude="home-assistant_v2.db"', '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', '--exclude="/.esphome/build"', NEW_BACKUP_DIR]
+        #cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude="home-assistant_v2.db"', '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', '--exclude="/.esphome/build"', '--exclude="nextcloud\mariadb\*"', NEW_BACKUP_DIR]
+        
+        cmd = ['rsync', '-avz', '--delete', BACKUP_DIR,  '--link-dest', FULL_BACKUP_LATEST, '--exclude-from=/myapps/exclude.txt', NEW_BACKUP_DIR]
         logging.info(" ".join(cmd))
         process = subprocess.Popen(cmd,
                 stdout=subprocess.PIPE)
@@ -563,7 +571,7 @@ if _BACKUP:
                 #print(len(apps))
                 topic = "sectorq/amd/backups"
                 msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)}
-                            send_mqtt_message(msg)
+                send_mqtt_message(topic,msg)
                 progress = progress + step
         # input(apps)
 
@@ -571,7 +579,7 @@ if _BACKUP:
         #     logging.info(f"App {a}")
         #     topic = "sectorq/amd/backups"
         #     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
-                    #     send_mqtt_message(msg) 
+        #     send_mqtt_message(topic,msg) 
         #     logger.debug(cmnd)
         #     if _FIRST:
         #         cmnd = f"rsync -avz --delete {SOURCE_DIR}  --exclude=\"jellyfin/cache/transcodes\"  --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}"     
@@ -590,7 +598,7 @@ if _BACKUP:
         #     progress = progress + step
         #     topic = "sectorq/amd/backups"
         #     msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
-                    #     send_mqtt_message(msg) 
+        #     send_mqtt_message(topic,msg) 
                 
             
         cmnd = f"rm -rf {FULL_BACKUP_LATEST}"
@@ -625,12 +633,13 @@ if _BACKUP:
         if _EXECUTE:
             status, output = subprocess.getstatusoutput(cmnd)
             for f in output.splitlines():
-                            pattern = r"^[0-9]{4}.*"  # regex pattern: string starts with 'abc'
+                pattern = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}$"  # regex pattern: string starts with 'abc'
                 logging.info(f"Checking {f}")
                 if re.match(pattern, f):
                     logging.info("Match!")
                     dt = datetime.datetime.strptime(f, "%Y-%m-%d_%H-%M-%S")
                     epoch_time = int(dt.timestamp())
+                    
                     now_epoch = int(datetime.datetime.now().timestamp())
                     x = now_epoch - epoch_time
                     # logging.info(epoch_time)  # Output: 45
@@ -648,7 +657,7 @@ if _BACKUP:
         if _EXECUTE:
             status, output = subprocess.getstatusoutput(cmnd)
             for f in output.splitlines():
-                            dir_path = "{BACKUP_ROOT}/{f}"
+                dir_path = f"{BACKUP_ROOT}/{f}"
                 logging.info(f"removing {dir_path}")
                 shutil.rmtree(dir_path)
 
@@ -657,7 +666,7 @@ if _BACKUP:
         #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0}
         finished.append(b)
         msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
-                    send_mqtt_message(msg)
+        send_mqtt_message(topic,msg)
 
     logging.info("Getting size of FS")
     cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1"
@@ -672,8 +681,7 @@ if _BACKUP:
     msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space}
     logging.info(msg)
     
-                send_mqtt_message(msg)
+    send_mqtt_message(topic,msg)
-                os.remove(filename.path)
 
     
     topic = "sectorq/amd/restore"
@@ -690,7 +698,7 @@ if _BACKUP:
         msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
         logging.info(msg)
         
-                send_mqtt_message(msg)
+        send_mqtt_message(topic,msg)
         if is_port_open(s,22):
             ssh = paramiko.SSHClient()
             ssh.load_system_host_keys()
@@ -748,6 +756,9 @@ if _BACKUP:
         #         print(x.text)  
         #     except:
         #         pass
+    message= "finished1"
+    client.publish(topic, message,qos=2,retain=True)
+    return 0
 
 if _SSH_TEST:                
     user = "root"
@@ -762,7 +773,7 @@ if _SSH_TEST:
         msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
         logging.info(msg)
         
-        send_mqtt_message(msg)
+        send_mqtt_message(topic,msg)
         if s != "rack.home.lan":
             continue
         if is_port_open(s,22):
@@ -788,3 +799,41 @@ if _SSH_TEST:
             for line in iter(ssh_stderr.readline, ""):
                 logger.info(line)
             ssh.close()
+            
+# Define actions based on payload
+def handle_payload(payload):
+    payload = payload.lower()
+    if payload == 'm-server':
+        logging.info("💡 Starting backup job")
+        backup_job(payload)
+    else:
+        logging.error(f"⚠️ Unknown command: {payload}")
+
+# Callback when connected
+def on_connect(client, userdata, flags, rc):
+    if rc == 0:
+        logging.info("✅ Connected securely to broker")
+        client.subscribe(TOPIC)
+        logging.info(f"📡 Subscribed to topic: {TOPIC}")
+    else:
+        logging.error(f"❌ Connection failed with code {rc}")
+
+# Callback when a message is received
+def on_message(client, userdata, msg):
+    payload = msg.payload.decode()
+    logging.info(f"📨 Received message: {payload} on topic: {msg.topic}")
+    handle_payload(payload)
+
+# MQTT client setup
+client = mqtt.Client()
+client.username_pw_set(USERNAME, PASSWORD)
+client.on_connect = on_connect
+client.on_message = on_message
+
+# Use TLS for encrypted connection
+if USE_TLS:
+    client.tls_set()  # You can customize this with certs if needed
+
+# Connect and loop forever
+client.connect(BROKER, PORT, keepalive=60)
+client.loop_forever()