From 78306c75ff8e77c93204dbd5f86d9eca0440e520 Mon Sep 17 00:00:00 2001 From: jaydee Date: Sat, 11 Jan 2025 00:59:51 +0100 Subject: [PATCH] lala --- omv_backup_v2.pyw | 479 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 omv_backup_v2.pyw diff --git a/omv_backup_v2.pyw b/omv_backup_v2.pyw new file mode 100644 index 0000000..3554eeb --- /dev/null +++ b/omv_backup_v2.pyw @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +import datetime +import logging +from paho.mqtt import client as mqtt_client +import getopt +import json +import time +import socket +import subprocess +import sys +import os +import re +import platform +import requests +import fnmatch +from wakeonlan import send_magic_packet +pid = os.getpid() + +host = platform.node().lower() +cmnd = "ps -ef|grep omv_backups.py|grep -v grep |grep -v {}|wc -l".format(pid) +status, output = subprocess.getstatusoutput(cmnd) + +def is_port_open(host, port): + try: + sock = socket.create_connection((host, port)) + sock.close() + return True + except socket.error: + return False +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +# doesn't even have to be reachable +s.connect(('192.168.77.1', 1)) +IP = s.getsockname()[0] +print(IP) +print(output) +if int(output) > 0: + print("Running already!") + sys.exit() +broker = 'mqtt.home.lan' +port = 1883 +topic_sum = "sectorq/omv/backups" +mqtt_username = 'jaydee' +mqtt_password = 'jaydee1' +print("1") +try: + opts, args = getopt.getopt(sys.argv[1:], "amftdr:b", ["command=", "help", "output="]) +except getopt.GetoptError as err: + #usage() + sys.exit(2) +output = None +# QJ : getopts +_MODE = "manual" +_FIRST = _TEST = _RESTORE = _BACKUP = False +_EXECUTE = True +for o, a in opts: + if o == "-a": + _MODE = "auto" + elif o in ("-m", "--manual"): + _MODE = "manual" + elif o in ("-f", "--first"): + _FIRST = True + elif o in ("-t", "--test"): + _TEST = True + elif o in ("-r", "--restore"): + _RESTORE = True + _APP = a + print("RESTORE") + elif o in ("-b", "--backup"): + _BACKUP = True + elif o in ("-d", "--dry"): + _EXECUTE = False + +print("2") +client = mqtt_client.Client() +client.username_pw_set(mqtt_username, mqtt_password) +client.connect(broker,1883,60) + + +backups = { + "nas": { + "login": "admin@nas.home.lan", + "jobs": { + "github": + {"source":"/share/Data/__GITHUB", + "exclude":"", + "active": True + }, + "photo": { + "source":"/share/Photo/Years", + "exclude":"", + "active":True + } + } + }, + "m-server":{ + "login": "jd@m-server.home.lan", + "jobs": { + "docker_data":{ + "source":"/share/docker_data/", + "exclude":"", + "active":True + }, + "fail2ban":{ + "source":"/etc/fail2ban/", + "exclude":"", + "active":True + } + } + }, + "rpi5.home.lan":{ + "docker_data":{ + "source":"/share/docker_data/", + "exclude":"", + "active":True + }, + "fail2ban":{ + "source":"/etc/fail2ban/", + "exclude":"", + "active":True + } + } +} +BACKUP_FS = "/srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8" +BACKUP_HOST = "omv.home.lan" +#BACKUP_HOST = "morefine.home.lan" + +print("Test connection") +print("3") +hm = socket.gethostbyaddr(BACKUP_HOST) + +print("Starting") +print(_RESTORE) +if _RESTORE: + print("Starting Restore") + if _APP == "all": + _APP = ["nginx","ha","gitlab","mailu","bitwarden","esphome","grafana","ingluxdb","kestra","matter-server","mosquitto","octoprint","octoprint2","pihole","unify_block","webhub"] + else: + _APP = _APP.split(",") + + + for app in _APP: + topic = "sectorq/omv/restore/{}".format(app) + client.connect(broker,1883,60) + msg = {"status":"inactive","bak_name":app,"start_time":"inactive","end_time":"inactive","progress":0} + + client.publish(topic, json.dumps(msg)) + client.disconnect() + now = datetime.datetime.now() + DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") + BACKUP_HOST = f"jd@{host}" + BACKUP_DEVICE = "/srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8" + BACKUP_DIR = f"/backup/{host}" + if app == "fail2ban": + print("?>?????") + NEW_BACKUP_DIR = f"/backup/m-server/fail2ban/latest/" + SOURCE_DIR = f"/etc/fail2ban" + else: + NEW_BACKUP_DIR = f"/backup/m-server/docker_data/latest/{app}" + SOURCE_DIR = f"/share/docker_data/" + if _FIRST: + BACKUP_PATH="{}/initial".format(BACKUP_DIR) + else: + BACKUP_PATH="{}/{}".format(BACKUP_DIR, DATETIME) + LATEST_LINK="{}/latest".format(BACKUP_DIR) + FULL_BACKUP_LATEST = f"{NEW_BACKUP_DIR}/latest" + LATEST_LINK = f"/{host}/{app}/latest" + + + msg = {"status":"started","bak_name":app,"start_time":DATETIME,"end_time":"in progress", "progress":0} + client.connect(broker,1883,60) + client.publish(topic, json.dumps(msg)) + client.disconnect() + + print("Create backup dir") + print(cmnd) + + + #cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH) + + + if app == "heimdall": + print("Stopping docker") + cmnd = "docker stop heimdall" + status, output = subprocess.getstatusoutput(cmnd) + cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + ans = "y" + print(cmnd) + print("Sync files") + if _TEST: + ans = input("continue?") or "n" + if ans == "y" and _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + entries = ["Home Assistant","Nginx Proxy Manager","Portainer","Roundcube","Authentik","Kestra"] + for e in entries: + cmnd = f"sqlite3 /share/docker_data/heimdall/config/www/app.sqlite \"SELECT url FROM items WHERE title = '{e}'\"" + print(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + regex = re.compile(r'[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}') + contents = re.sub(regex, IP , output) + cmnd = f"sqlite3 /share/docker_data/heimdall/config/www/app.sqlite \"UPDATE items SET url = '{contents}' WHERE title = '{e}'\"" + print(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + cmnd = "docker start heimdall" + status, output = subprocess.getstatusoutput(cmnd) + if app == "ha": + print("Stopping docker") + cmnd = "docker stop heimdall" + status, output = subprocess.getstatusoutput(cmnd) + cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + ans = "y" + print(cmnd) + print("Sync files") + if _TEST: + ans = input("continue?") or "n" + if ans == "y" and _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + print("Start docker") + cmnd = "docker start heimdall" + status, output = subprocess.getstatusoutput(cmnd) + if app == "fail2ban": + print("Stopping docker") + cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + ans = "y" + print(cmnd) + print("Sync files") + if _TEST: + ans = input("continue?") or "n" + if ans == "y" and _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + print("Start docker") + cmnd = "docker start heimdall" + status, output = subprocess.getstatusoutput(cmnd) + elif app == "nginx": + print("Stopping docker") + cmnd = "docker stop nginx-app-1" + status, output = subprocess.getstatusoutput(cmnd) + cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + ans = "y" + print(cmnd) + print("Sync files") + if _TEST: + ans = input("continue?") or "n" + if ans == "y" and _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + domains = ["sectorq.eu","gitlab.sectorq.eu","ha.sectorq.eu","mail.sectorq.eu","pw.sectorq.eu","semaphore.sectorq.eu","kestra.sectorq.eu","auth.sectorq.eu"] + for d in domains: + cmnd = f'sqlite3 /share/docker_data/nginx/data/database.sqlite "UPDATE proxy_host SET forward_host = \'{IP}\' WHERE domain_names = \'[\\"{d}\\"]\'"' + print(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + + cmnd = 'egrep -l "# kestra.sectorq.eu|# auth.sectorq.eu|# ha.sectorq.eu|# pw.sectorq.eu|# semaphore.sectorq.eu|# sectorq.eu|# gitlab.sectorq.eu|# ha.sectorq.eu" /share/docker_data/nginx/data/nginx/proxy_host/*' + status, output = subprocess.getstatusoutput(cmnd) + print(output.splitlines()) + for file in output.splitlines(): + print(file) + f = open(file) + contents = f.read() + f.close() + regex = re.compile(r'\n\s+set \$server\s+\"\w+.\w+.\w+.\w+\";') + contents = re.sub(regex, f'\n set $server \"{IP}\";', contents) + #print(contents) + print(regex) + f = open(file, "w") + contents = f.write(contents) + f.close() + status, output = subprocess.getstatusoutput(cmnd) + print("Starting docker") + cmnd = "docker start nginx-app-1" + status, output = subprocess.getstatusoutput(cmnd) + else: + cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + ans = "y" + print(cmnd) + print("Sync files") + if _TEST: + ans = input("continue?") or "n" + if ans == "y" and _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + now = datetime.datetime.now() + ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + msg = {"status":"finished","bak_name":app,"start_time":DATETIME,"end_time":ENDTIME,"progress":0} + client.connect(broker,1883,10) + client.publish(topic, json.dumps(msg)) + client.disconnect() + + now = datetime.datetime.now() + ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") + print("Sending finished status") + msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":"?"} + print(msg) + client.connect(broker,1883,10) + client.publish(topic_sum, json.dumps(msg)) + client.disconnect() + + if _MODE == "auto": + cmnd = "ssh root@omv.home.lan 'systemctl suspend &'" + status, output = subprocess.getstatusoutput(cmnd) + + + +if _BACKUP: + + while True: + directory = '/backups/' + count = len(fnmatch.filter(os.listdir(directory), '*')) + + print('File Count:', count) + if count == 0: + time.sleep(10) + continue + else: + finished = [] + now = datetime.datetime.now() + STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + client.publish(topic_sum, json.dumps(msg)); + client.disconnect() + # iterate over files in + # that directory + + for filename in os.scandir(directory): + if filename.is_file(): + print(filename.path) + print(filename.name) + host = filename.name + print("Backup") + for b in backups[host]["jobs"]: + topic = "sectorq/omv/backups" + if not backups[host]["jobs"][b]["active"]: + print("Backup {} is not active!".format(b)) + client.connect(broker,1883,60) + msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0} + + client.publish(topic, json.dumps(msg)) + client.disconnect() + continue + + SOURCE_DIR = backups[host]["jobs"][b]["source"] + now = datetime.datetime.now() + BACKUP_HOST = backups[host]["login"] + BACKUP_DEVICE = "/srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8" + BACKUP_DIR = f"{BACKUP_HOST}:{SOURCE_DIR}" + BACKUP_ROOT = f"{BACKUP_DEVICE}/backup/{host}/{b}" + DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") + + if _FIRST: + NEW_BACKUP_DIR = f"{BACKUP_ROOT}/initial" + else: + NEW_BACKUP_DIR = f"{BACKUP_ROOT}/{DATETIME}" + + FULL_BACKUP_LATEST = f"{BACKUP_ROOT}/latest" + + + # msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0} + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + client.connect(broker,1883,60) + client.publish(topic, json.dumps(msg)) + client.disconnect() + + cmnd = "mkdir -p " + NEW_BACKUP_DIR + print(cmnd) + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + print(output) + print(status) + print("Create backup dir") + + + + #cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH) + if _FIRST: + cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" + else: + cmnd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST} --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" + + ans = "y" + print(cmnd) + print("Sync files") + #input("??????") + if _TEST: + + ans = input("continue?") or "n" + if ans == "y" and _EXECUTE: + + # rsync --info=progress2 -avz --delete /share/docker_data/ --link-dest /m-server/docker_data/latest --exclude="gitlab/data/" --exclude="esphome/config/.esphome" --exclude="gitlab/logs/prometheus" --exclude=".cache" --exclude=".git" --exclude="var_lib_motioneye" /m-server/m-server/docker_data/newone1 + + + # input("????") + + status, output = subprocess.getstatusoutput(cmnd) + #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True) + + cmnd = f"rm -rf {FULL_BACKUP_LATEST}" + + print(cmnd) + print("Removing latest link") + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + if _FIRST: + cmnd = f"cd {BACKUP_ROOT}; ln -s initial latest" + else: + cmnd = f"cd {BACKUP_ROOT}; ln -s {DATETIME} latest" + print("Creating new latest link") + #print(cmnd) + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + + + #Remove old + print("Removing old dirs") + # input("????") + #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR) + cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;" + #print(cmnd) + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + now = datetime.datetime.now() + ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0} + finished.append(b) + msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + client.connect(broker,1883,10) + client.publish(topic, json.dumps(msg)) + client.disconnect() + + print("Getting size of FS") + cmnd = "df -h /srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8|awk '{ print $3 }'|tail -1" + print(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + used_space = (output.split())[0] + now = datetime.datetime.now() + ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") + print("Size : {}".format(used_space)) + print("Sending finished status") + #msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space} + msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished)} + print(msg) + + + client.connect(broker,1883,10) + client.publish(topic_sum, json.dumps(msg)) + client.disconnect() + + os.remove(filename.path) + # if _MODE == "auto": + # hostup = True + # cmnd = "ssh root@omv.home.lan 'systemctl suspend &'" + # status, output = subprocess.getstatusoutput(cmnd) + # while hostup: + # #HOST_UP = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0 + # cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}" + # status, output = subprocess.getstatusoutput(cmnd) + # # print(status) + # # print(output) + + + # if status == 0: + # print(f"Backup host up, waiting - {n}\r", end="") + # time.sleep(5) + # n += 1 + # else: + # print("Backup host down " ) + # hostup = False + + + + # try: + # url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off" + + # x = requests.post(url) + + # print(x.text) + # except: + # pass + + \ No newline at end of file