From 1ee67ce3e9d7ac57bd31259123f705cebbd9343d Mon Sep 17 00:00:00 2001 From: jaydee Date: Mon, 5 May 2025 18:27:55 +0200 Subject: [PATCH] build --- omv_backup.py | 900 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 575 insertions(+), 325 deletions(-) diff --git a/omv_backup.py b/omv_backup.py index c5d2b99..fa8f045 100755 --- a/omv_backup.py +++ b/omv_backup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/myapps/venv/bin/python3 import datetime import logging from paho.mqtt import client as mqtt_client @@ -7,19 +7,39 @@ import json import time import socket import subprocess +from subprocess import Popen, PIPE, CalledProcessError import sys import os import re import platform import requests import fnmatch +import yaml +import paramiko +import numpy as np +file_path = os.path.realpath(__file__) +dir_path = os.path.dirname(file_path) +VERSION="1.0.1" +print(file_path) +print(dir_path) +os.chdir(dir_path) from wakeonlan import send_magic_packet pid = os.getpid() - +def is_port_open(host, port): + try: + sock = socket.create_connection((host, port)) + sock.close() + return True + except socket.error: + return False +servers = ["rpi5.home.lan","nas.home.lan","rack.home.lan","m-server.home.lan"] host = platform.node().lower() +#input(host) cmnd = "ps -ef|grep omv_backups.py|grep -v grep |grep -v {}|wc -l".format(pid) status, output = subprocess.getstatusoutput(cmnd) - +if int(output) > 0: + print("Running already!") + sys.exit() def is_port_open(host, port): try: sock = socket.create_connection((host, port)) @@ -29,80 +49,120 @@ def is_port_open(host, port): return False s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # doesn't even have to be reachable -s.connect(('192.168.77.1', 1)) -IP = s.getsockname()[0] -print(IP) -print(output) -if int(output) > 0: - print("Running already!") - sys.exit() +conn = False +while not conn: + try: + s.connect(('192.168.77.1', 1)) + IP = s.getsockname()[0] + print(IP) + print(output) + conn = True + except: + time.sleep(5) + broker = 'mqtt.home.lan' port = 1883 -topic_sum = "sectorq/omv/backups" +topic_sum = "sectorq/amd/backups" mqtt_username = 'jaydee' mqtt_password = 'jaydee1' print("1") try: - opts, args = getopt.getopt(sys.argv[1:], "amftdr:b", ["command=", "help", "output="]) + opts, args = getopt.getopt(sys.argv[1:], "hTamftDr:bd:sSOl:", ["command=", "help", "output="]) except getopt.GetoptError as err: #usage() sys.exit(2) output = None # QJ : getopts _MODE = "manual" -_FIRST = _TEST = _RESTORE = _BACKUP = False +_FIRST = _TEST = _RESTORE = _BACKUP = _SYNC = _START = _STOP = _SSH_TEST = False _EXECUTE = True +_DATE = "pick" +_LOG_LEVEL = "" for o, a in opts: if o == "-a": _MODE = "auto" elif o in ("-m", "--manual"): _MODE = "manual" + elif o in ("-l", "--level"): + _LOG_LEVEL = a.upper() elif o in ("-f", "--first"): _FIRST = True + elif o in ("-d", "--date"): + _DATE = a elif o in ("-t", "--test"): _TEST = True + elif o in ("-s", "--sync"): + _SYNC = True + elif o in ("-S", "--start"): + _START = True + elif o in ("-O", "--stop"): + _STOP = True elif o in ("-r", "--restore"): _RESTORE = True _APP = a print("RESTORE") elif o in ("-b", "--backup"): _BACKUP = True - elif o in ("-d", "--dry"): + elif o in ("-D", "--dry"): _EXECUTE = False - -print("2") -client = mqtt_client.Client() + elif o in ("-T", "--dry"): + _SSH_TEST = True + elif o in ("-h", "--help"): + print(VERSION) + sys.exit() +LOG_FILE = "omv_backup.log" +if _LOG_LEVEL == "DEBUG": + logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + logging.debug('using debug loging') +elif _LOG_LEVEL == "ERROR": + logging.basicConfig(filename=LOG_FILE, level=logging.ERROR, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + logging.info('using error loging') +elif _LOG_LEVEL == "SCAN": + logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') + logging.info('using error loging') +else: + logging.basicConfig(filename=LOG_FILE, level=logging.INFO, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') +logging.info("script started") + +logger = logging.getLogger(__name__) +print("2") +client_id = "dasdasdasd333" +try: + client = mqtt_client.Client(mqtt_client.CallbackAPIVersion.VERSION1, client_id) +except: + client = mqtt_client.Client() client.username_pw_set(mqtt_username, mqtt_password) -client.connect(broker,1883,60) -now = datetime.datetime.now() -STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") -msg = {"mode":_MODE, "status":"started","bak_name":"complete","start_time":STARTTIME,"end_time":"in progress","progress":0} -client.publish(topic_sum, json.dumps(msg)); -client.disconnect() + backups = { "nas": { - "github": - {"source":"/share/Data/__GITHUB", - "exclude":"", - "active": True - }, - "photo": { - "source":"/share/Photo/Years", - "exclude":"", - "active":True - }, + "login": "admin@nas.home.lan", + "jobs": { + "github": + {"source":"/share/Data/__GITHUB", + "exclude":"", + "active": True + }, + "photo": { + "source":"/share/Photo/Years", + "exclude":"", + "active":True + } + } }, "m-server":{ - "docker_data":{ - "source":"/share/docker_data/", - "exclude":"", - "active":True - }, - "fail2ban":{ - "source":"/etc/fail2ban/", - "exclude":"", - "active":True + "login": "root@m-server.home.lan", + "jobs": { + "docker_data":{ + "source":"/share/docker_data/", + "exclude":"", + "active":True + }, + "fail2ban":{ + "source":"/etc/fail2ban/", + "exclude":"", + "active":True + } } }, "rpi5.home.lan":{ @@ -118,162 +178,236 @@ backups = { } } } -BACKUP_FS = "/srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8" -BACKUP_HOST = "omv.home.lan" + +BACKUP_FS = "/media/backup/" +BACKUP_HOST = "amd.home.lan" #BACKUP_HOST = "morefine.home.lan" -if not host in backups and _BACKUP: - print(f"No backup jobs for {host}") - sys.exit() -print("Test connection") -print("3") + +logging.info("Test connection") hm = socket.gethostbyaddr(BACKUP_HOST) - -hostdown = True -n=0 -try: - url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=on" - - x = requests.post(url) - - print(x.text) -except: - pass -while hostdown: - #HOST_UP = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0 - cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}" - status, output = subprocess.getstatusoutput(cmnd) - # print(status) - # print(output) - - - if status != 0: - send_magic_packet('88:c9:b3:b5:23:d8') - print(f"Backup host down, waiting - {n}\r", end="") - time.sleep(5) - n += 1 - else: - print("Backup host up " ) - hostdown = False - -port = 22 # Replace with the port you want to test -n=0 -while not is_port_open(BACKUP_HOST, port): - print(f"Port {port} on {BACKUP_HOST} is closed. {n}\r", end="") - time.sleep(5) - n += 1 -print(f"Port {port} on {BACKUP_HOST} is open.") - -print("Starting") -print(_RESTORE) -if _RESTORE: - print("Starting Restore") - if _APP == "all": - _APP = ["nginx","ha","gitlab","mailu","bitwarden","esphome","grafana","ingluxdb","kestra","matter-server","mosquitto","octoprint","octoprint2","pihole","unify_block","webhub"] - else: - _APP = _APP.split(",") - - - for app in _APP: - topic = "sectorq/omv/restore/{}".format(app) +logging.info(_RESTORE) +def send_mqtt_message(msg): + try: client.connect(broker,1883,60) - msg = {"status":"inactive","bak_name":app,"start_time":"inactive","end_time":"inactive","progress":0} - client.publish(topic, json.dumps(msg)) client.disconnect() + except ValueError as e: + logging.error("Failed to send") + print("Failed to send") + print(e) + +if _SYNC: + containers = ["HomeAssistant","webhub-web-1","heimdall","pihole","mosquitto-mosquitto-1","mailu3-redis-1","mailu3-webmail-1","mailu3-resolver-1","mailu3-antispam-1","mailu3-webdav-1","mailu3-smtp-1","mailu3-oletools-1","mailu3-front-1","mailu3-fetchmail-1","mailu3-imap-1","matter-server","piper-en","openwakeword","whisper-en","auth-worker-1","auth-server-1","auth-authentik_ldap-1","auth-redis-1","auth-postgresql-1","nginx-app-1"] + + cmnd = f"curl -H 'Authorization: Bearer l4c1j4yd33Du5lo' 192.168.77.238:8094/v1/update" + logging.info(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + + if _START: + for c in containers: + cmnd = f"docker start {c}" + print(cmnd) + status, output = subprocess.getstatusoutput(cmnd) +if _STOP: + cmnd = "docker ps" + status, running_containers = subprocess.getstatusoutput(cmnd) + + logging.info(running_containers) + for c in running_containers.splitlines(): + print(c.split()[-1]) + if c.split()[-1] == "watchtower-watchtower-1": + continue + cmnd = f"docker stop {c.split()[-1]}" + status, running_containers = subprocess.getstatusoutput(cmnd) + +if _RESTORE: + logging.info("Starting Restore") + print("Starting Restore") + now = datetime.datetime.now() + STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + if _APP == "all": + _DATE = "latest" + if host == "rpi5.home.lan": + _APP = ["nginx","ha","gitea","gitlab","mailu","bitwarden","esphome","grafana","ingluxdb","kestra","matter-server","mosquitto","octoprint","octoprint2","pihole","unify_block","webhub","homepage","watchtower"] + else: + cmnd = "ssh root@amd.home.lan 'ls /mnt/raid/backup/m-server/docker_data/latest'" + status, output = subprocess.getstatusoutput(cmnd) + _APP = output.splitlines() + logging.info(_APP) + #input("????") + else: + _APP = _APP.split(",") + + + + PROGRESS = 0 + topic = "sectorq/amd/restore" + step = 100 / len(_APP) + for app in _APP: + msg = {"mode":_MODE, "status":"restore","bak_name":"Restore","host":host,"cur_job":app,"start_time":STARTTIME,"end_time":"","progress":str(round(np.ceil(PROGRESS))) + "%","finished":1,"used_space":1} + logging.info(msg) + + send_mqtt_message(msg) + PROGRESS = PROGRESS + step now = datetime.datetime.now() DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") - BACKUP_HOST = "root@omv.home.lan" - BACKUP_DEVICE = "/srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8" + BACKUP_HOST = f"root@amd.home.lan" + BACKUP_DEVICE = "/mnt/raid" BACKUP_DIR = f"/backup/{host}" + + + if _DATE == "pick": + cmnd = f"ssh root@amd.home.lan 'ls {BACKUP_DEVICE}/backup/m-server/docker_data'" + status, output = subprocess.getstatusoutput(cmnd) + #print(output) + dates = output.splitlines() + n = 1 + for i in dates: + logging.info(f"{n} - {i}" ) + n += 1 + + ans = input("Pick a backup to restore : ") + _DATE = dates[int(ans) - 1] + + if app == "fail2ban": - print("?>?????") - NEW_BACKUP_DIR = f"/backup/m-server/fail2ban/latest/" + logging.info("?>?????") + NEW_BACKUP_DIR = f"/backup/m-server/fail2ban/{_DATE}/" SOURCE_DIR = f"/etc/fail2ban" else: - NEW_BACKUP_DIR = f"/backup/m-server/docker_data/latest/{app}" + NEW_BACKUP_DIR = f"/backup/m-server/docker_data/{_DATE}/{app}" SOURCE_DIR = f"/share/docker_data/" if _FIRST: BACKUP_PATH="{}/initial".format(BACKUP_DIR) else: BACKUP_PATH="{}/{}".format(BACKUP_DIR, DATETIME) - LATEST_LINK="{}/latest".format(BACKUP_DIR) - FULL_BACKUP_LATEST = f"{NEW_BACKUP_DIR}/latest" - LATEST_LINK = f"/{host}/{app}/latest" + LATEST_LINK="{}/{}".format(BACKUP_DIR,_DATE) + FULL_BACKUP_LATEST = f"{NEW_BACKUP_DIR}/{_DATE}" + LATEST_LINK = f"/{host}/{app}/{_DATE}" - - msg = {"status":"started","bak_name":app,"start_time":DATETIME,"end_time":"in progress", "progress":0} - client.connect(broker,1883,60) - client.publish(topic, json.dumps(msg)) - client.disconnect() - - print("Create backup dir") - print(cmnd) + logging.info("Create backup dir") + logging.info(cmnd) #cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH) if app == "heimdall": - print("Stopping docker") + logging.info("Stopping docker") cmnd = "docker stop heimdall" status, output = subprocess.getstatusoutput(cmnd) - cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" - print(cmnd) - print("Sync files") + logging.info(cmnd) + logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) + + entries = ["Home Assistant","Nginx Proxy Manager","Portainer","Roundcube","Authentik","Kestra"] for e in entries: cmnd = f"sqlite3 /share/docker_data/heimdall/config/www/app.sqlite \"SELECT url FROM items WHERE title = '{e}'\"" - print(cmnd) + logging.info(cmnd) status, output = subprocess.getstatusoutput(cmnd) regex = re.compile(r'[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}') contents = re.sub(regex, IP , output) cmnd = f"sqlite3 /share/docker_data/heimdall/config/www/app.sqlite \"UPDATE items SET url = '{contents}' WHERE title = '{e}'\"" - print(cmnd) + logging.info(cmnd) status, output = subprocess.getstatusoutput(cmnd) - cmnd = "docker start heimdall" - status, output = subprocess.getstatusoutput(cmnd) + # cmnd = "docker start heimdall" + # status, output = subprocess.getstatusoutput(cmnd) + + + + if app == "ha": - print("Stopping docker") + logging.info("Stopping docker") cmnd = "docker stop heimdall" status, output = subprocess.getstatusoutput(cmnd) - cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" - print(cmnd) - print("Sync files") + logging.info(cmnd) + logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) - print("Start docker") - cmnd = "docker start heimdall" - status, output = subprocess.getstatusoutput(cmnd) - if app == "fail2ban": - print("Stopping docker") - cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + logging.info("Start docker") + # cmnd = "docker start heimdall" + # status, output = subprocess.getstatusoutput(cmnd) + elif app == "fail2ban": + logging.info("Stopping docker") + cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" - print(cmnd) - print("Sync files") + logging.info(cmnd) + logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) - print("Start docker") - cmnd = "docker start heimdall" - status, output = subprocess.getstatusoutput(cmnd) - elif app == "nginx": - print("Stopping docker") + logging.info("Start docker") + # cmnd = "docker start heimdall" + # status, output = subprocess.getstatusoutput(cmnd) + elif app == "homepage": + logging.info("Stopping docker") + cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" + ans = "y" + logging.info(cmnd) + + if _TEST: + ans = input("continue?") or "n" + if ans == "y" and _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + + file = "/share/docker_data/homepage/config/widgets.yaml" + with open(file, 'r') as stream: + try: + loaded = yaml.load(stream, Loader=yaml.FullLoader) + except yaml.YAMLError as exc: + logging.info(exc) + + # Modify the fields from the dict + #loaded['logo']['icon'] = "/images/morefine2.png" + logging.info(json.dumps(loaded, indent=2)) + i = 0 + for y in loaded: + logging.info(i) + logging.info(y) + + if "logo" in y: + if host == "rpi5.home.lan": + loaded[i]['logo']['icon'] = "/images/rpi5.png" + elif host == "nas.home.lan": + loaded[i]['logo']['icon'] = "/images/qnap_nas.png" + else: + loaded[i]['logo']['icon'] = "/images/morefine2.png" + i+=1 + + # Save it again + logging.info(f"writing to file {file}") + with open(file, 'w') as stream: + try: + yaml.dump(loaded, stream, default_flow_style=False) + except yaml.YAMLError as exc: + print("failed") + print(exc) + + + + logging.info("Start docker") + # cmnd = "docker start heimdall" + # status, output = subprocess.getstatusoutput(cmnd) + elif app == "nginx1": + logging.info("Stopping docker") cmnd = "docker stop nginx-app-1" status, output = subprocess.getstatusoutput(cmnd) - cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" - print(cmnd) - print("Sync files") + logging.info(cmnd) + logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: @@ -281,218 +415,334 @@ if _RESTORE: domains = ["sectorq.eu","gitlab.sectorq.eu","ha.sectorq.eu","mail.sectorq.eu","pw.sectorq.eu","semaphore.sectorq.eu","kestra.sectorq.eu","auth.sectorq.eu"] for d in domains: cmnd = f'sqlite3 /share/docker_data/nginx/data/database.sqlite "UPDATE proxy_host SET forward_host = \'{IP}\' WHERE domain_names = \'[\\"{d}\\"]\'"' - print(cmnd) + logging.info(cmnd) status, output = subprocess.getstatusoutput(cmnd) - cmnd = 'egrep -l "# kestra.sectorq.eu|# auth.sectorq.eu|# ha.sectorq.eu|# pw.sectorq.eu|# semaphore.sectorq.eu|# sectorq.eu|# gitlab.sectorq.eu|# ha.sectorq.eu" /share/docker_data/nginx/data/nginx/proxy_host/*' + cmnd = 'egrep -l "# bazarr.sectorq.eu|# gitea.sectorq.eu|# jf.sectorq.eu|# kestra.sectorq.eu|# auth.sectorq.eu|# ha.sectorq.eu|# pw.sectorq.eu|# semaphore.sectorq.eu|# sectorq.eu|# gitlab.sectorq.eu|# ha.sectorq.eu" /share/docker_data/nginx/data/nginx/proxy_host/*' status, output = subprocess.getstatusoutput(cmnd) - print(output.splitlines()) + logging.info(output.splitlines()) for file in output.splitlines(): - print(file) + logging.info(file) f = open(file) contents = f.read() f.close() regex = re.compile(r'\n\s+set \$server\s+\"\w+.\w+.\w+.\w+\";') contents = re.sub(regex, f'\n set $server \"{IP}\";', contents) #print(contents) - print(regex) + logging.info(regex) f = open(file, "w") contents = f.write(contents) f.close() status, output = subprocess.getstatusoutput(cmnd) - print("Starting docker") - cmnd = "docker start nginx-app-1" - status, output = subprocess.getstatusoutput(cmnd) + logging.info("Starting docker") + # cmnd = "docker start nginx-app-1" + # status, output = subprocess.getstatusoutput(cmnd) else: - cmnd = f"rsync -avz --delete rsync://{BACKUP_HOST}{NEW_BACKUP_DIR} {SOURCE_DIR}" + cmnd = f"rsync -avz --delete {BACKUP_HOST}:{BACKUP_DEVICE}{NEW_BACKUP_DIR} {SOURCE_DIR}" ans = "y" - print(cmnd) - print("Sync files") + logging.info(cmnd) + logging.info("Sync files") if _TEST: ans = input("continue?") or "n" if ans == "y" and _EXECUTE: status, output = subprocess.getstatusoutput(cmnd) - now = datetime.datetime.now() - ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S") - msg = {"status":"finished","bak_name":app,"start_time":DATETIME,"end_time":ENDTIME,"progress":0} - client.connect(broker,1883,10) - client.publish(topic, json.dumps(msg)) - client.disconnect() - - now = datetime.datetime.now() - ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") - print("Sending finished status") - msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":"?"} - print(msg) - client.connect(broker,1883,10) - client.publish(topic_sum, json.dumps(msg)) - client.disconnect() - + now = datetime.datetime.now() + ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") + logging.info("Sending finished status") + + msg = {"mode":_MODE, "status":"restore","bak_name":"Restore","host":host,"cur_job":app,"start_time":STARTTIME,"end_time":"","progress":100,"finished":ENDJOB,"used_space":1} + logging.info(msg) + + send_mqtt_message(msg) if _MODE == "auto": - cmnd = "ssh root@omv.home.lan 'systemctl suspend &'" + cmnd = "ssh root@amd.home.lan 'systemctl suspend &'" status, output = subprocess.getstatusoutput(cmnd) - - if _BACKUP: - + last = 1 while True: directory = '/backups/' - count = len(fnmatch.filter(os.listdir(directory), '*.*')) - print('File Count:', count) - time.sleep(10) - continue - - - - # iterate over files in - # that directory - for filename in os.scandir(directory): - if filename.is_file(): - - print(filename.path) - print("Backup") - for b in backups[host]: - topic = "sectorq/omv/backups/{}".format(b.lower()) - if not backups[host][b]["active"]: - print("Backup {} is not active!".format(b)) - client.connect(broker,1883,60) - msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0} - - client.publish(topic, json.dumps(msg)) - client.disconnect() - continue - - SOURCE_DIR = backups[host][b]["source"] - now = datetime.datetime.now() - BACKUP_HOST = "root@omv.home.lan" - BACKUP_DEVICE = "/srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8" - BACKUP_DIR = f"/backup/{host}/{b}" - NEW_BACKUP_DIR = f"{BACKUP_DEVICE}/backup/{host}/{b}" - DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") - if _FIRST: - BACKUP_PATH="{}/initial".format(BACKUP_DIR) - else: - BACKUP_PATH="{}/{}".format(BACKUP_DIR, DATETIME) - LATEST_LINK="{}/latest".format(BACKUP_DIR) - FULL_BACKUP_LATEST = f"{NEW_BACKUP_DIR}/latest" - LATEST_LINK = f"/{host}/{b}/latest" - - - msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0} - client.connect(broker,1883,60) - client.publish(topic, json.dumps(msg)) - client.disconnect() - - cmnd = "ssh root@omv.home.lan 'mkdir -p " + NEW_BACKUP_DIR + "'" - - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - print("Create backup dir") - print(cmnd) - - - #cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH) - if _FIRST: - cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" - else: - cmnd = f"rsync -avz --delete {SOURCE_DIR} --link-dest {LATEST_LINK} --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" - - ans = "y" - print(cmnd) - print("Sync files") - if _TEST: - - - ans = input("continue?") or "n" - if ans == "y" and _EXECUTE: - - # rsync --info=progress2 -avz --delete /share/docker_data/ --link-dest /m-server/docker_data/latest --exclude="gitlab/data/" --exclude="esphome/config/.esphome" --exclude="gitlab/logs/prometheus" --exclude=".cache" --exclude=".git" --exclude="var_lib_motioneye" /m-server/m-server/docker_data/newone1 - - - # input("????") - - status, output = subprocess.getstatusoutput(cmnd) - - cmnd = f"ssh root@omv.home.lan 'rm -rf {FULL_BACKUP_LATEST}'" - - #print(cmnd) - print("Removing latest link") - # input("????") - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - if _FIRST: - cmnd = f"ssh root@omv.home.lan 'cd {NEW_BACKUP_DIR}; ln -s initial latest'" - else: - cmnd = f"ssh root@omv.home.lan 'cd {NEW_BACKUP_DIR}; ln -s {DATETIME} latest'" - print("Creating new latest link") - #print(cmnd) - # input("????") - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - - - #Remove old - print("Removing old dirs") - # input("????") - #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR) - cmnd = f"cd {NEW_BACKUP_DIR} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;" - #print(cmnd) - # input("????") - if _EXECUTE: - status, output = subprocess.getstatusoutput(cmnd) - now = datetime.datetime.now() - ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S") - msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0} - client.connect(broker,1883,10) - client.publish(topic, json.dumps(msg)) - client.disconnect() - - print("Getting size of FS") - #cmnd = "du -h --max-depth=0 {}".format(BACKUP_FS) - cmnd = "ssh root@omv.home.lan 'df -h /srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8|awk '\\''{ print $3 }'\\''|tail -1'" - print(cmnd) - status, output = subprocess.getstatusoutput(cmnd) - used_space = (output.split())[0] + count = len(fnmatch.filter(os.listdir(directory), '*')) + if last != count: + logging.info(f'File Count: {count}') + last = count + if count == 0: + time.sleep(10) + continue + else: + finished = [] now = datetime.datetime.now() - ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") - print("Size : {}".format(used_space)) - print("Sending finished status") - msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space} - print(msg) - client.connect(broker,1883,10) - client.publish(topic_sum, json.dumps(msg)) - client.disconnect() + STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + topic = "sectorq/amd/backups" + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + send_mqtt_message(msg) + # iterate over files in + # that directory + + for filename in os.scandir(directory): + if filename.is_file(): + logging.info(filename.path) + logging.info(filename.name) + host = filename.name + logging.info("Backup") + for b in backups[host]["jobs"]: + + if not backups[host]["jobs"][b]["active"]: + logging.info("Backup {} is not active!".format(b)) + msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0} + send_mqtt_message(msg) + continue - if _MODE == "auto": - hostup = True - cmnd = "ssh root@omv.home.lan 'systemctl suspend &'" - status, output = subprocess.getstatusoutput(cmnd) - while hostup: - #HOST_UP = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0 - cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}" - status, output = subprocess.getstatusoutput(cmnd) - # print(status) - # print(output) + SOURCE_DIR = backups[host]["jobs"][b]["source"] + now = datetime.datetime.now() + BACKUP_HOST = backups[host]["login"] + BACKUP_DEVICE = "/mnt/raid" + BACKUP_DIR = f"{BACKUP_HOST}:{SOURCE_DIR}" + BACKUP_ROOT = f"{BACKUP_DEVICE}/backup/{host}/{b}" + DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S") - - if status == 0: - print(f"Backup host up, waiting - {n}\r", end="") - time.sleep(5) - n += 1 + if _FIRST: + NEW_BACKUP_DIR = f"{BACKUP_ROOT}/initial" else: - print("Backup host down " ) - hostup = False - - - - try: - url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off" + NEW_BACKUP_DIR = f"{BACKUP_ROOT}/{DATETIME}_running" + + FULL_BACKUP_LATEST = f"{BACKUP_ROOT}/latest" - x = requests.post(url) + # msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0} + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + client.connect(broker,1883,60) + client.publish(topic, json.dumps(msg),qos=0, retain=True) + client.disconnect() - print(x.text) - except: - pass + + cmnd = "mkdir -p " + NEW_BACKUP_DIR + logging.info(cmnd) + + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + logging.info(output) + logging.info(status) + logging.info("Create backup dir") + + cmnd = f"ssh {BACKUP_HOST} 'ls {SOURCE_DIR}'" + logger.debug(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + logger.debug(output) + apps = output.splitlines() + c = len(apps) + print(apps) + print(len(apps)) + + step = round(100 / c,1) + progress = 0 + #cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" + #cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR] + + cmd = ['rsync', '-avz', '--delete', BACKUP_DIR, '--link-dest', FULL_BACKUP_LATEST, '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', NEW_BACKUP_DIR] + logging.info(" ".join(cmd)) + process = subprocess.Popen(cmd, + stdout=subprocess.PIPE) + while process.poll() is None: + line = process.stdout.readline().decode("utf-8").split("/") + print(line[0]) + if line[0] in apps: + logging.info(f"Working on app {line[0]}") + while True: + if line[0] != apps[0]: + del apps[0] + progress = progress + step + else: + break + apps.remove(line[0]) + #print(len(apps)) + topic = "sectorq/amd/backups" + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)} + send_mqtt_message(msg) + progress = progress + step + # input(apps) + + + # for a in apps: + # logging.info(f"App {a}") + # topic = "sectorq/amd/backups" + # msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} + # send_mqtt_message(msg) + # logger.debug(cmnd) + # if _FIRST: + # cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" + # else: + # cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" + + # ans = "y" + # logging.info(cmnd) + # logging.info("Sync files1") + # #input("??????") + # if _TEST: + # ans = input("continue?") or "n" + # if ans == "y" and _EXECUTE: + # status, output = subprocess.getstatusoutput(cmnd) + # #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True) + # progress = progress + step + # topic = "sectorq/amd/backups" + # msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} + # send_mqtt_message(msg) + + + cmnd = f"rm -rf {FULL_BACKUP_LATEST}" + + logging.info(cmnd) + logging.info("Removing latest link") + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + if _FIRST: + cmnd = f"cd {BACKUP_ROOT}; ln -s initial latest" + else: + cmnd = f"cd {BACKUP_ROOT}; mv {DATETIME}_running {DATETIME};ln -s {DATETIME} latest" + logging.info("Creating new latest link") + #print(cmnd) + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + + + #Remove old + logging.info("Removing old dirs") + # input("????") + #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR) + cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;" + #print(cmnd) + # input("????") + if _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + now = datetime.datetime.now() + ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + #msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0} + finished.append(b) + msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} + send_mqtt_message(msg) + + logging.info("Getting size of FS") + cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1" + logging.info(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + used_space = (output.split())[0] + now = datetime.datetime.now() + ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S") + logging.info("Size : {}".format(used_space)) + logging.info("Sending finished status") + #msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space} + msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space} + logging.info(msg) - \ No newline at end of file + send_mqtt_message(msg) + os.remove(filename.path) + + + topic = "sectorq/amd/restore" + for s in servers: + #if s != "rack.home.lan": + if s == "m-server.home.lan": + continue + elif s == "nas.home.lan": + user = "admin" + cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all" + else: + user = "root" + cmnd = "sudo /myapps/omv_backup.py -r all" + msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1} + logging.info(msg) + + send_mqtt_message(msg) + if is_port_open(s,22): + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + # Add SSH host key automatically if needed. + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + # Connect to router using username/password authentication. + logger.info(f"Sync {s}") + print(f"Sync {s}") + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa") + ssh.connect(s, + username=user, + look_for_keys=False, + allow_agent=False, + pkey=pkey) + print(cmnd) + ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd) + for line in iter(ssh_stdout.readline, ""): + logger.info(line) + print(line, end="") + for line in iter(ssh_stderr.readline, ""): + logger.info(line) + ssh.close() + # if _MODE == "auto": + # hostup = True + # cmnd = "ssh root@omv.home.lan 'systemctl suspend &'" + # status, output = subprocess.getstatusoutput(cmnd) + # while hostup: + # #HOST_UP = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0 + # cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}" + # status, output = subprocess.getstatusoutput(cmnd) + # # print(status) + # # print(output) + + + # if status == 0: + # print(f"Backup host up, waiting - {n}\r", end="") + # time.sleep(5) + # n += 1 + # else: + # print("Backup host down " ) + # hostup = False + + + + # try: + # url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off" + + # x = requests.post(url) + + # print(x.text) + # except: + # pass +if _SSH_TEST: + user = "root" + cmnd = "ls -la" + topic = "sectorq/amd/backups" + for s in servers: + # if s == "m-server.home.lan": + # continue + # elif s == "nas.home.lan": + # user = "admin" + # cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all" + msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1} + logging.info(msg) + + send_mqtt_message(msg) + if s != "rack.home.lan": + continue + if is_port_open(s,22): + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + # Add SSH host key automatically if needed. + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + # Connect to router using username/password authentication. + logger.info(f"Sync {s}") + print(f"Sync {s}") + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + pkey = paramiko.RSAKey.from_private_key_file("/home/jd/.ssh/id_rsa") + ssh.connect(s, + username=user, + look_for_keys=False, + allow_agent=False, + pkey=pkey) + print(cmnd) + ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd) + for line in iter(ssh_stdout.readline, ""): + logger.info(line) + print(line, end="") + for line in iter(ssh_stderr.readline, ""): + logger.info(line) + ssh.close()