From 5d07cca0e27f332c1bb18169280d8e3991a3397c Mon Sep 17 00:00:00 2001 From: jaydee Date: Thu, 3 Apr 2025 23:30:55 +0200 Subject: [PATCH] lala --- omv_backup_v3.py | 168 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 117 insertions(+), 51 deletions(-) diff --git a/omv_backup_v3.py b/omv_backup_v3.py index 72b09cf..e9f1da8 100644 --- a/omv_backup_v3.py +++ b/omv_backup_v3.py @@ -15,10 +15,22 @@ import requests import fnmatch import yaml import paramiko +file_path = os.path.realpath(__file__) +dir_path = os.path.dirname(file_path) + +print(file_path) +print(dir_path) +os.chdir(dir_path) from wakeonlan import send_magic_packet pid = os.getpid() - -servers = ["rpi5.home.lan","nas.home.lan","rack.home.lan","nas.home.lan"] +def is_port_open(host, port): + try: + sock = socket.create_connection((host, port)) + sock.close() + return True + except socket.error: + return False +servers = ["rpi5.home.lan","nas.home.lan","rack.home.lan","m-server.home.lan"] host = platform.node().lower() #input(host) cmnd = "ps -ef|grep omv_backups.py|grep -v grep |grep -v {}|wc -l".format(pid) @@ -53,7 +65,7 @@ mqtt_username = 'jaydee' mqtt_password = 'jaydee1' print("1") try: - opts, args = getopt.getopt(sys.argv[1:], "TamftDr:bd:sSO", ["command=", "help", "output="]) + opts, args = getopt.getopt(sys.argv[1:], "TamftDr:bd:sSOl:", ["command=", "help", "output="]) except getopt.GetoptError as err: #usage() sys.exit(2) @@ -63,12 +75,14 @@ _MODE = "manual" _FIRST = _TEST = _RESTORE = _BACKUP = _SYNC = _START = _STOP = _SSH_TEST = False _EXECUTE = True _DATE = "pick" - +_LOG_LEVEL = "" for o, a in opts: if o == "-a": _MODE = "auto" elif o in ("-m", "--manual"): _MODE = "manual" + elif o in ("-l", "--level"): + _LOG_LEVEL = a.upper() elif o in ("-f", "--first"): _FIRST = True elif o in ("-d", "--date"): @@ -91,8 +105,8 @@ for o, a in opts: _EXECUTE = False elif o in ("-T", "--dry"): _SSH_TEST = True -_LOG_LEVEL = "" -LOG_FILE = "/var/log/omv_backup.log" + +LOG_FILE = "omv_backup.log" if _LOG_LEVEL == "DEBUG": logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') logging.debug('using debug loging') @@ -173,8 +187,10 @@ def send_mqtt_message(msg): client.connect(broker,1883,60) client.publish(topic, json.dumps(msg)) client.disconnect() - except: + except ValueError as e: logging.error("Failed to send") + print("Failed to send") + print(e) if _SYNC: containers = ["HomeAssistant","webhub-web-1","heimdall","pihole","mosquitto-mosquitto-1","mailu3-redis-1","mailu3-webmail-1","mailu3-resolver-1","mailu3-antispam-1","mailu3-webdav-1","mailu3-smtp-1","mailu3-oletools-1","mailu3-front-1","mailu3-fetchmail-1","mailu3-imap-1","matter-server","piper-en","openwakeword","whisper-en","auth-worker-1","auth-server-1","auth-authentik_ldap-1","auth-redis-1","auth-postgresql-1","nginx-app-1"] @@ -202,6 +218,7 @@ if _STOP: if _RESTORE: logging.info("Starting Restore") + print("Starting Restore") now = datetime.datetime.now() STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") if _APP == "all": @@ -445,7 +462,7 @@ if _BACKUP: directory = '/backups/' count = len(fnmatch.filter(os.listdir(directory), '*')) - logging.info('File Count:', count) + logging.info(f'File Count: {count}') if count == 0: time.sleep(10) continue @@ -453,6 +470,7 @@ if _BACKUP: finished = [] now = datetime.datetime.now() STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") + topic = "sectorq/amd/backups" msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} send_mqtt_message(msg) # iterate over files in @@ -465,7 +483,7 @@ if _BACKUP: host = filename.name logging.info("Backup") for b in backups[host]["jobs"]: - topic = "sectorq/amd/backups" + if not backups[host]["jobs"][b]["active"]: logging.info("Backup {} is not active!".format(b)) msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0} @@ -502,32 +520,42 @@ if _BACKUP: logging.info(status) logging.info("Create backup dir") + cmnd = f"ssh {BACKUP_HOST} 'ls {SOURCE_DIR}'" + logger.debug(cmnd) + status, output = subprocess.getstatusoutput(cmnd) + logger.debug(output) + apps = output.splitlines() + c = len(apps) + print(apps) + print(len(apps)) + + step = round(100 / c,1) + progress = 0 + for a in apps: + logging.info(f"App {a}") + topic = "sectorq/amd/backups" + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} + send_mqtt_message(msg) + logger.debug(cmnd) + if _FIRST: + cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" + else: + cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR} - - #cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH) - if _FIRST: - cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" - else: - cmnd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" - - ans = "y" - logging.info(cmnd) - logging.info("Sync files") - #input("??????") - if _TEST: - - ans = input("continue?") or "n" - if ans == "y" and _EXECUTE: - - # rsync --info=progress2 -avz --delete /share/docker_data/ --link-dest /m-server/docker_data/latest --exclude="gitlab/data/" --exclude="esphome/config/.esphome" --exclude="gitlab/logs/prometheus" --exclude=".cache" --exclude=".git" --exclude="var_lib_motioneye" /m-server/m-server/docker_data/newone1 - - - # input("????") - - status, output = subprocess.getstatusoutput(cmnd) - #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True) - - + ans = "y" + logging.info(cmnd) + logging.info("Sync files") + #input("??????") + if _TEST: + ans = input("continue?") or "n" + if ans == "y" and _EXECUTE: + status, output = subprocess.getstatusoutput(cmnd) + #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True) + progress = progress + step + topic = "sectorq/amd/backups" + msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)} + send_mqtt_message(msg) + cmnd = f"rm -rf {FULL_BACKUP_LATEST}" @@ -578,21 +606,40 @@ if _BACKUP: send_mqtt_message(msg) os.remove(filename.path) + + user = "root" + cmnd = "/myapps/venv/bin/python3 /myapps/omv_backup.py -r all" + topic = "sectorq/amd/backups" + for s in servers: + if s == "m-server.home.lan": + continue + elif s == "nas.home.lan": + user = "admin" + cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all" + msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1} + logging.info(msg) - for s in servers: - if s == "m-server.home.lan": - continue + send_mqtt_message(msg) + if is_port_open(s,22): ssh = paramiko.SSHClient() ssh.load_system_host_keys() # Add SSH host key automatically if needed. ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Connect to router using username/password authentication. + logger.info(f"Sync {s}") + print(f"Sync {s}") ssh.connect(s, - username="jd", + username=user, look_for_keys=True, allow_agent=False, key_filename="/root/.ssh/id_rsa") - ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('touch /tmp/lala') + print(cmnd) + ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd) + for line in iter(ssh_stdout.readline, ""): + logger.info(line) + print(line, end="") + for line in iter(ssh_stderr.readline, ""): + logger.info(line) ssh.close() # if _MODE == "auto": # hostup = True @@ -625,18 +672,37 @@ if _BACKUP: # except: # pass if _SSH_TEST: + user = "root" + cmnd = "/myapps/venv/bin/python3 /myapps/omv_backup.py -r all" + topic = "sectorq/amd/backups" for s in servers: if s == "m-server.home.lan": continue - ssh = paramiko.SSHClient() - ssh.load_system_host_keys() - # Add SSH host key automatically if needed. - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - # Connect to router using username/password authentication. - ssh.connect(s, - username="jd", - look_for_keys=True, - allow_agent=False, - key_filename="/root/.ssh/id_rsa") - ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('touch /tmp/lala') - ssh.close() \ No newline at end of file + elif s == "nas.home.lan": + user = "admin" + cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all" + msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1} + logging.info(msg) + + send_mqtt_message(msg) + if is_port_open(s,22): + ssh = paramiko.SSHClient() + ssh.load_system_host_keys() + # Add SSH host key automatically if needed. + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + # Connect to router using username/password authentication. + logger.info(f"Sync {s}") + print(f"Sync {s}") + ssh.connect(s, + username=user, + look_for_keys=True, + allow_agent=False, + key_filename="/root/.ssh/id_rsa") + print(cmnd) + ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd) + for line in iter(ssh_stdout.readline, ""): + logger.info(line) + print(line, end="") + for line in iter(ssh_stderr.readline, ""): + logger.info(line) + ssh.close() \ No newline at end of file