jaydee/omv_backup
1
0
Fork 0
mirror of https://gitlab.sectorq.eu/jaydee/omv_backup.git synced 2025-10-31 02:21:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

lala

Browse Source
This commit is contained in:
jaydee
2025-04-03 23:30:55 +02:00
parent bbf0e360ef
commit 5d07cca0e2
1 changed files with 117 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
omv_backup_v3.py
View File

@@ -15,10 +15,22 @@ import requests
import fnmatch import fnmatch
import yaml import yaml
import paramiko import paramiko
file_path = os.path.realpath(__file__)
dir_path = os.path.dirname(file_path)
print(file_path)
print(dir_path)
os.chdir(dir_path)
from wakeonlan import send_magic_packet from wakeonlan import send_magic_packet
pid = os.getpid() pid = os.getpid()
def is_port_open(host, port):
servers = ["rpi5.home.lan","nas.home.lan","rack.home.lan","nas.home.lan"] try:
sock = socket.create_connection((host, port))
sock.close()
return True
except socket.error:
return False
servers = ["rpi5.home.lan","nas.home.lan","rack.home.lan","m-server.home.lan"]
host = platform.node().lower() host = platform.node().lower()
#input(host) #input(host)
cmnd = "ps -ef|grep omv_backups.py|grep -v grep |grep -v {}|wc -l".format(pid) cmnd = "ps -ef|grep omv_backups.py|grep -v grep |grep -v {}|wc -l".format(pid)
@@ -53,7 +65,7 @@ mqtt_username = 'jaydee'
mqtt_password = 'jaydee1' mqtt_password = 'jaydee1'
print("1") print("1")
try: try:
opts, args = getopt.getopt(sys.argv[1:], "TamftDr:bd:sSO", ["command=", "help", "output="]) opts, args = getopt.getopt(sys.argv[1:], "TamftDr:bd:sSOl:", ["command=", "help", "output="])
except getopt.GetoptError as err: except getopt.GetoptError as err:
#usage() #usage()
sys.exit(2) sys.exit(2)
@@ -63,12 +75,14 @@ _MODE = "manual"
_FIRST = _TEST = _RESTORE = _BACKUP = _SYNC = _START = _STOP = _SSH_TEST = False _FIRST = _TEST = _RESTORE = _BACKUP = _SYNC = _START = _STOP = _SSH_TEST = False
_EXECUTE = True _EXECUTE = True
_DATE = "pick" _DATE = "pick"
_LOG_LEVEL = ""
for o, a in opts: for o, a in opts:
if o == "-a": if o == "-a":
_MODE = "auto" _MODE = "auto"
elif o in ("-m", "--manual"): elif o in ("-m", "--manual"):
_MODE = "manual" _MODE = "manual"
elif o in ("-l", "--level"):
_LOG_LEVEL = a.upper()
elif o in ("-f", "--first"): elif o in ("-f", "--first"):
_FIRST = True _FIRST = True
elif o in ("-d", "--date"): elif o in ("-d", "--date"):
@@ -91,8 +105,8 @@ for o, a in opts:
_EXECUTE = False _EXECUTE = False
elif o in ("-T", "--dry"): elif o in ("-T", "--dry"):
_SSH_TEST = True _SSH_TEST = True
_LOG_LEVEL = ""
LOG_FILE = "/var/log/omv_backup.log" LOG_FILE = "omv_backup.log"
if _LOG_LEVEL == "DEBUG": if _LOG_LEVEL == "DEBUG":
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.debug('using debug loging') logging.debug('using debug loging')
@@ -173,8 +187,10 @@ def send_mqtt_message(msg):
client.connect(broker,1883,60) client.connect(broker,1883,60)
client.publish(topic, json.dumps(msg)) client.publish(topic, json.dumps(msg))
client.disconnect() client.disconnect()
except: except ValueError as e:
logging.error("Failed to send") logging.error("Failed to send")
print("Failed to send")
print(e)
if _SYNC: if _SYNC:
containers = ["HomeAssistant","webhub-web-1","heimdall","pihole","mosquitto-mosquitto-1","mailu3-redis-1","mailu3-webmail-1","mailu3-resolver-1","mailu3-antispam-1","mailu3-webdav-1","mailu3-smtp-1","mailu3-oletools-1","mailu3-front-1","mailu3-fetchmail-1","mailu3-imap-1","matter-server","piper-en","openwakeword","whisper-en","auth-worker-1","auth-server-1","auth-authentik_ldap-1","auth-redis-1","auth-postgresql-1","nginx-app-1"] containers = ["HomeAssistant","webhub-web-1","heimdall","pihole","mosquitto-mosquitto-1","mailu3-redis-1","mailu3-webmail-1","mailu3-resolver-1","mailu3-antispam-1","mailu3-webdav-1","mailu3-smtp-1","mailu3-oletools-1","mailu3-front-1","mailu3-fetchmail-1","mailu3-imap-1","matter-server","piper-en","openwakeword","whisper-en","auth-worker-1","auth-server-1","auth-authentik_ldap-1","auth-redis-1","auth-postgresql-1","nginx-app-1"]
@@ -202,6 +218,7 @@ if _STOP:
if _RESTORE: if _RESTORE:
logging.info("Starting Restore") logging.info("Starting Restore")
print("Starting Restore")
now = datetime.datetime.now() now = datetime.datetime.now()
STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
if _APP == "all": if _APP == "all":
@@ -445,7 +462,7 @@ if _BACKUP:
directory = '/backups/' directory = '/backups/'
count = len(fnmatch.filter(os.listdir(directory), '*')) count = len(fnmatch.filter(os.listdir(directory), '*'))
logging.info('File Count:', count) logging.info(f'File Count: {count}')
if count == 0: if count == 0:
time.sleep(10) time.sleep(10)
continue continue
@@ -453,6 +470,7 @@ if _BACKUP:
finished = [] finished = []
now = datetime.datetime.now() now = datetime.datetime.now()
STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S") STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
topic = "sectorq/amd/backups"
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)} msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
send_mqtt_message(msg) send_mqtt_message(msg)
# iterate over files in # iterate over files in
@@ -465,7 +483,7 @@ if _BACKUP:
host = filename.name host = filename.name
logging.info("Backup") logging.info("Backup")
for b in backups[host]["jobs"]: for b in backups[host]["jobs"]:
topic = "sectorq/amd/backups"
if not backups[host]["jobs"][b]["active"]: if not backups[host]["jobs"][b]["active"]:
logging.info("Backup {} is not active!".format(b)) logging.info("Backup {} is not active!".format(b))
msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0} msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0}
@@ -502,31 +520,41 @@ if _BACKUP:
logging.info(status) logging.info(status)
logging.info("Create backup dir") logging.info("Create backup dir")
cmnd = f"ssh {BACKUP_HOST} 'ls {SOURCE_DIR}'"
logger.debug(cmnd)
status, output = subprocess.getstatusoutput(cmnd)
logger.debug(output)
apps = output.splitlines()
c = len(apps)
print(apps)
print(len(apps))
step = round(100 / c,1)
#cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH) progress = 0
for a in apps:
logging.info(f"App {a}")
topic = "sectorq/amd/backups"
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
send_mqtt_message(msg)
logger.debug(cmnd)
if _FIRST: if _FIRST:
cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}" cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}"
else: else:
cmnd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}" cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}
ans = "y" ans = "y"
logging.info(cmnd) logging.info(cmnd)
logging.info("Sync files") logging.info("Sync files")
#input("??????") #input("??????")
if _TEST: if _TEST:
ans = input("continue?") or "n" ans = input("continue?") or "n"
if ans == "y" and _EXECUTE: if ans == "y" and _EXECUTE:
# rsync --info=progress2 -avz --delete /share/docker_data/ --link-dest /m-server/docker_data/latest --exclude="gitlab/data/" --exclude="esphome/config/.esphome" --exclude="gitlab/logs/prometheus" --exclude=".cache" --exclude=".git" --exclude="var_lib_motioneye" /m-server/m-server/docker_data/newone1
# input("????")
status, output = subprocess.getstatusoutput(cmnd) status, output = subprocess.getstatusoutput(cmnd)
#proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True) #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True)
progress = progress + step
topic = "sectorq/amd/backups"
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
send_mqtt_message(msg)
cmnd = f"rm -rf {FULL_BACKUP_LATEST}" cmnd = f"rm -rf {FULL_BACKUP_LATEST}"
@@ -579,20 +607,39 @@ if _BACKUP:
send_mqtt_message(msg) send_mqtt_message(msg)
os.remove(filename.path) os.remove(filename.path)
user = "root"
cmnd = "/myapps/venv/bin/python3 /myapps/omv_backup.py -r all"
topic = "sectorq/amd/backups"
for s in servers: for s in servers:
if s == "m-server.home.lan": if s == "m-server.home.lan":
continue continue
elif s == "nas.home.lan":
user = "admin"
cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
logging.info(msg)
send_mqtt_message(msg)
if is_port_open(s,22):
ssh = paramiko.SSHClient() ssh = paramiko.SSHClient()
ssh.load_system_host_keys() ssh.load_system_host_keys()
# Add SSH host key automatically if needed. # Add SSH host key automatically if needed.
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connect to router using username/password authentication. # Connect to router using username/password authentication.
logger.info(f"Sync {s}")
print(f"Sync {s}")
ssh.connect(s, ssh.connect(s,
username="jd", username=user,
look_for_keys=True, look_for_keys=True,
allow_agent=False, allow_agent=False,
key_filename="/root/.ssh/id_rsa") key_filename="/root/.ssh/id_rsa")
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('touch /tmp/lala') print(cmnd)
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd)
for line in iter(ssh_stdout.readline, ""):
logger.info(line)
print(line, end="")
for line in iter(ssh_stderr.readline, ""):
logger.info(line)
ssh.close() ssh.close()
# if _MODE == "auto": # if _MODE == "auto":
# hostup = True # hostup = True
@@ -625,18 +672,37 @@ if _BACKUP:
# except: # except:
# pass # pass
if _SSH_TEST: if _SSH_TEST:
user = "root"
cmnd = "/myapps/venv/bin/python3 /myapps/omv_backup.py -r all"
topic = "sectorq/amd/backups"
for s in servers: for s in servers:
if s == "m-server.home.lan": if s == "m-server.home.lan":
continue continue
elif s == "nas.home.lan":
user = "admin"
cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
logging.info(msg)
send_mqtt_message(msg)
if is_port_open(s,22):
ssh = paramiko.SSHClient() ssh = paramiko.SSHClient()
ssh.load_system_host_keys() ssh.load_system_host_keys()
# Add SSH host key automatically if needed. # Add SSH host key automatically if needed.
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Connect to router using username/password authentication. # Connect to router using username/password authentication.
logger.info(f"Sync {s}")
print(f"Sync {s}")
ssh.connect(s, ssh.connect(s,
username="jd", username=user,
look_for_keys=True, look_for_keys=True,
allow_agent=False, allow_agent=False,
key_filename="/root/.ssh/id_rsa") key_filename="/root/.ssh/id_rsa")
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('touch /tmp/lala') print(cmnd)
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd)
for line in iter(ssh_stdout.readline, ""):
logger.info(line)
print(line, end="")
for line in iter(ssh_stderr.readline, ""):
logger.info(line)
ssh.close() ssh.close()