jaydee/omv_backup
1
0
Fork 0
mirror of https://gitlab.sectorq.eu/jaydee/omv_backup.git synced 2025-09-13 12:10:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: jaydee:171481767372cd8650ca57a2139c6c366070c890
Branches Tags
jaydee:main
...
compare: jaydee:d3e15e973a55736ec3b76258770cb280a4dcfed8
Branches Tags
jaydee:main

16 Commits
1714817673 ... d3e15e973a

Author SHA1 Message Date
jaydee
d3e15e973a build 2025-08-26 16:00:01 +02:00
jaydee
cee1cf353c build 2025-08-26 15:58:31 +02:00
jaydee
f9e7654c47 build 2025-08-26 15:51:51 +02:00
jaydee
1816bd516c build 2025-08-26 15:43:06 +02:00
jaydee
dad1c49409 build 2025-08-26 15:38:20 +02:00
jaydee
66644b6bb9 build 2025-08-26 15:30:19 +02:00
jaydee
781a74ba20 build 2025-08-26 15:27:23 +02:00
jaydee
29cb028c5c build 2025-08-26 15:24:05 +02:00
jaydee
1667ce0079 build 2025-08-26 15:18:01 +02:00
jaydee
984b3e09a0 build 2025-08-26 15:15:26 +02:00
jaydee
64bea0a833 build 2025-08-26 15:07:51 +02:00
jaydee
dd51f14699 build 2025-08-26 15:04:35 +02:00
jaydee
35aa35a3b5 build 2025-08-26 14:57:00 +02:00
jaydee
4f868047be build 2025-08-26 14:48:39 +02:00
jaydee
ba09aeb502 build 2025-08-26 13:38:19 +02:00
jaydee
8e9e915cdc build 2025-08-26 13:35:58 +02:00
1 changed files with 48 additions and 100 deletions
Expand all files Collapse all files

146
omv_backup.py
View File

@@ -17,11 +17,18 @@ import fnmatch
import yaml
import paramiko
import shutil
import signal
import paho.mqtt.client as mqtt
#import numpy as np
def signal_handler(sig, frame):
print('You pressed Ctrl+C!')
conn.close()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
file_path = os.path.realpath(__file__)
dir_path = os.path.dirname(file_path)
VERSION="1.0.9"
@@ -45,6 +52,7 @@ status, output = subprocess.getstatusoutput(cmnd)
if int(output) > 0:
print("Running already!")
sys.exit()
def is_port_open(host, port):
try:
sock = socket.create_connection((host, port))
@@ -103,8 +111,6 @@ for o, a in opts:
_RESTORE = True
_APP = a
print("RESTORE")
elif o in ("-b", "--backup"):
_BACKUP = True
elif o in ("-D", "--dry"):
_EXECUTE = False
elif o in ("-T", "--dry"):
@@ -198,14 +204,14 @@ logging.info("Test connection")
hm = socket.gethostbyaddr(BACKUP_HOST)
logging.info(_RESTORE)
def send_mqtt_message(topic,msg):
def send_mqtt_message(topic,msg,qos=0,retain=False):
client2 = mqtt.Client()
client2.username_pw_set(USERNAME, PASSWORD)
client2.username_pw_set("jaydee", "jaydee1")
try:
client2.connect(broker,1883,60)
client2.publish(topic, json.dumps(msg))
client2.connect("mqtt.home.lan",1883,60)
client2.publish(topic, json.dumps(msg), qos=qos, retain=retain)
client2.disconnect()
logging.info(f"Message sent {topic}, {msg}")
logging.info(f"Message1 sent {topic}, {msg}")
except ValueError as e:
logging.error("Failed to send")
print("Failed to send")
@@ -235,10 +241,7 @@ if _STOP:
cmnd = f"docker stop {c.split()[-1]}"
status, running_containers = subprocess.getstatusoutput(cmnd)
def restore_job():
logging.info(f'starting restore job')
return 0
logging.info("Starting Restore")
print("Starting Restore")
now = datetime.datetime.now()
STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
if _APP == "all":
@@ -470,26 +473,28 @@ def restore_job():
msg = {"mode":_MODE, "status":"restore","bak_name":"Restore","host":host,"cur_job":app,"start_time":STARTTIME,"end_time":"","progress":100,"finished":ENDJOB,"used_space":1}
logging.info(msg)
send_mqtt_message(topic,msg)
if _MODE == "auto":
cmnd = "ssh root@amd.home.lan 'systemctl suspend &'"
status, output = subprocess.getstatusoutput(cmnd)
def backup_job(server):
client = mqtt.Client()
client.username_pw_set("jaydee", "jaydee1")
client.connect("mqtt.home.lan",1883,60)
logging.info(f'starting backup job')
message = "start"
topic = "sectorq/backups/start"
finished = []
now = datetime.datetime.now()
STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
topic = "sectorq/amd/restore"
msg = {"mode":"restore", "status":"restore","bak_name":"s","host":0,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":0,"used_space":0}
send_mqtt_message(topic,msg)
client.publish(topic, json.dumps(msg),qos=0, retain=True)
#client.publish(topic, msg)
topic = "sectorq/amd/backups"
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
send_mqtt_message(topic,msg)
client.publish(topic, json.dumps(msg),qos=0, retain=True)
# iterate over files in
# that directory
@@ -502,7 +507,7 @@ def backup_job(server):
if not backups[host]["jobs"][b]["active"]:
logging.info("Backup {} is not active!".format(b))
msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0}
send_mqtt_message(topic,msg)
client.publish(topic, json.dumps(msg),qos=0, retain=True)
continue
SOURCE_DIR = backups[host]["jobs"][b]["source"]
@@ -522,9 +527,9 @@ def backup_job(server):
# msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0}
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
client.connect(broker,1883,60)
client.publish(topic, json.dumps(msg),qos=0, retain=True)
client.disconnect()
cmnd = "mkdir -p " + NEW_BACKUP_DIR
@@ -547,15 +552,11 @@ def backup_job(server):
step = round(100 / c,1)
progress = 0
#cmd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST}/ --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
#cmd = [ 'rsync', '-avz','--info=progress2', BACKUP_DIR , NEW_BACKUP_DIR]
#cmd = ['rsync', '-avz', '--delete', BACKUP_DIR, '--link-dest', FULL_BACKUP_LATEST, '--exclude="home-assistant_v2.db"', '--exclude="jellyfin/cache/transcodes"', '--exclude=".@__thumb/"', '--exclude="gitlab/logs/prometheus"', '--exclude="home-assistant.log"', '--exclude="gitlab/logs/*"', '--exclude="esphome/config/.esphome"', '--exclude=".cache"', '--exclude=".git"', '--exclude="var_lib_motioneye"', '--exclude="/.esphome/build"', '--exclude="nextcloud\mariadb\*"', NEW_BACKUP_DIR]
cmd = ['rsync', '-avz', '--delete', BACKUP_DIR, '--link-dest', FULL_BACKUP_LATEST, '--exclude-from=/myapps/exclude.txt', NEW_BACKUP_DIR]
logging.info(" ".join(cmd))
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE)
while process.poll() is None:
line = process.stdout.readline().decode("utf-8").split("/")
print(line[0])
@@ -571,39 +572,12 @@ def backup_job(server):
#print(len(apps))
topic = "sectorq/amd/backups"
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":line[0],"start_time":STARTTIME,"end_time":"in progress","progress":str(round(progress)) + "%","finished":",".join(finished)}
send_mqtt_message(topic,msg)
client.publish(topic, json.dumps(msg),qos=0, retain=False)
progress = progress + step
# input(apps)
# for a in apps:
# logging.info(f"App {a}")
# topic = "sectorq/amd/backups"
# msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
# send_mqtt_message(topic,msg)
# logger.debug(cmnd)
# if _FIRST:
# cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}"
# else:
# cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
# ans = "y"
# logging.info(cmnd)
# logging.info("Sync files1")
# #input("??????")
# if _TEST:
# ans = input("continue?") or "n"
# if ans == "y" and _EXECUTE:
# status, output = subprocess.getstatusoutput(cmnd)
# #proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True)
# progress = progress + step
# topic = "sectorq/amd/backups"
# msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
# send_mqtt_message(topic,msg)
cmnd = f"rm -rf {FULL_BACKUP_LATEST}"
logging.info(cmnd)
#logging.info(cmnd)
logging.info("Removing latest link")
# input("????")
if _EXECUTE:
@@ -618,25 +592,18 @@ def backup_job(server):
if _EXECUTE:
status, output = subprocess.getstatusoutput(cmnd)
#Remove old
logging.info("Removing old dirs")
# input("????")
#cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR)
#cmnd = f"cd {BACKUP_ROOT} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \\;"
cmnd = f"ls {BACKUP_ROOT}"
#print(cmnd)
# input("????")
if _EXECUTE:
status, output = subprocess.getstatusoutput(cmnd)
for f in output.splitlines():
pattern = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2}$" # regex pattern: string starts with 'abc'
logging.info(f"Checking {f}")
# logging.info(f"Checking {f}")
if re.match(pattern, f):
logging.info("Match!")
# logging.info("Match!")
dt = datetime.datetime.strptime(f, "%Y-%m-%d_%H-%M-%S")
epoch_time = int(dt.timestamp())
@@ -651,8 +618,6 @@ def backup_job(server):
print("No match.")
cmnd = f"ls {BACKUP_ROOT}|grep _running"
#print(cmnd)
# input("????")
logging.info(f"removing obsolete dirs")
if _EXECUTE:
status, output = subprocess.getstatusoutput(cmnd)
@@ -666,7 +631,7 @@ def backup_job(server):
#msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0}
finished.append(b)
msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":ENDTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
send_mqtt_message(topic,msg)
client.publish(topic, json.dumps(msg),qos=0, retain=True)
logging.info("Getting size of FS")
cmnd = "df -h /mnt/raid|awk '{ print $3 }'|tail -1"
@@ -681,11 +646,17 @@ def backup_job(server):
msg = {"mode":_MODE, "status":"finished","bak_name":"complete","host":host,"cur_job":b,"start_time":STARTTIME,"end_time":ENDTIME,"progress":0,"finished":",".join(finished),"used_space":used_space}
logging.info(msg)
send_mqtt_message(topic,msg)
client.publish(topic, json.dumps(msg),qos=0, retain=True)
topic = "sectorq/backups/start"
logging.info(f"LALA : {topic}")
client.publish(topic, "finished",qos=0, retain=True)
client.disconnect()
return "finished"
topic = "sectorq/amd/restore"
for s in servers:
logging.info(f"Restoring {s}")
#if s != "rack.home.lan":
if s == "m-server.home.lan":
continue
@@ -696,9 +667,10 @@ def backup_job(server):
user = "jd"
cmnd = "sudo /myapps/omv_backup.py -r all"
msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
logging.info(msg)
#logging.info(msg)
send_mqtt_message(topic,msg)
continue
if is_port_open(s,22):
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
@@ -726,40 +698,13 @@ def backup_job(server):
os.remove("/backups/restore")
except:
pass
# if _MODE == "auto":
# hostup = True
# cmnd = "ssh root@omv.home.lan 'systemctl suspend &'"
# status, output = subprocess.getstatusoutput(cmnd)
# while hostup:
# #HOST_UP = os.system(f"ping -c 1 -w 2 omv.home.lan") == 0
# cmnd = f"ping -c 1 -w 2 {BACKUP_HOST}"
# status, output = subprocess.getstatusoutput(cmnd)
# # print(status)
# # print(output)
return "finished1"
# if status == 0:
# print(f"Backup host up, waiting - {n}\r", end="")
# time.sleep(5)
# n += 1
# else:
# print("Backup host down " )
# hostup = False
# try:
# url = "http://m-server.home.lan:8123/api/webhook/-0eWYFhSTzdusAO8jwQS9t1AT?mode=off"
# x = requests.post(url)
# print(x.text)
# except:
# pass
message= "finished1"
client.publish(topic, message,qos=2,retain=True)
return 0
if _RESTORE:
restore_job()
sys.exit()
if _SSH_TEST:
user = "root"
cmnd = "ls -la"
@@ -771,7 +716,7 @@ if _SSH_TEST:
# user = "admin"
# cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
logging.info(msg)
#logging.info(msg)
send_mqtt_message(topic,msg)
if s != "rack.home.lan":
@@ -806,6 +751,8 @@ def handle_payload(payload):
if payload == 'm-server':
logging.info("💡 Starting backup job")
backup_job(payload)
logging.info(f"💡 Finished backup job")
else:
logging.error(f"⚠️ Unknown command: {payload}")
@@ -836,4 +783,5 @@ if USE_TLS:
# Connect and loop forever
client.connect(BROKER, PORT, keepalive=60)
client.publish("sectorq/backups/start", "finished", qos=0, retain=False)
client.loop_forever()