python/omv_backups_v3.py

import datetime
import logging
from paho.mqtt import client as mqtt_client
import getopt
import json
import time
import subprocess
import sys
import os
import platform
pid = os.getpid()

host = platform.node().lower()
cmnd = "ps -ef|grep omv_backups.py|grep -v grep |grep -v {}|wc -l".format(pid)
status, output = subprocess.getstatusoutput(cmnd)

print(output)
if int(output) > 0:
    print("Running already!")
    sys.exit()
broker = 'mqtt.home.lan'
port = 1883
topic_sum = "sectorq/omv/backups"
mqtt_username = 'jaydee'
mqtt_password = 'jaydee1'

try:
    opts, args = getopt.getopt(sys.argv[1:], "am", ["command=", "help", "output="])
except getopt.GetoptError as err:
    #usage()
    sys.exit(2)
output = None
# QJ : getopts
_MODE = "manual"
for o, a in opts:
    if o == "-a":
        _MODE = "auto"
    elif o in ("-m", "--manual"):
       _MODE = "manual"


client = mqtt_client.Client()
client.username_pw_set(mqtt_username, mqtt_password)
client.connect(broker,1883,60)
now = datetime.datetime.now()
STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
msg = {"mode":_MODE, "status":"started","bak_name":"complete","start_time":STARTTIME,"end_time":"in progress","progress":0}
client.publish(topic_sum, json.dumps(msg));
client.disconnect()

backups = {
    "nas": {
        "github":
            {"source":"/share/Data/__GITHUB",
            "exclude":"",
            "active": True
            },
        "photo": {
            "source":"/share/Photo/Years",
            "exclude":"",
            "active":True
        },
    },
    "m-server":{
        "docker_data":{
            "source":"/share/docker_data/",
            "exclude":"",
            "active":True
        },
        "fail2ban":{
            "source":"/etc/fail2ban/",
            "exclude":"",
            "active":True
        }
    }
}
BACKUP_FS = "/srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8"
if not host in backups:
    print(f"No backup jobs for {host}")
    sys.exit()
for b in backups[host]:
    topic = "sectorq/omv/backups/{}".format(b.lower())
    if not backups[host][b]["active"]:
        print("Backup {} is not active!".format(b))
        client.connect(broker,1883,60)
        msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0}

        client.publish(topic, json.dumps(msg))
        client.disconnect()
        continue

    SOURCE_DIR = backups[host][b]["source"]
    now = datetime.datetime.now()
    BACKUP_HOST = "root@omv.home.lan"
    BACKUP_DEVICE = "/srv/dev-disk-by-uuid-2f843500-95b6-43b0-bea1-9b67032989b8"
    BACKUP_DIR = f"/backup/{host}/{b}"
    NEW_BACKUP_DIR = f"{BACKUP_DEVICE}/backup/{host}/{b}"
    DATETIME = now.strftime("%Y-%m-%d_%H-%M-%S")
    BACKUP_PATH="{}/{}".format(BACKUP_DIR, DATETIME)
    LATEST_LINK="{}/latest".format(BACKUP_DIR)
    FULL_BACKUP_LATEST = f"{NEW_BACKUP_DIR}/latest"
    LATEST_LINK = f"/{host}/{b}/latest"


    msg = {"status":"started","bak_name":b,"start_time":DATETIME,"end_time":"in progress", "progress":0}
    client.connect(broker,1883,60)
    client.publish(topic, json.dumps(msg));
    client.disconnect()

    cmnd = "ssh root@omv.home.lan 'mkdir -p " + NEW_BACKUP_DIR + "'"


    status, output = subprocess.getstatusoutput(cmnd)
    print(cmnd)


    #cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH)


    cmnd = f"rsync -avz --delete {SOURCE_DIR} --link-dest {LATEST_LINK} --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}"

    # rsync --info=progress2 -avz --delete /share/docker_data/ --link-dest /m-server/docker_data/latest --exclude="gitlab/data/" --exclude="esphome/config/.esphome" --exclude="gitlab/logs/prometheus" --exclude=".cache" --exclude=".git" --exclude="var_lib_motioneye" /m-server/m-server/docker_data/newone1


    print(cmnd)
#    input("????")
    status, output = subprocess.getstatusoutput(cmnd)
    cmnd = f"ssh root@omv.home.lan 'rm -rf {FULL_BACKUP_LATEST}'"
    print(cmnd)
#    input("????")
    status, output = subprocess.getstatusoutput(cmnd)

    cmnd = f"ssh root@omv.home.lan 'cd {NEW_BACKUP_DIR}; ln -s {DATETIME} latest'"
    print(cmnd)
#    input("????")
    status, output = subprocess.getstatusoutput(cmnd)


    #Remove old
    print("Removing old dirs")
#    input("????")
    #cmnd = "find {} -maxdepth 1 -type d -mtime +30 -exec rm -rf {{}} \;".format(BACKUP_DIR)
    cmnd = f"cd {NEW_BACKUP_DIR} find ./ -maxdepth 1 -type d -mmin +30 -exec rm -rf {{}} \;"
    print(cmnd)
#   input("????")
    #status, output = subprocess.getstatusoutput(cmnd)
    now = datetime.datetime.now()
    ENDTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
    msg = {"status":"finished","bak_name":b,"start_time":DATETIME,"end_time":ENDTIME,"progress":0}
    client.connect(broker,1883,10)
    client.publish(topic, json.dumps(msg))
    client.disconnect()

print("Getting size of FS")
#cmnd = "du -h --max-depth=0 {}".format(BACKUP_FS)
cmnd = "df -h /srv/dev-disk-by-uuid-02fbe97a-cd9a-4511-8bd5-21f8516353ee |awk '{ print $3 }'|tail -1"
status, output = subprocess.getstatusoutput(cmnd)
used_space = (output.split())[0]
now = datetime.datetime.now()
ENDJOB = now.strftime("%Y-%m-%d_%H:%M:%S")
print("Size : {}".format(used_space))
print("Sending finished status")
msg = {"mode":_MODE,"status":"finished","bak_name":"complete","start_time":STARTTIME,"end_time":ENDJOB,"progress":0,"used_space":used_space}
print(msg)
client.connect(broker,1883,10)
client.publish(topic_sum, json.dumps(msg))
client.disconnect()

if _MODE == "auto":
    cmnd = "systemctl suspend"
    status, output = subprocess.getstatusoutput(cmnd)