mirror of
https://gitlab.sectorq.eu/jaydee/omv_backup.git
synced 2025-07-01 23:58:33 +02:00
lala
This commit is contained in:
114
omv_backup_v3.py
114
omv_backup_v3.py
@ -15,10 +15,22 @@ import requests
|
||||
import fnmatch
|
||||
import yaml
|
||||
import paramiko
|
||||
file_path = os.path.realpath(__file__)
|
||||
dir_path = os.path.dirname(file_path)
|
||||
|
||||
print(file_path)
|
||||
print(dir_path)
|
||||
os.chdir(dir_path)
|
||||
from wakeonlan import send_magic_packet
|
||||
pid = os.getpid()
|
||||
|
||||
servers = ["rpi5.home.lan","nas.home.lan","rack.home.lan","nas.home.lan"]
|
||||
def is_port_open(host, port):
|
||||
try:
|
||||
sock = socket.create_connection((host, port))
|
||||
sock.close()
|
||||
return True
|
||||
except socket.error:
|
||||
return False
|
||||
servers = ["rpi5.home.lan","nas.home.lan","rack.home.lan","m-server.home.lan"]
|
||||
host = platform.node().lower()
|
||||
#input(host)
|
||||
cmnd = "ps -ef|grep omv_backups.py|grep -v grep |grep -v {}|wc -l".format(pid)
|
||||
@ -53,7 +65,7 @@ mqtt_username = 'jaydee'
|
||||
mqtt_password = 'jaydee1'
|
||||
print("1")
|
||||
try:
|
||||
opts, args = getopt.getopt(sys.argv[1:], "TamftDr:bd:sSO", ["command=", "help", "output="])
|
||||
opts, args = getopt.getopt(sys.argv[1:], "TamftDr:bd:sSOl:", ["command=", "help", "output="])
|
||||
except getopt.GetoptError as err:
|
||||
#usage()
|
||||
sys.exit(2)
|
||||
@ -63,12 +75,14 @@ _MODE = "manual"
|
||||
_FIRST = _TEST = _RESTORE = _BACKUP = _SYNC = _START = _STOP = _SSH_TEST = False
|
||||
_EXECUTE = True
|
||||
_DATE = "pick"
|
||||
|
||||
_LOG_LEVEL = ""
|
||||
for o, a in opts:
|
||||
if o == "-a":
|
||||
_MODE = "auto"
|
||||
elif o in ("-m", "--manual"):
|
||||
_MODE = "manual"
|
||||
elif o in ("-l", "--level"):
|
||||
_LOG_LEVEL = a.upper()
|
||||
elif o in ("-f", "--first"):
|
||||
_FIRST = True
|
||||
elif o in ("-d", "--date"):
|
||||
@ -91,8 +105,8 @@ for o, a in opts:
|
||||
_EXECUTE = False
|
||||
elif o in ("-T", "--dry"):
|
||||
_SSH_TEST = True
|
||||
_LOG_LEVEL = ""
|
||||
LOG_FILE = "/var/log/omv_backup.log"
|
||||
|
||||
LOG_FILE = "omv_backup.log"
|
||||
if _LOG_LEVEL == "DEBUG":
|
||||
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
|
||||
logging.debug('using debug loging')
|
||||
@ -173,8 +187,10 @@ def send_mqtt_message(msg):
|
||||
client.connect(broker,1883,60)
|
||||
client.publish(topic, json.dumps(msg))
|
||||
client.disconnect()
|
||||
except:
|
||||
except ValueError as e:
|
||||
logging.error("Failed to send")
|
||||
print("Failed to send")
|
||||
print(e)
|
||||
|
||||
if _SYNC:
|
||||
containers = ["HomeAssistant","webhub-web-1","heimdall","pihole","mosquitto-mosquitto-1","mailu3-redis-1","mailu3-webmail-1","mailu3-resolver-1","mailu3-antispam-1","mailu3-webdav-1","mailu3-smtp-1","mailu3-oletools-1","mailu3-front-1","mailu3-fetchmail-1","mailu3-imap-1","matter-server","piper-en","openwakeword","whisper-en","auth-worker-1","auth-server-1","auth-authentik_ldap-1","auth-redis-1","auth-postgresql-1","nginx-app-1"]
|
||||
@ -202,6 +218,7 @@ if _STOP:
|
||||
|
||||
if _RESTORE:
|
||||
logging.info("Starting Restore")
|
||||
print("Starting Restore")
|
||||
now = datetime.datetime.now()
|
||||
STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
|
||||
if _APP == "all":
|
||||
@ -445,7 +462,7 @@ if _BACKUP:
|
||||
directory = '/backups/'
|
||||
count = len(fnmatch.filter(os.listdir(directory), '*'))
|
||||
|
||||
logging.info('File Count:', count)
|
||||
logging.info(f'File Count: {count}')
|
||||
if count == 0:
|
||||
time.sleep(10)
|
||||
continue
|
||||
@ -453,6 +470,7 @@ if _BACKUP:
|
||||
finished = []
|
||||
now = datetime.datetime.now()
|
||||
STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
|
||||
topic = "sectorq/amd/backups"
|
||||
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":"","cur_job":"","start_time":STARTTIME,"end_time":"in progress","progress":0,"finished":",".join(finished)}
|
||||
send_mqtt_message(msg)
|
||||
# iterate over files in
|
||||
@ -465,7 +483,7 @@ if _BACKUP:
|
||||
host = filename.name
|
||||
logging.info("Backup")
|
||||
for b in backups[host]["jobs"]:
|
||||
topic = "sectorq/amd/backups"
|
||||
|
||||
if not backups[host]["jobs"][b]["active"]:
|
||||
logging.info("Backup {} is not active!".format(b))
|
||||
msg = {"status":"inactive","bak_name":b,"start_time":"inactive","end_time":"inactive","progress":0}
|
||||
@ -502,31 +520,41 @@ if _BACKUP:
|
||||
logging.info(status)
|
||||
logging.info("Create backup dir")
|
||||
|
||||
cmnd = f"ssh {BACKUP_HOST} 'ls {SOURCE_DIR}'"
|
||||
logger.debug(cmnd)
|
||||
status, output = subprocess.getstatusoutput(cmnd)
|
||||
logger.debug(output)
|
||||
apps = output.splitlines()
|
||||
c = len(apps)
|
||||
print(apps)
|
||||
print(len(apps))
|
||||
|
||||
|
||||
#cmnd = "rsync -av --delete {}/ --link-dest {} --exclude=\".cache\" {}".format(SOURCE_DIR, LATEST_LINK, BACKUP_PATH)
|
||||
step = round(100 / c,1)
|
||||
progress = 0
|
||||
for a in apps:
|
||||
logging.info(f"App {a}")
|
||||
topic = "sectorq/amd/backups"
|
||||
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
|
||||
send_mqtt_message(msg)
|
||||
logger.debug(cmnd)
|
||||
if _FIRST:
|
||||
cmnd = f"rsync -avz --delete {SOURCE_DIR} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" rsync://{BACKUP_HOST}{BACKUP_PATH}"
|
||||
else:
|
||||
cmnd = f"rsync -avz --delete {BACKUP_DIR} --link-dest {FULL_BACKUP_LATEST} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}"
|
||||
cmnd = f"rsync -avz --delete {BACKUP_DIR}{a} --link-dest {FULL_BACKUP_LATEST}/{a} --exclude=\"jellyfin/cache/transcodes\" --exclude=\"gitlab/logs/prometheus\" --exclude=\"home-assistant.log\" --exclude=\"gitlab/logs/*\" --exclude=\"esphome/config/.esphome\" --exclude=\".cache\" --exclude=\".git\" --exclude=\"var_lib_motioneye\" {NEW_BACKUP_DIR}
|
||||
|
||||
ans = "y"
|
||||
logging.info(cmnd)
|
||||
logging.info("Sync files")
|
||||
#input("??????")
|
||||
if _TEST:
|
||||
|
||||
ans = input("continue?") or "n"
|
||||
if ans == "y" and _EXECUTE:
|
||||
|
||||
# rsync --info=progress2 -avz --delete /share/docker_data/ --link-dest /m-server/docker_data/latest --exclude="gitlab/data/" --exclude="esphome/config/.esphome" --exclude="gitlab/logs/prometheus" --exclude=".cache" --exclude=".git" --exclude="var_lib_motioneye" /m-server/m-server/docker_data/newone1
|
||||
|
||||
|
||||
# input("????")
|
||||
|
||||
status, output = subprocess.getstatusoutput(cmnd)
|
||||
#proc = subprocess.Popen(cmnd,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd = "/myapps/",shell=True)
|
||||
|
||||
progress = progress + step
|
||||
topic = "sectorq/amd/backups"
|
||||
msg = {"mode":_MODE, "status":"started","bak_name":"complete","host":host,"cur_job":b,"sub":a,"start_time":STARTTIME,"end_time":"in progress","progress":round(progress),"finished":",".join(finished)}
|
||||
send_mqtt_message(msg)
|
||||
|
||||
|
||||
cmnd = f"rm -rf {FULL_BACKUP_LATEST}"
|
||||
@ -579,20 +607,39 @@ if _BACKUP:
|
||||
send_mqtt_message(msg)
|
||||
os.remove(filename.path)
|
||||
|
||||
user = "root"
|
||||
cmnd = "/myapps/venv/bin/python3 /myapps/omv_backup.py -r all"
|
||||
topic = "sectorq/amd/backups"
|
||||
for s in servers:
|
||||
if s == "m-server.home.lan":
|
||||
continue
|
||||
elif s == "nas.home.lan":
|
||||
user = "admin"
|
||||
cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
|
||||
msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
|
||||
logging.info(msg)
|
||||
|
||||
send_mqtt_message(msg)
|
||||
if is_port_open(s,22):
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.load_system_host_keys()
|
||||
# Add SSH host key automatically if needed.
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
# Connect to router using username/password authentication.
|
||||
logger.info(f"Sync {s}")
|
||||
print(f"Sync {s}")
|
||||
ssh.connect(s,
|
||||
username="jd",
|
||||
username=user,
|
||||
look_for_keys=True,
|
||||
allow_agent=False,
|
||||
key_filename="/root/.ssh/id_rsa")
|
||||
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('touch /tmp/lala')
|
||||
print(cmnd)
|
||||
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd)
|
||||
for line in iter(ssh_stdout.readline, ""):
|
||||
logger.info(line)
|
||||
print(line, end="")
|
||||
for line in iter(ssh_stderr.readline, ""):
|
||||
logger.info(line)
|
||||
ssh.close()
|
||||
# if _MODE == "auto":
|
||||
# hostup = True
|
||||
@ -625,18 +672,37 @@ if _BACKUP:
|
||||
# except:
|
||||
# pass
|
||||
if _SSH_TEST:
|
||||
user = "root"
|
||||
cmnd = "/myapps/venv/bin/python3 /myapps/omv_backup.py -r all"
|
||||
topic = "sectorq/amd/backups"
|
||||
for s in servers:
|
||||
if s == "m-server.home.lan":
|
||||
continue
|
||||
elif s == "nas.home.lan":
|
||||
user = "admin"
|
||||
cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
|
||||
msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
|
||||
logging.info(msg)
|
||||
|
||||
send_mqtt_message(msg)
|
||||
if is_port_open(s,22):
|
||||
ssh = paramiko.SSHClient()
|
||||
ssh.load_system_host_keys()
|
||||
# Add SSH host key automatically if needed.
|
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
# Connect to router using username/password authentication.
|
||||
logger.info(f"Sync {s}")
|
||||
print(f"Sync {s}")
|
||||
ssh.connect(s,
|
||||
username="jd",
|
||||
username=user,
|
||||
look_for_keys=True,
|
||||
allow_agent=False,
|
||||
key_filename="/root/.ssh/id_rsa")
|
||||
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('touch /tmp/lala')
|
||||
print(cmnd)
|
||||
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command(cmnd)
|
||||
for line in iter(ssh_stdout.readline, ""):
|
||||
logger.info(line)
|
||||
print(line, end="")
|
||||
for line in iter(ssh_stderr.readline, ""):
|
||||
logger.info(line)
|
||||
ssh.close()
|
||||
Reference in New Issue
Block a user