build

.gitignore

@@ -0,0 +1 @@
+.gitlab-ci.yml

.gitlab-ci.yml

@@ -1,9 +1,30 @@
-stages:
+# This file is a template, and might need editing before it works on your project.
-  - build
+# This is a sample GitLab CI/CD configuration file that should run without any modifications.
+# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts,
+# it uses echo commands to simulate the pipeline execution.
+#
+# A pipeline is composed of independent jobs that run scripts, grouped into stages.
+# Stages run in sequential order, but jobs within stages run in parallel.
+#
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/#stages
+#
+# You can copy and paste this template into a new `.gitlab-ci.yml` file.
+# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
+#
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/development/cicd/templates/
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
 
-build_job:
+stages:          # List of stages for jobs, and their order of execution
+  - build
+  
+  
+build-job:       # This job runs in the build stage, which runs first.
   stage: build
   script:
-    - echo "Running build pipeline"
+    - column=":"
-
+    - echo "${flow_id}"
-
+    - curl -X POST https://kestra.sectorq.eu/api/v1/executions/webhook/jaydee/ansible-all/${flow_id} -d '{"tag":["setup","omv_backup"],"target":["servers"]}' -H "Content-Type${column} application/json"
+  rules:
+    - if: '$CI_COMMIT_MESSAGE =~ /build/'

docker_backups.py

@@ -1,18 +1,52 @@
 import subprocess
 import requests
 import datetime
+import os
+import shutil
 now = datetime.datetime.now()
+PASSWORD = "l4c1j4yd33Du5lo"
 DATETIME = now.strftime("%Y%m%d%H%M%S")
+os.chdir("/share/docker_data/__backups/")
 print("Backup gitlab")
 cmnd = 'docker exec -t gitlab gitlab-backup create SKIP=artifacts,repositories,registry,uploads,builds,pages,lfs,packages,terraform_state'
 status, output = subprocess.getstatusoutput(cmnd)
+allfiles = os.listdir("/share/docker_data/gitlab/data/backups/")
+for f in allfiles:
+    shutil.move(f"/share/docker_data/gitlab/data/backups/{f}", "/share/docker_data/__backups/")
+
+
 print("Backup nextcloud")
 cmnd = 'docker exec --user www-data nextcloud-app-1 php occ maintenance:mode --on'
 status, output = subprocess.getstatusoutput(cmnd)
-cmnd = f"docker exec nextcloud-db-1 sh -c 'exec mysqldump --single-transaction -h localhost -u nextcloud -pl4c1j4yd33Du5lo nextcloud' > /share/docker_data/__backups/nextcloudDB_{DATETIME}.sql"
+cmnd = f"docker exec nextcloud-db-1 sh -c 'exec mysqldump --single-transaction -h localhost -u nextcloud -p{PASSWORD} nextcloud' > /share/docker_data/__backups/nextcloudDB_{DATETIME}.sql"
 status, output = subprocess.getstatusoutput(cmnd)
 cmnd = "docker exec --user www-data nextcloud-app-1 php occ maintenance:mode --off"
 status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} nextcloudDB_{DATETIME}.sql.zip nextcloudDB_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"nextcloudDB_{DATETIME}.sql")
+
+print("Backup Bookstack")
+cmnd = f"docker exec bookstack-db-1 sh -c 'exec mysqldump --single-transaction -h localhost -u bookstack -p{PASSWORD} bookstackapp' > /share/docker_data/__backups/bookstack_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} bookstack_{DATETIME}.sql.zip bookstack_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"bookstack_{DATETIME}.sql")
+
+print("Backup Kestra")
+cmnd = f"docker exec kestra-postgres-1 sh -c 'pg_dump -h localhost -p 5432 -U kestra -d kestra' > /share/docker_data/__backups/kestra_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} kestra_{DATETIME}.sql.zip kestra_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"kestra_{DATETIME}.sql")
+
+print("Backup Authentik")
+cmnd = f"docker exec authentik-postgresql-1 sh -c 'pg_dump -h localhost -p 5432 -U authentik -d authentik' > /share/docker_data/__backups/authentik_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} authentik_{DATETIME}.sql.zip authentik_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"authentik_{DATETIME}.sql")
+
 
 print("Backup portainer")
 headers = {
@@ -21,7 +55,7 @@ headers = {
 }
 
 json_data = {
-    'password': 'l4c1j4yd33Du5lo',
+    'password': PASSWORD,
 }
 
 response = requests.post('https://portainer.sectorq.eu/api/backup', headers=headers, json=json_data, verify=True)
@@ -32,4 +66,8 @@ response = requests.post('https://portainer.sectorq.eu/api/backup', headers=head
 #response = requests.post('https://portainer.sectorq.eu/api/backup', headers=headers, data=data, verify=False)
 
 with open(f'/share/docker_data/__backups/portainer_snapshot_{DATETIME}.tar.gz', 'wb') as f:
-    f.write(response.content)
+    f.write(response.content)
+os.chdir("/share/docker_data/__backups/")
+print("Remove Old Files")
+cmnd = f" find ./ -maxdepth 1 -type f -mtime +15 -exec rm -f {{}} \\;"
+status, output = subprocess.getstatusoutput(cmnd)

omv_backup.py

@@ -17,9 +17,10 @@ import fnmatch
 import yaml
 import paramiko
 import numpy as np
+
 file_path = os.path.realpath(__file__)
 dir_path = os.path.dirname(file_path)
-VERSION="1.0.3"
+VERSION="1.0.7"
 # print(file_path)
 # print(dir_path)
 os.chdir(dir_path)
@@ -216,7 +217,6 @@ if _STOP:
             continue
         cmnd = f"docker stop {c.split()[-1]}"
         status, running_containers = subprocess.getstatusoutput(cmnd)
-        
 if _RESTORE:
     logging.info("Starting Restore")
     print("Starting Restore")
@@ -224,8 +224,8 @@ if _RESTORE:
     STARTTIME = now.strftime("%Y-%m-%d_%H:%M:%S")
     if _APP == "all":
         _DATE = "latest"
-        if host == "rpi5.home.lan":
+        if host == "rpi5.home.lan" or host == "rpi5":
-            _APP = ["nginx","ha","gitea","gitlab","mailu","bitwarden","esphome","grafana","ingluxdb","kestra","matter-server","mosquitto","octoprint","octoprint2","pihole","unify_block","webhub","homepage","watchtower"]
+            _APP = ["__backups", "nginx","ha","gitea","gitlab","mailu","bitwarden","esphome","grafana","ingluxdb","kestra","matter-server","mosquitto","octoprint","octoprint2","pihole","unify_block","webhub","homepage","watchtower"]
         else:
             cmnd = "ssh root@amd.home.lan 'ls /mnt/raid/backup/m-server/docker_data/latest'"
             status, output = subprocess.getstatusoutput(cmnd)
@@ -234,9 +234,6 @@ if _RESTORE:
         #input("????")
     else:
         _APP = _APP.split(",")
- 
- 
-     
     PROGRESS = 0  
     topic = "sectorq/amd/restore"
     step = 100 / len(_APP)
@@ -256,11 +253,11 @@ if _RESTORE:
         if _DATE == "pick":
             cmnd = f"ssh root@amd.home.lan 'ls {BACKUP_DEVICE}/backup/m-server/docker_data'"
             status, output = subprocess.getstatusoutput(cmnd)
-            #print(output)
+            print(output)
             dates = output.splitlines() 
             n = 1
             for i in dates:
-                logging.info(f"{n} - {i}" )
+                print(f"{n} - {i}" )
                 n += 1
                 
             ans = input("Pick a backup to restore : ")    
@@ -369,15 +366,19 @@ if _RESTORE:
             #loaded['logo']['icon'] = "/images/morefine2.png"
             logging.info(json.dumps(loaded, indent=2))
             i = 0
+            
+            
             for y in loaded:
                 logging.info(i)
                 logging.info(y)
                 
                 if "logo" in y:
-                    if host == "rpi5.home.lan":
+                    if host == "rpi5.home.lan" or host == "rpi5":
                         loaded[i]['logo']['icon'] = "/images/rpi5.png"
                     elif host == "nas.home.lan":
                         loaded[i]['logo']['icon'] = "/images/qnap_nas.png"
+                    elif host == "rack.home.lan":
+                        loaded[i]['logo']['icon'] = "/images/rack.png"
                     else:
                         loaded[i]['logo']['icon'] = "/images/morefine2.png"
                 i+=1
@@ -479,6 +480,8 @@ if _BACKUP:
                 if filename.is_file():
                     logging.info(filename.path)
                     logging.info(filename.name)
+                if filename.name == "restore":
+                    break
                 host = filename.name
                 logging.info("Backup")
                 for b in backups[host]["jobs"]:
@@ -644,7 +647,7 @@ if _BACKUP:
                     user = "admin"
                     cmnd = "/share/Data/__GITLAB/omv_backup/venv/bin/python3 /share/Data/__GITLAB/omv_backup/omv_backup.py -r all"
                 else:
-                    user = "root"
+                    user = "jd"
                     cmnd = "sudo /myapps/omv_backup.py -r all"
                 msg = {"mode":_MODE, "status":"restore","bak_name":"s","host":s,"cur_job":"aaa","start_time":1,"end_time":1,"progress":0,"finished":1,"used_space":1}
                 logging.info(msg)
@@ -673,6 +676,10 @@ if _BACKUP:
                     for line in iter(ssh_stderr.readline, ""):
                         logger.info(line)
                     ssh.close()
+            try:
+                os.remove("/backups/restore")
+            except:
+                pass
                 # if _MODE == "auto":
                 #     hostup = True
                 #     cmnd = "ssh root@omv.home.lan 'systemctl suspend &'"