build

.gitignore

@@ -0,0 +1 @@
+.gitlab-ci.yml

.gitlab-ci.yml

@@ -1,12 +1,30 @@
-stages:
+# This file is a template, and might need editing before it works on your project.
+# This is a sample GitLab CI/CD configuration file that should run without any modifications.
+# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts,
+# it uses echo commands to simulate the pipeline execution.
+#
+# A pipeline is composed of independent jobs that run scripts, grouped into stages.
+# Stages run in sequential order, but jobs within stages run in parallel.
+#
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/#stages
+#
+# You can copy and paste this template into a new `.gitlab-ci.yml` file.
+# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
+#
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/development/cicd/templates/
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
+
+stages:          # List of stages for jobs, and their order of execution
   - build
-
-build_job:
-
+  
+  
+build-job:       # This job runs in the build stage, which runs first.
   stage: build
-  rules:
-    - if: '$CI_COMMIT_TITLE == "build"'
   script:
-    - curl -X GET https://kestra.sectorq.eu/api/v1/executions/webhook/jaydee/omv_distribute/f851511c32ca9450
-
-
+    - column=":"
+    - echo "${flow_id}"
+    - curl -X POST https://kestra.sectorq.eu/api/v1/executions/webhook/jaydee/ansible-all/${flow_id} -d '{"tag":["setup","omv_backup"],"target":["servers"]}' -H "Content-Type${column} application/json"
+  rules:
+    - if: '$CI_COMMIT_MESSAGE =~ /build/'

docker_backups.py

@@ -1,18 +1,52 @@
 import subprocess
 import requests
 import datetime
+import os
+import shutil
 now = datetime.datetime.now()
+PASSWORD = "l4c1j4yd33Du5lo"
 DATETIME = now.strftime("%Y%m%d%H%M%S")
+os.chdir("/share/docker_data/__backups/")
 print("Backup gitlab")
 cmnd = 'docker exec -t gitlab gitlab-backup create SKIP=artifacts,repositories,registry,uploads,builds,pages,lfs,packages,terraform_state'
 status, output = subprocess.getstatusoutput(cmnd)
+allfiles = os.listdir("/share/docker_data/gitlab/data/backups/")
+for f in allfiles:
+    shutil.move(f"/share/docker_data/gitlab/data/backups/{f}", "/share/docker_data/__backups/")
+
+
 print("Backup nextcloud")
 cmnd = 'docker exec --user www-data nextcloud-app-1 php occ maintenance:mode --on'
 status, output = subprocess.getstatusoutput(cmnd)
-cmnd = f"docker exec nextcloud-db-1 sh -c 'exec mysqldump --single-transaction -h localhost -u nextcloud -pl4c1j4yd33Du5lo nextcloud' > /share/docker_data/__backups/nextcloudDB_{DATETIME}.sql"
+cmnd = f"docker exec nextcloud-db-1 sh -c 'exec mysqldump --single-transaction -h localhost -u nextcloud -p{PASSWORD} nextcloud' > /share/docker_data/__backups/nextcloudDB_{DATETIME}.sql"
 status, output = subprocess.getstatusoutput(cmnd)
 cmnd = "docker exec --user www-data nextcloud-app-1 php occ maintenance:mode --off"
 status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} nextcloudDB_{DATETIME}.sql.zip nextcloudDB_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"nextcloudDB_{DATETIME}.sql")
+
+print("Backup Bookstack")
+cmnd = f"docker exec bookstack-db-1 sh -c 'exec mysqldump --single-transaction -h localhost -u bookstack -p{PASSWORD} bookstackapp' > /share/docker_data/__backups/bookstack_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} bookstack_{DATETIME}.sql.zip bookstack_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"bookstack_{DATETIME}.sql")
+
+print("Backup Kestra")
+cmnd = f"docker exec kestra-postgres-1 sh -c 'pg_dump -h localhost -p 5432 -U kestra -d kestra' > /share/docker_data/__backups/kestra_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} kestra_{DATETIME}.sql.zip kestra_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"kestra_{DATETIME}.sql")
+
+print("Backup Authentik")
+cmnd = f"docker exec authentik-postgresql-1 sh -c 'pg_dump -h localhost -p 5432 -U authentik -d authentik' > /share/docker_data/__backups/authentik_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+cmnd = f"zip -P {PASSWORD} authentik_{DATETIME}.sql.zip authentik_{DATETIME}.sql"
+status, output = subprocess.getstatusoutput(cmnd)
+os.remove(f"authentik_{DATETIME}.sql")
+
 
 print("Backup portainer")
 headers = {
@@ -21,7 +55,7 @@ headers = {
 }
 
 json_data = {
-    'password': 'l4c1j4yd33Du5lo',
+    'password': PASSWORD,
 }
 
 response = requests.post('https://portainer.sectorq.eu/api/backup', headers=headers, json=json_data, verify=True)
@@ -32,4 +66,8 @@ response = requests.post('https://portainer.sectorq.eu/api/backup', headers=head
 #response = requests.post('https://portainer.sectorq.eu/api/backup', headers=headers, data=data, verify=False)
 
 with open(f'/share/docker_data/__backups/portainer_snapshot_{DATETIME}.tar.gz', 'wb') as f:
-    f.write(response.content)
+    f.write(response.content)
+os.chdir("/share/docker_data/__backups/")
+print("Remove Old Files")
+cmnd = f" find ./ -maxdepth 1 -type f -mtime +15 -exec rm -f {{}} \\;"
+status, output = subprocess.getstatusoutput(cmnd)

omv_backup.py

@@ -17,9 +17,10 @@ import fnmatch
 import yaml
 import paramiko
 import numpy as np
+
 file_path = os.path.realpath(__file__)
 dir_path = os.path.dirname(file_path)
-VERSION="1.0.4"
+VERSION="1.0.7"
 # print(file_path)
 # print(dir_path)
 os.chdir(dir_path)
@@ -216,7 +217,6 @@ if _STOP:
             continue
         cmnd = f"docker stop {c.split()[-1]}"
         status, running_containers = subprocess.getstatusoutput(cmnd)
-        
 if _RESTORE:
     logging.info("Starting Restore")
     print("Starting Restore")
@@ -225,7 +225,7 @@ if _RESTORE:
     if _APP == "all":
         _DATE = "latest"
         if host == "rpi5.home.lan" or host == "rpi5":
-            _APP = ["nginx","ha","gitea","gitlab","mailu","bitwarden","esphome","grafana","ingluxdb","kestra","matter-server","mosquitto","octoprint","octoprint2","pihole","unify_block","webhub","homepage","watchtower"]
+            _APP = ["__backups", "nginx","ha","gitea","gitlab","mailu","bitwarden","esphome","grafana","ingluxdb","kestra","matter-server","mosquitto","octoprint","octoprint2","pihole","unify_block","webhub","homepage","watchtower"]
         else:
             cmnd = "ssh root@amd.home.lan 'ls /mnt/raid/backup/m-server/docker_data/latest'"
             status, output = subprocess.getstatusoutput(cmnd)
@@ -234,9 +234,6 @@ if _RESTORE:
         #input("????")
     else:
         _APP = _APP.split(",")
- 
- 
-     
     PROGRESS = 0  
     topic = "sectorq/amd/restore"
     step = 100 / len(_APP)
@@ -256,11 +253,11 @@ if _RESTORE:
         if _DATE == "pick":
             cmnd = f"ssh root@amd.home.lan 'ls {BACKUP_DEVICE}/backup/m-server/docker_data'"
             status, output = subprocess.getstatusoutput(cmnd)
-            #print(output)
+            print(output)
             dates = output.splitlines() 
             n = 1
             for i in dates:
-                logging.info(f"{n} - {i}" )
+                print(f"{n} - {i}" )
                 n += 1
                 
             ans = input("Pick a backup to restore : ")