jaydee/auto_import
1
0
Fork 0
mirror of https://gitlab.sectorq.eu/jaydee/auto_import.git synced 2025-12-14 02:24:53 +01:00
Code Issues Packages Projects Releases Wiki Activity

build

Browse Source
This commit is contained in:
Ladislav Dusa
2025-08-03 02:45:00 +02:00
parent 35a0a46d6f
commit 7db491464a
2 changed files with 228 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

343
auto_import.py
View File

@@ -1,3 +1,5 @@
#!/myapps/venv/bin/python3
# Auto import script for media devices
import os import os
from posixpath import ismount from posixpath import ismount
import sys import sys
@@ -6,133 +8,244 @@ import subprocess
import datetime import datetime
import shutil import shutil
import json import json
import logging
from random import randrange
from paho.mqtt import client as mqtt_client
_DEST = "/media/nas/nas-photo/imported"
class mqtt_sender():
def __init__(self, broker, port, name, password):
self.broker = broker
self.name = name
self.password = password
self.port = port
self.client_id = "dasdasdasd333"
try:
self.client = mqtt_client.Client(mqtt_client.CallbackAPIVersion.VERSION1, self.client_id)
except:
self.client = mqtt_client.Client()
self.client.username_pw_set(self.name, self.password)
def send_mqtt_message(_self, topic, msg):
try:
_self.client.connect(_self.broker,_self.port,60)
_self.client.publish(topic, json.dumps(msg))
_self.client.disconnect()
print("Message send")
except ValueError as e:
print("Failed to send")
print(e)
mqtt_s = mqtt_sender("mqtt.home.lan", 1883, "jaydee", "jaydee1")
mqtt_s.send_mqtt_message("sectorq/auto_import/job", {"status": "started","file":"","device":""})
# Log file location and log level
LOG_FILE = "/tmp/auto_import.log"
_LOG_LEVEL = "DEBUG"
# Configure logging based on log level
if _LOG_LEVEL == "DEBUG":
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.debug('using debug loging')
elif _LOG_LEVEL == "ERROR":
logging.basicConfig(filename=LOG_FILE, level=logging.ERROR, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.info('using error loging')
elif _LOG_LEVEL == "SCAN":
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.info('using error loging')
else:
logging.basicConfig(filename=LOG_FILE, level=logging.INFO, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.info("script started")
# Get current process id and check for duplicate running processes
pid = os.getpid()
cmnd = "ps -ef|grep test.py|grep -v grep"
status, output = subprocess.getstatusoutput(cmnd)
logging.info(output)
logging.info(pid)
pid = os.getpid()
cmnd = "ps -ef|grep test.py|grep -v grep |grep -v {}|wc -l".format(pid)
status, output = subprocess.getstatusoutput(cmnd)
if int(output) > 0:
logging.info("Running already!")
sys.exit()
# Set destination and initialize variables
_DEST = "/share/Photo/imported"
_DEV = "" _DEV = ""
drives = ["6362-3938"] files_imported = 0
myCmd = "ls /dev/disk/by-uuid"
status, output = subprocess.getstatusoutput(myCmd)
input(output)
for l in output.splitlines():
if l in drives:
print(l)
input("????")
_DEV_MOUNT = f"/mnt/{l}"
_DEV_DIR = f"{_DEV_MOUNT}/DCIM"
_IMP_FILE = f"{_DEV_MOUNT}/auto_import"
_DEV_DATA = f"{_DEV_MOUNT}/MISC/version.txt"
print("###############")
#sys.exit()
while True: while True:
if os.path.ismount(_DEV_MOUNT): myCmd = "mount| grep /share/external/DEV3301| grep -v grep | awk '{ print $3 }'"
if os.path.exists(_DEV_DATA): status, output = subprocess.getstatusoutput(myCmd)
file = open(_DEV_DATA, 'r') print(output)
content = file.read() print(status)
file.close() devs = output.splitlines()
strToReplace = ','
replacementStr = '' if len(devs) == 0:
strToReplaceReversed = strToReplace[::-1] time.sleep(10)
replacementStrReversed = replacementStr[::-1] else:
for _DEV_MOUNT in devs:
print("Device mounted")
myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=101"
status, output = subprocess.getstatusoutput(myCmd)
strValue = content[::-1].replace(strToReplaceReversed, replacementStrReversed, 1)[::-1] # _DEV_MOUNT = f"/share/external/DEV3301_1"
c = content.replace("\n","") _DEV_DIR = f"{_DEV_MOUNT}/DCIM"
_IMP_FILE = f"{_DEV_MOUNT}/auto_import"
print("--------------") _DEV_DATA = f"{_DEV_MOUNT}/MISC/version.txt"
print(c[-2])
print("--------------") # If version.txt exists, parse device info
print(strValue) if os.path.exists(_DEV_DATA):
_DEV_CONT = json.loads(strValue) file = open(_DEV_DATA, 'r')
print(_DEV_CONT)
print(_DEV_CONT["wifi mac"])
wifimac = _DEV_CONT["wifi mac"]
if wifimac == "2474f742c017":
_DEV = "gopro9_1"
print("gopro9_1 registered")
elif wifimac == "2474f7421111":
_DEV = "dji_mini_4"
print("dji_mini_4 registered")
if os.path.exists(_IMP_FILE) and _DEV == "":
print("path exist")
time.sleep(5)
if _DEV == "":
file = open(_IMP_FILE, 'r')
content = file.read() content = file.read()
_DEV = json.loads(content)["device"] file.close()
file.close() strToReplace = ','
x = datetime.datetime.now() replacementStr = ''
print(x.year) strToReplaceReversed = strToReplace[::-1]
_DAT = x.strftime("%Y%m%d") replacementStrReversed = replacementStr[::-1]
if len(os.listdir(_DEV_DIR)) == 0:
print("Folder is empty!") strValue = content[::-1].replace(strToReplaceReversed, replacementStrReversed, 1)[::-1]
myCmd = "umount /share/external/DEV3304_1" c = content.replace("\n","")
status, output = subprocess.getstatusoutput(myCmd)
logging.debug("--------------")
logging.debug(c[-2])
logging.debug("--------------")
logging.debug(strValue)
_DEV_CONT = json.loads(strValue)
logging.debug(_DEV_CONT)
logging.debug(_DEV_CONT["wifi mac"])
wifimac = _DEV_CONT["wifi mac"]
# Identify device by wifi mac
if wifimac == "2474f742c017":
_DEV = "gopro9_1"
logging.debug("gopro9_1 registered")
elif wifimac == "2474f742c017":
_DEV = "gopro9_1"
logging.debug("gopro9_1 registered")
elif wifimac == "XXXDJIMINI4":
_DEV = "dji_mini_4"
logging.debug("dji_mini_4 registered")
# If auto_import file exists and device is not set, read device info
if os.path.exists(_IMP_FILE) and _DEV == "":
logging.debug("path exist")
time.sleep(5) time.sleep(5)
continue if _DEV == "":
# myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=101;sleep 1;/sbin/hal_app --se_buzzer enc_id=0,mode=101;sleep 1;/sbin/hal_app --se_buzzer enc_id=0,mode=101" try:
# status, output = subprocess.getstatusoutput(myCmd) file = open(_IMP_FILE, 'r')
if not _DEV == "": except:
if os.path.exists(_DEV_DIR): break
for file_or_dir in os.listdir(_DEV_DIR): content = file.read()
FULL_PATH = os.path.join(_DEV_DIR, file_or_dir) _DEV = json.loads(content)["device"]
print(f"Full path : {FULL_PATH}")
if os.path.isdir(FULL_PATH): logging.debug(f"{_DEV} registered")
print(f"Folder: {file_or_dir}") file.close()
for filename in os.listdir(FULL_PATH): x = datetime.datetime.now()
print(filename) logging.debug(x.year)
c_time = os.path.getctime("{}/{}".format(FULL_PATH,filename)) _DAT = x.strftime("%Y%m%d")
dt_c = datetime.datetime.fromtimestamp(c_time) # If DCIM folder is empty, unmount and continue
print(dt_c) if len(os.listdir(_DEV_DIR)) == 0:
_CDAT = dt_c.strftime("%Y-%m-%d") logging.debug("Folder is empty!")
print(_CDAT) myCmd = f"umount {_DEV_DIR}"
_FDEST = "{}/{}/{}".format(_DEST,_DEV, _CDAT) status, output = subprocess.getstatusoutput(myCmd)
if not os.path.exists(_FDEST): time.sleep(5)
cmnd = "mkdir -p {}".format(_FDEST) continue
status, output = subprocess.getstatusoutput(cmnd) # myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=101;sleep 1;/sbin/hal_app --se_buzzer enc_id=0,mode=101;sleep 1;/sbin/hal_app --se_buzzer enc_id=0,mode=101"
print("Moving file {} into {}".format(filename, _FDEST)) # status, output = subprocess.getstatusoutput(myCmd)
# If device is set and DCIM exists, move files by date
moved_fine = True
if not _DEV == "":
if os.path.exists(_DEV_DIR):
for file_or_dir in os.listdir(_DEV_DIR):
FULL_PATH = os.path.join(_DEV_DIR, file_or_dir)
logging.debug(f"Full path : {FULL_PATH}")
if os.path.isdir(FULL_PATH):
logging.debug(f"Folder: {file_or_dir}")
for filename in os.listdir(FULL_PATH):
logging.debug(filename)
c_time = os.path.getctime("{}/{}".format(FULL_PATH,filename))
dt_c = datetime.datetime.fromtimestamp(c_time)
logging.debug(dt_c)
_CDAT = dt_c.strftime("%Y-%m-%d")
logging.debug(_CDAT)
_FDEST = "{}/{}/{}".format(_DEST,_DEV, _CDAT)
if not os.path.exists(_FDEST):
cmnd = "mkdir -p {}".format(_FDEST)
status, output = subprocess.getstatusoutput(cmnd)
logging.debug("Moving file {} into {}".format(filename, _FDEST))
mqtt_s.send_mqtt_message("sectorq/auto_import/job", {"status": "started","file":filename,"device":_DEV})
try:
#shutil.copy("{}/{}".format(FULL_PATH,filename),_FDEST)
myCmd = f"rsync -avz {FULL_PATH}/{filename} {_FDEST}"
status, output = subprocess.getstatusoutput(myCmd)
files_imported += 1
except:
try:
#shutil.copy("{}/{}".format(FULL_PATH,filename),"{}/{}".format(_FDEST,filename))
myCmd = f"rsync -avz {FULL_PATH}/{filename} {_FDEST}"
logging.debug(myCmd)
status, output = subprocess.getstatusoutput(myCmd)
logging.debug(output)
logging.debug(status)
files_imported += 1
except:
logging.debug("failed to move file {}".format(filename))
moved_fine = False
#myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=100"
#subprocess.getstatusoutput(myCmd)
else:
logging.debug(f"File: {file_or_dir}")
mqtt_s.send_mqtt_message("sectorq/auto_import/job", {"status": "started","file":file_or_dir,"device":_DEV})
try: try:
shutil.move("{}/{}".format(FULL_PATH,filename),_FDEST) #shutil.copy("{}/{}".format(_DEV_DIR,file_or_dir),_FDEST)
myCmd = f"rsync -avz {_DEV_DIR}/{file_or_dir} {_FDEST}"
logging.debug(myCmd)
status, output = subprocess.getstatusoutput(myCmd)
files_imported += 1
except: except:
try: try:
shutil.move("{}/{}".format(FULL_PATH,filename),"{}/{}".format(_FDEST,filename)) #shutil.copy("{}/{}".format(_DEV_DIR,file_or_dir),"{}/{}".format(_FDEST,file_or_dir))
myCmd = f"rsync -avz {_DEV_DIR}/{file_or_dir} {_FDEST}"
logging.debug(myCmd)
status, output = subprocess.getstatusoutput(myCmd)
logging.debug(output)
logging.debug(status)
files_imported += 1
except: except:
print("failed to move file {}".format(filename)) logging.debug("failed to move file {}".format(file_or_dir))
print("Umounting") moved_fine = False
myCmd = "umount /share/external/DEV3304_1"
if moved_fine:
print("All files moved successfully")
print(f"Removing {_DEV_DIR}*")
myCmd = f"rm -rf {_DEV_DIR}/*"
status, output = subprocess.getstatusoutput(myCmd) status, output = subprocess.getstatusoutput(myCmd)
print(output) logging.debug("Umounting")
print(status)
myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=100"
subprocess.getstatusoutput(myCmd)
time.sleep(5) myCmd = f"sudo umount {_DEV_MOUNT}"
else: status, output = subprocess.getstatusoutput(myCmd)
print(f"File: {file_or_dir}") logging.debug(output)
try: logging.debug(status)
shutil.move("{}/{}".format(_DEV_DIR,file_or_dir),_FDEST) myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=101;sleep 1;/sbin/hal_app --se_buzzer enc_id=0,mode=101"
except: status, output = subprocess.getstatusoutput(myCmd)
try: else:
shutil.move("{}/{}".format(_DEV_DIR,file_or_dir),"{}/{}".format(_FDEST,file_or_dir)) logging.debug("Nothing to import")
except: mqtt_s.send_mqtt_message("sectorq/auto_import/job", {"status": "finished","file":f"{files_imported}","device":_DEV})
print("failed to move file {}".format(file_or_dir))
else: else:
print("Nothing to import") logging.debug("Unknown device")
time.sleep(5)
else:
print("Unknown device")
time.sleep(5)
else:
print("Usb does not connected")
time.sleep(5)

251
auto_import_v2_auto.py
View File

@@ -1,251 +0,0 @@
#!/myapps/venv/bin/python3
# Auto import script for media devices
import os
from posixpath import ismount
import sys
import time
import subprocess
import datetime
import shutil
import json
import logging
from random import randrange
from paho.mqtt import client as mqtt_client
class mqtt_sender():
def __init__(self, broker, port, name, password):
self.broker = broker
self.name = name
self.password = password
self.port = port
self.client_id = "dasdasdasd333"
try:
self.client = mqtt_client.Client(mqtt_client.CallbackAPIVersion.VERSION1, self.client_id)
except:
self.client = mqtt_client.Client()
self.client.username_pw_set(self.name, self.password)
def send_mqtt_message(_self, topic, msg):
try:
_self.client.connect(_self.broker,_self.port,60)
_self.client.publish(topic, json.dumps(msg))
_self.client.disconnect()
print("Message send")
except ValueError as e:
print("Failed to send")
print(e)
mqtt_s = mqtt_sender("mqtt.home.lan", 1883, "jaydee", "jaydee1")
mqtt_s.send_mqtt_message("sectorq/auto_import/job", {"status": "started","file":"","device":""})
# Log file location and log level
LOG_FILE = "/tmp/auto_import.log"
_LOG_LEVEL = "DEBUG"
# Configure logging based on log level
if _LOG_LEVEL == "DEBUG":
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.debug('using debug loging')
elif _LOG_LEVEL == "ERROR":
logging.basicConfig(filename=LOG_FILE, level=logging.ERROR, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.info('using error loging')
elif _LOG_LEVEL == "SCAN":
logging.basicConfig(filename=LOG_FILE, level=logging.DEBUG, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.info('using error loging')
else:
logging.basicConfig(filename=LOG_FILE, level=logging.INFO, format='%(asctime)s : %(levelname)s : %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p')
logging.info("script started")
# Get current process id and check for duplicate running processes
pid = os.getpid()
cmnd = "ps -ef|grep test.py|grep -v grep"
status, output = subprocess.getstatusoutput(cmnd)
logging.info(output)
logging.info(pid)
pid = os.getpid()
cmnd = "ps -ef|grep test.py|grep -v grep |grep -v {}|wc -l".format(pid)
status, output = subprocess.getstatusoutput(cmnd)
if int(output) > 0:
logging.info("Running already!")
sys.exit()
# Set destination and initialize variables
_DEST = "/share/Photo/imported"
_DEV = ""
files_imported = 0
while True:
myCmd = "mount| grep /share/external/DEV3301| grep -v grep | awk '{ print $3 }'"
status, output = subprocess.getstatusoutput(myCmd)
print(output)
print(status)
devs = output.splitlines()
if len(devs) == 0:
time.sleep(10)
else:
for _DEV_MOUNT in devs:
print("Device mounted")
myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=101"
status, output = subprocess.getstatusoutput(myCmd)
# _DEV_MOUNT = f"/share/external/DEV3301_1"
_DEV_DIR = f"{_DEV_MOUNT}/DCIM"
_IMP_FILE = f"{_DEV_MOUNT}/auto_import"
_DEV_DATA = f"{_DEV_MOUNT}/MISC/version.txt"
# If version.txt exists, parse device info
if os.path.exists(_DEV_DATA):
file = open(_DEV_DATA, 'r')
content = file.read()
file.close()
strToReplace = ','
replacementStr = ''
strToReplaceReversed = strToReplace[::-1]
replacementStrReversed = replacementStr[::-1]
strValue = content[::-1].replace(strToReplaceReversed, replacementStrReversed, 1)[::-1]
c = content.replace("\n","")
logging.debug("--------------")
logging.debug(c[-2])
logging.debug("--------------")
logging.debug(strValue)
_DEV_CONT = json.loads(strValue)
logging.debug(_DEV_CONT)
logging.debug(_DEV_CONT["wifi mac"])
wifimac = _DEV_CONT["wifi mac"]
# Identify device by wifi mac
if wifimac == "2474f742c017":
_DEV = "gopro9_1"
logging.debug("gopro9_1 registered")
elif wifimac == "2474f742c017":
_DEV = "gopro9_1"
logging.debug("gopro9_1 registered")
elif wifimac == "XXXDJIMINI4":
_DEV = "dji_mini_4"
logging.debug("dji_mini_4 registered")
# If auto_import file exists and device is not set, read device info
if os.path.exists(_IMP_FILE) and _DEV == "":
logging.debug("path exist")
time.sleep(5)
if _DEV == "":
try:
file = open(_IMP_FILE, 'r')
except:
break
content = file.read()
_DEV = json.loads(content)["device"]
logging.debug(f"{_DEV} registered")
file.close()
x = datetime.datetime.now()
logging.debug(x.year)
_DAT = x.strftime("%Y%m%d")
# If DCIM folder is empty, unmount and continue
if len(os.listdir(_DEV_DIR)) == 0:
logging.debug("Folder is empty!")
myCmd = f"umount {_DEV_DIR}"
status, output = subprocess.getstatusoutput(myCmd)
time.sleep(5)
continue
# myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=101;sleep 1;/sbin/hal_app --se_buzzer enc_id=0,mode=101;sleep 1;/sbin/hal_app --se_buzzer enc_id=0,mode=101"
# status, output = subprocess.getstatusoutput(myCmd)
# If device is set and DCIM exists, move files by date
moved_fine = True
if not _DEV == "":
if os.path.exists(_DEV_DIR):
for file_or_dir in os.listdir(_DEV_DIR):
FULL_PATH = os.path.join(_DEV_DIR, file_or_dir)
logging.debug(f"Full path : {FULL_PATH}")
if os.path.isdir(FULL_PATH):
logging.debug(f"Folder: {file_or_dir}")
for filename in os.listdir(FULL_PATH):
logging.debug(filename)
c_time = os.path.getctime("{}/{}".format(FULL_PATH,filename))
dt_c = datetime.datetime.fromtimestamp(c_time)
logging.debug(dt_c)
_CDAT = dt_c.strftime("%Y-%m-%d")
logging.debug(_CDAT)
_FDEST = "{}/{}/{}".format(_DEST,_DEV, _CDAT)
if not os.path.exists(_FDEST):
cmnd = "mkdir -p {}".format(_FDEST)
status, output = subprocess.getstatusoutput(cmnd)
logging.debug("Moving file {} into {}".format(filename, _FDEST))
mqtt_s.send_mqtt_message("sectorq/auto_import/job", {"status": "started","file":filename,"device":_DEV})
try:
#shutil.copy("{}/{}".format(FULL_PATH,filename),_FDEST)
myCmd = f"rsync -avz {FULL_PATH}/{filename} {_FDEST}"
status, output = subprocess.getstatusoutput(myCmd)
files_imported += 1
except:
try:
#shutil.copy("{}/{}".format(FULL_PATH,filename),"{}/{}".format(_FDEST,filename))
myCmd = f"rsync -avz {FULL_PATH}/{filename} {_FDEST}"
logging.debug(myCmd)
status, output = subprocess.getstatusoutput(myCmd)
logging.debug(output)
logging.debug(status)
files_imported += 1
except:
logging.debug("failed to move file {}".format(filename))
moved_fine = False
#myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=100"
#subprocess.getstatusoutput(myCmd)
else:
logging.debug(f"File: {file_or_dir}")
mqtt_s.send_mqtt_message("sectorq/auto_import/job", {"status": "started","file":file_or_dir,"device":_DEV})
try:
#shutil.copy("{}/{}".format(_DEV_DIR,file_or_dir),_FDEST)
myCmd = f"rsync -avz {_DEV_DIR}/{file_or_dir} {_FDEST}"
logging.debug(myCmd)
status, output = subprocess.getstatusoutput(myCmd)
files_imported += 1
except:
try:
#shutil.copy("{}/{}".format(_DEV_DIR,file_or_dir),"{}/{}".format(_FDEST,file_or_dir))
myCmd = f"rsync -avz {_DEV_DIR}/{file_or_dir} {_FDEST}"
logging.debug(myCmd)
status, output = subprocess.getstatusoutput(myCmd)
logging.debug(output)
logging.debug(status)
files_imported += 1
except:
logging.debug("failed to move file {}".format(file_or_dir))
moved_fine = False
if moved_fine:
print("All files moved successfully")
print(f"Removing {_DEV_DIR}*")
myCmd = f"rm -rf {_DEV_DIR}/*"
status, output = subprocess.getstatusoutput(myCmd)
logging.debug("Umounting")
myCmd = f"sudo umount {_DEV_MOUNT}"
status, output = subprocess.getstatusoutput(myCmd)
logging.debug(output)
logging.debug(status)
myCmd = "/sbin/hal_app --se_buzzer enc_id=0,mode=101;sleep 1;/sbin/hal_app --se_buzzer enc_id=0,mode=101"
status, output = subprocess.getstatusoutput(myCmd)
else:
logging.debug("Nothing to import")
mqtt_s.send_mqtt_message("sectorq/auto_import/job", {"status": "finished","file":f"{files_imported}","device":_DEV})
else:
logging.debug("Unknown device")