rewrite email handling
continuous-integration/drone/push Build is passing
Details
continuous-integration/drone/push Build is passing
Details
This commit is contained in:
parent
8a22747315
commit
5f8d2a7109
|
@ -1,4 +1,5 @@
|
||||||
__pycache__/
|
__pycache__/
|
||||||
.vscode/
|
.vscode/
|
||||||
.env
|
.env
|
||||||
|
pylokid/temp_test.py
|
||||||
test.py
|
test.py
|
|
@ -1,7 +1,3 @@
|
||||||
"""
|
"""
|
||||||
Pylokid. From Mail to Lodur - all automated.
|
Pylokid. From Mail to Lodur - all automated.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
__version__ = "3.0.2"
|
|
||||||
__git_version__ = "0"
|
|
||||||
__url__ = "https://github.com/tobru/pylokid"
|
|
||||||
|
|
|
@ -36,6 +36,8 @@ class EmailHandling:
|
||||||
def search_emails(self):
|
def search_emails(self):
|
||||||
""" searches for emails matching the configured subject """
|
""" searches for emails matching the configured subject """
|
||||||
|
|
||||||
|
msg_ids = []
|
||||||
|
|
||||||
self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
|
self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
|
||||||
try:
|
try:
|
||||||
typ, msg_ids = self.imap.search(
|
typ, msg_ids = self.imap.search(
|
||||||
|
@ -49,22 +51,48 @@ class EmailHandling:
|
||||||
self.logger.error("IMAP search aborted - exiting: %s", str(err))
|
self.logger.error("IMAP search aborted - exiting: %s", str(err))
|
||||||
raise SystemExit(1)
|
raise SystemExit(1)
|
||||||
|
|
||||||
num_messages = len(msg_ids[0].split())
|
msg_list = msg_ids[0].split()
|
||||||
self.logger.info("Found %s matching messages", str(num_messages))
|
self.logger.info("Found %s matching messages", str(len(msg_list)))
|
||||||
|
|
||||||
return num_messages, msg_ids
|
# Retrieve subjects
|
||||||
|
msg_id_subject = {}
|
||||||
|
for msg in msg_list:
|
||||||
|
msg_id = msg.decode("utf-8")
|
||||||
|
typ, msg_data = self.imap.fetch(msg, "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
|
||||||
|
if typ != "OK":
|
||||||
|
self.logger.error("Error fetching subject")
|
||||||
|
msg_id_subject[msg_id] = "unknown"
|
||||||
|
else:
|
||||||
|
mail = email.message_from_string(str(msg_data[0][1], "utf-8"))
|
||||||
|
subject = mail["subject"]
|
||||||
|
self.logger.info("Message ID %s has subject '%s'", msg_id, subject)
|
||||||
|
msg_id_subject[msg_id] = subject
|
||||||
|
|
||||||
def store_attachments(self, msg_ids):
|
# Deduplicate messages - usually the same message arrives multiple times
|
||||||
""" stores the attachments to filesystem """
|
self.logger.info("Deduplicating messages")
|
||||||
|
temp = []
|
||||||
|
msg_id_subject_deduplicated = dict()
|
||||||
|
for key, val in msg_id_subject.items():
|
||||||
|
if val not in temp:
|
||||||
|
temp.append(val)
|
||||||
|
msg_id_subject_deduplicated[key] = val
|
||||||
|
self.logger.info(
|
||||||
|
"Adding Message ID %s '%s' to list to process", msg_id, subject
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.mark_seen(key, key)
|
||||||
|
|
||||||
|
return msg_id_subject_deduplicated
|
||||||
|
|
||||||
|
def store_attachment(self, msg_id):
|
||||||
|
""" stores the attachment to filesystem """
|
||||||
|
|
||||||
data = {}
|
|
||||||
for msg_id in msg_ids[0].split():
|
|
||||||
# download message from imap
|
# download message from imap
|
||||||
typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
|
typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
|
||||||
|
|
||||||
if typ != "OK":
|
if typ != "OK":
|
||||||
self.logger.error("Error fetching message")
|
self.logger.error("Error fetching message")
|
||||||
continue
|
return None, None
|
||||||
|
|
||||||
# extract attachment
|
# extract attachment
|
||||||
for response_part in msg_data:
|
for response_part in msg_data:
|
||||||
|
@ -81,9 +109,7 @@ class EmailHandling:
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
self.logger.info(
|
self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
|
||||||
'[%s] Extracting attachment "%s"', f_id, file_name
|
|
||||||
)
|
|
||||||
|
|
||||||
if bool(file_name):
|
if bool(file_name):
|
||||||
f_type, _ = self.parse_subject(subject)
|
f_type, _ = self.parse_subject(subject)
|
||||||
|
@ -99,11 +125,10 @@ class EmailHandling:
|
||||||
file.write(part.get_payload(decode=True))
|
file.write(part.get_payload(decode=True))
|
||||||
file.close()
|
file.close()
|
||||||
|
|
||||||
data[subject] = renamed_file_name
|
return renamed_file_name
|
||||||
|
|
||||||
return data
|
def mark_seen(self, msg_id, f_id):
|
||||||
|
self.logger.info("[%s] Marking E-Mail message as seen", f_id)
|
||||||
def mark_seen(self, msg_id):
|
|
||||||
self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
|
self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
|
||||||
|
|
||||||
def parse_subject(self, subject):
|
def parse_subject(self, subject):
|
||||||
|
|
|
@ -43,7 +43,7 @@ def main():
|
||||||
# Logging configuration
|
# Logging configuration
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
format="%(asctime)s - %(levelname)s - %(message)s",
|
||||||
)
|
)
|
||||||
logger = logging.getLogger("pylokid")
|
logger = logging.getLogger("pylokid")
|
||||||
logger.info("Starting pylokid version %s", version("pylokid"))
|
logger.info("Starting pylokid version %s", version("pylokid"))
|
||||||
|
@ -80,18 +80,24 @@ def main():
|
||||||
pdf = PDFParsing()
|
pdf = PDFParsing()
|
||||||
|
|
||||||
# Main Loop
|
# Main Loop
|
||||||
|
logger.info("** Starting to process E-Mails **")
|
||||||
while True:
|
while True:
|
||||||
attachments = {}
|
|
||||||
num_messages, msg_ids = imap_client.search_emails()
|
|
||||||
if num_messages:
|
|
||||||
attachments = imap_client.store_attachments(msg_ids)
|
|
||||||
|
|
||||||
if attachments:
|
# Search for matchting E-Mails
|
||||||
for subject in attachments:
|
msg_ids = imap_client.search_emails()
|
||||||
|
|
||||||
|
for msg, subject in msg_ids.items():
|
||||||
|
logger.info("Processing IMAP message ID %s", msg)
|
||||||
|
file_name = imap_client.store_attachment(msg)
|
||||||
|
|
||||||
|
# If the message couldn't be parsed, skip to next message
|
||||||
|
if not file_name:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Figure out event type and F ID by parsing the subject
|
||||||
f_type, f_id = imap_client.parse_subject(subject)
|
f_type, f_id = imap_client.parse_subject(subject)
|
||||||
file_name = attachments[subject]
|
|
||||||
|
|
||||||
# Upload file to cloud
|
# Upload extracted attachment to cloud
|
||||||
webdav_client.upload(file_name, f_id)
|
webdav_client.upload(file_name, f_id)
|
||||||
|
|
||||||
# Take actions - depending on the type
|
# Take actions - depending on the type
|
||||||
|
@ -133,6 +139,8 @@ def main():
|
||||||
|
|
||||||
if webdav_client.get_lodur_data(f_id):
|
if webdav_client.get_lodur_data(f_id):
|
||||||
logger.info("[%s] Lodur data already retrieved", f_id)
|
logger.info("[%s] Lodur data already retrieved", f_id)
|
||||||
|
# Marking message as seen, no need to reprocess again
|
||||||
|
imap_client.mark_seen(msg, f_id)
|
||||||
else:
|
else:
|
||||||
# Retrieve data from Lodur
|
# Retrieve data from Lodur
|
||||||
lodur_id = lodur_client.get_einsatzrapport_id(f_id)
|
lodur_id = lodur_client.get_einsatzrapport_id(f_id)
|
||||||
|
@ -149,9 +157,7 @@ def main():
|
||||||
)
|
)
|
||||||
|
|
||||||
lodur_data = lodur_client.retrieve_form_data(lodur_id)
|
lodur_data = lodur_client.retrieve_form_data(lodur_id)
|
||||||
webdav_client.store_data(
|
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
|
||||||
f_id, f_id + "_lodur.json", lodur_data
|
|
||||||
)
|
|
||||||
|
|
||||||
# upload Alarmdepesche PDF to Lodur
|
# upload Alarmdepesche PDF to Lodur
|
||||||
lodur_client.upload_alarmdepesche(
|
lodur_client.upload_alarmdepesche(
|
||||||
|
@ -161,9 +167,7 @@ def main():
|
||||||
)
|
)
|
||||||
|
|
||||||
# Marking message as seen, no need to reprocess again
|
# Marking message as seen, no need to reprocess again
|
||||||
for msg_id in msg_ids:
|
imap_client.mark_seen(msg, f_id)
|
||||||
logger.info("[%s] Marking E-Mail message as seen", f_id)
|
|
||||||
imap_client.mark_seen(msg_id)
|
|
||||||
else:
|
else:
|
||||||
logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
|
logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
|
||||||
|
|
||||||
|
@ -208,9 +212,7 @@ def main():
|
||||||
)
|
)
|
||||||
|
|
||||||
# Marking message as seen, no need to reprocess again
|
# Marking message as seen, no need to reprocess again
|
||||||
for msg_id in msg_ids:
|
imap_client.mark_seen(msg, f_id)
|
||||||
logger.info("[%s] Marking E-Mail message as seen", f_id)
|
|
||||||
imap_client.mark_seen(msg_id)
|
|
||||||
|
|
||||||
else:
|
else:
|
||||||
logger.warn(
|
logger.warn(
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "pylokid"
|
name = "pylokid"
|
||||||
version = "3.0.3"
|
version = "3.1.0"
|
||||||
description = ""
|
description = ""
|
||||||
authors = ["Tobias Brunner <tobias@tobru.ch>"]
|
authors = ["Tobias Brunner <tobias@tobru.ch>"]
|
||||||
license = "MIT"
|
license = "MIT"
|
||||||
|
|
Loading…
Reference in New Issue