rewrite email handling
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Tobias Brunner 2021-03-02 21:52:37 +01:00
parent 8a22747315
commit 5f8d2a7109
5 changed files with 213 additions and 189 deletions

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
__pycache__/ __pycache__/
.vscode/ .vscode/
.env .env
pylokid/temp_test.py
test.py test.py

View File

@ -1,7 +1,3 @@
""" """
Pylokid. From Mail to Lodur - all automated. Pylokid. From Mail to Lodur - all automated.
""" """
__version__ = "3.0.2"
__git_version__ = "0"
__url__ = "https://github.com/tobru/pylokid"

View File

@ -36,6 +36,8 @@ class EmailHandling:
def search_emails(self): def search_emails(self):
""" searches for emails matching the configured subject """ """ searches for emails matching the configured subject """
msg_ids = []
self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS) self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
try: try:
typ, msg_ids = self.imap.search( typ, msg_ids = self.imap.search(
@ -49,61 +51,84 @@ class EmailHandling:
self.logger.error("IMAP search aborted - exiting: %s", str(err)) self.logger.error("IMAP search aborted - exiting: %s", str(err))
raise SystemExit(1) raise SystemExit(1)
num_messages = len(msg_ids[0].split()) msg_list = msg_ids[0].split()
self.logger.info("Found %s matching messages", str(num_messages)) self.logger.info("Found %s matching messages", str(len(msg_list)))
return num_messages, msg_ids
def store_attachments(self, msg_ids):
""" stores the attachments to filesystem """
data = {}
for msg_id in msg_ids[0].split():
# download message from imap
typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
# Retrieve subjects
msg_id_subject = {}
for msg in msg_list:
msg_id = msg.decode("utf-8")
typ, msg_data = self.imap.fetch(msg, "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
if typ != "OK": if typ != "OK":
self.logger.error("Error fetching message") self.logger.error("Error fetching subject")
continue msg_id_subject[msg_id] = "unknown"
else:
mail = email.message_from_string(str(msg_data[0][1], "utf-8"))
subject = mail["subject"]
self.logger.info("Message ID %s has subject '%s'", msg_id, subject)
msg_id_subject[msg_id] = subject
# extract attachment # Deduplicate messages - usually the same message arrives multiple times
for response_part in msg_data: self.logger.info("Deduplicating messages")
if isinstance(response_part, tuple): temp = []
mail = email.message_from_string(str(response_part[1], "utf-8")) msg_id_subject_deduplicated = dict()
subject = mail["subject"] for key, val in msg_id_subject.items():
f_type, f_id = self.parse_subject(subject) if val not in temp:
self.logger.info('[%s] Getting attachment from "%s"', f_id, subject) temp.append(val)
for part in mail.walk(): msg_id_subject_deduplicated[key] = val
file_name = part.get_filename() self.logger.info(
if not file_name: "Adding Message ID %s '%s' to list to process", msg_id, subject
self.logger.debug( )
"Most probably not an attachment as no filename found" else:
) self.mark_seen(key, key)
continue
return msg_id_subject_deduplicated
def store_attachment(self, msg_id):
""" stores the attachment to filesystem """
# download message from imap
typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
if typ != "OK":
self.logger.error("Error fetching message")
return None, None
# extract attachment
for response_part in msg_data:
if isinstance(response_part, tuple):
mail = email.message_from_string(str(response_part[1], "utf-8"))
subject = mail["subject"]
f_type, f_id = self.parse_subject(subject)
self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
for part in mail.walk():
file_name = part.get_filename()
if not file_name:
self.logger.debug(
"Most probably not an attachment as no filename found"
)
continue
self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
if bool(file_name):
f_type, _ = self.parse_subject(subject)
renamed_file_name = f_type + "_" + file_name
# save attachment to filesystem
file_path = os.path.join(self.tmp_dir, renamed_file_name)
self.logger.info( self.logger.info(
'[%s] Extracting attachment "%s"', f_id, file_name '[%s] Saving attachment to "%s"', f_id, file_path
) )
if not os.path.isfile(file_path):
file = open(file_path, "wb")
file.write(part.get_payload(decode=True))
file.close()
if bool(file_name): return renamed_file_name
f_type, _ = self.parse_subject(subject)
renamed_file_name = f_type + "_" + file_name
# save attachment to filesystem
file_path = os.path.join(self.tmp_dir, renamed_file_name)
self.logger.info( def mark_seen(self, msg_id, f_id):
'[%s] Saving attachment to "%s"', f_id, file_path self.logger.info("[%s] Marking E-Mail message as seen", f_id)
)
if not os.path.isfile(file_path):
file = open(file_path, "wb")
file.write(part.get_payload(decode=True))
file.close()
data[subject] = renamed_file_name
return data
def mark_seen(self, msg_id):
self.imap.store(msg_id, "+FLAGS", "(\\Seen)") self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
def parse_subject(self, subject): def parse_subject(self, subject):

View File

@ -43,7 +43,7 @@ def main():
# Logging configuration # Logging configuration
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", format="%(asctime)s - %(levelname)s - %(message)s",
) )
logger = logging.getLogger("pylokid") logger = logging.getLogger("pylokid")
logger.info("Starting pylokid version %s", version("pylokid")) logger.info("Starting pylokid version %s", version("pylokid"))
@ -80,170 +80,172 @@ def main():
pdf = PDFParsing() pdf = PDFParsing()
# Main Loop # Main Loop
logger.info("** Starting to process E-Mails **")
while True: while True:
attachments = {}
num_messages, msg_ids = imap_client.search_emails()
if num_messages:
attachments = imap_client.store_attachments(msg_ids)
if attachments: # Search for matchting E-Mails
for subject in attachments: msg_ids = imap_client.search_emails()
f_type, f_id = imap_client.parse_subject(subject)
file_name = attachments[subject]
# Upload file to cloud for msg, subject in msg_ids.items():
webdav_client.upload(file_name, f_id) logger.info("Processing IMAP message ID %s", msg)
file_name = imap_client.store_attachment(msg)
# Take actions - depending on the type # If the message couldn't be parsed, skip to next message
if f_type == "Einsatzausdruck_FW": if not file_name:
logger.info("[%s] Processing type %s", f_id, f_type) pass
# Check if the PDF isn't already parsed # Figure out event type and F ID by parsing the subject
if webdav_client.get_lodur_data(f_id, "_pdf.json"): f_type, f_id = imap_client.parse_subject(subject)
logger.info("[%s] PDF already parsed", f_id)
else:
# Extract information from PDF
pdf_data = pdf.extract_einsatzausdruck(
os.path.join(TMP_DIR, file_name),
f_id,
)
# publish Einsatz on Pushover # Upload extracted attachment to cloud
logger.info("[%s] Publishing message on Pushover", f_id) webdav_client.upload(file_name, f_id)
pushover.send_message(
"<b>{}</b>\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
pdf_data["einsatz"],
pdf_data["ort"],
pdf_data["melder"].replace("\n", " "),
pdf_data["hinweis"],
pdf_data["sondersignal"],
pdf_data["bemerkungen"],
pdf_data["disponierteeinheiten"],
),
title="Feuerwehr Einsatz - {}".format(f_id),
url="https://www.google.com/maps/search/?api=1&query={}".format(
pdf_data["ort"]
),
url_title="Ort auf Karte suchen",
html=1,
)
# Upload extracted data to cloud # Take actions - depending on the type
webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data) if f_type == "Einsatzausdruck_FW":
logger.info("[%s] Processing type %s", f_id, f_type)
if webdav_client.get_lodur_data(f_id): # Check if the PDF isn't already parsed
logger.info("[%s] Lodur data already retrieved", f_id) if webdav_client.get_lodur_data(f_id, "_pdf.json"):
else: logger.info("[%s] PDF already parsed", f_id)
# Retrieve data from Lodur else:
lodur_id = lodur_client.get_einsatzrapport_id(f_id) # Extract information from PDF
if lodur_id: pdf_data = pdf.extract_einsatzausdruck(
logger.info( os.path.join(TMP_DIR, file_name),
"[%s] Einsatzrapport available in Lodur with ID %s",
f_id,
lodur_id,
)
logger.info(
"%s?modul=36&what=144&event=%s&edit=1",
LODUR_BASE_URL,
lodur_id,
)
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(
f_id, f_id + "_lodur.json", lodur_data
)
# upload Alarmdepesche PDF to Lodur
lodur_client.upload_alarmdepesche(
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
# Marking message as seen, no need to reprocess again
for msg_id in msg_ids:
logger.info("[%s] Marking E-Mail message as seen", f_id)
imap_client.mark_seen(msg_id)
else:
logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
elif f_type == "Einsatzprotokoll":
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json")
logger.info(
"[%s] Processing type %s with Lodur ID %s",
f_id, f_id,
f_type,
lodur_id,
) )
# Retrieve Lodur data again and store it in Webdav # publish Einsatz on Pushover
lodur_data = lodur_client.retrieve_form_data(lodur_id) logger.info("[%s] Publishing message on Pushover", f_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) pushover.send_message(
"<b>{}</b>\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
pdf_data["einsatz"],
pdf_data["ort"],
pdf_data["melder"].replace("\n", " "),
pdf_data["hinweis"],
pdf_data["sondersignal"],
pdf_data["bemerkungen"],
pdf_data["disponierteeinheiten"],
),
title="Feuerwehr Einsatz - {}".format(f_id),
url="https://www.google.com/maps/search/?api=1&query={}".format(
pdf_data["ort"]
),
url_title="Ort auf Karte suchen",
html=1,
)
if ( # Upload extracted data to cloud
"aut_created_report" in lodur_data webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data)
and lodur_data["aut_created_report"] == "finished"
):
logger.info("[%s] Record in Lodur ready to be updated", f_id)
# Upload Einsatzprotokoll to Lodur if webdav_client.get_lodur_data(f_id):
logger.info("[%s] Lodur data already retrieved", f_id)
# Marking message as seen, no need to reprocess again
imap_client.mark_seen(msg, f_id)
else:
# Retrieve data from Lodur
lodur_id = lodur_client.get_einsatzrapport_id(f_id)
if lodur_id:
logger.info(
"[%s] Einsatzrapport available in Lodur with ID %s",
f_id,
lodur_id,
)
logger.info(
"%s?modul=36&what=144&event=%s&edit=1",
LODUR_BASE_URL,
lodur_id,
)
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
# upload Alarmdepesche PDF to Lodur
lodur_client.upload_alarmdepesche( lodur_client.upload_alarmdepesche(
f_id, f_id,
os.path.join(TMP_DIR, file_name), os.path.join(TMP_DIR, file_name),
webdav_client, webdav_client,
) )
# Update entry in Lodur
lodur_client.einsatzprotokoll(
f_id, lodur_data, pdf_data, webdav_client
)
# Einsatz finished - publish on pushover
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"Einsatz beendet",
title="Feuerwehr Einsatz beendet - {}".format(f_id),
)
# Marking message as seen, no need to reprocess again # Marking message as seen, no need to reprocess again
for msg_id in msg_ids: imap_client.mark_seen(msg, f_id)
logger.info("[%s] Marking E-Mail message as seen", f_id)
imap_client.mark_seen(msg_id)
else: else:
logger.warn( logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
"[%s] Record in Lodur NOT ready yet to be updated", f_id
)
# This is usually a scan from the Depot printer elif f_type == "Einsatzprotokoll":
elif f_type == "Einsatzrapport":
logger.info("[%s] Processing type %s", f_id, f_type) lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json")
logger.info(
"[%s] Processing type %s with Lodur ID %s",
f_id,
f_type,
lodur_id,
)
# Attach scan in Lodur if f_id is available # Retrieve Lodur data again and store it in Webdav
# f_id can be empty when scan was misconfigured lodur_data = lodur_client.retrieve_form_data(lodur_id)
if f_id != None: webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
# Retrieve Lodur data again and store it in Webdav
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
lodur_client.einsatzrapport_scan(
f_id,
lodur_data,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
logger.info("[%s] Publishing message on Pushover", f_id) if (
"aut_created_report" in lodur_data
and lodur_data["aut_created_report"] == "finished"
):
logger.info("[%s] Record in Lodur ready to be updated", f_id)
pushover.send_message( # Upload Einsatzprotokoll to Lodur
"Scan {} wurde bearbeitet und in Cloud geladen".format(f_id), lodur_client.upload_alarmdepesche(
title="Feuerwehr Scan bearbeitet - {}".format(f_id), f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
) )
# Update entry in Lodur
lodur_client.einsatzprotokoll(
f_id, lodur_data, pdf_data, webdav_client
)
# Einsatz finished - publish on pushover
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"Einsatz beendet",
title="Feuerwehr Einsatz beendet - {}".format(f_id),
)
# Marking message as seen, no need to reprocess again
imap_client.mark_seen(msg, f_id)
else: else:
logger.error("[%s] Unknown type: %s", f_id, f_type) logger.warn(
"[%s] Record in Lodur NOT ready yet to be updated", f_id
)
# This is usually a scan from the Depot printer
elif f_type == "Einsatzrapport":
logger.info("[%s] Processing type %s", f_id, f_type)
# Attach scan in Lodur if f_id is available
# f_id can be empty when scan was misconfigured
if f_id != None:
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
# Retrieve Lodur data again and store it in Webdav
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
lodur_client.einsatzrapport_scan(
f_id,
lodur_data,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
title="Feuerwehr Scan bearbeitet - {}".format(f_id),
)
else:
logger.error("[%s] Unknown type: %s", f_id, f_type)
# send heartbeat # send heartbeat
requests.get(HEARTBEAT_URL) requests.get(HEARTBEAT_URL)

View File

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "pylokid" name = "pylokid"
version = "3.0.3" version = "3.1.0"
description = "" description = ""
authors = ["Tobias Brunner <tobias@tobru.ch>"] authors = ["Tobias Brunner <tobias@tobru.ch>"]
license = "MIT" license = "MIT"