From 5f8d2a7109dfad8ddc7c51859f3d725e61132eec Mon Sep 17 00:00:00 2001 From: Tobias Brunner Date: Tue, 2 Mar 2021 21:52:37 +0100 Subject: [PATCH] rewrite email handling --- .gitignore | 1 + pylokid/__init__.py | 4 - pylokid/library/emailhandling.py | 119 +++++++------ pylokid/main.py | 276 ++++++++++++++++--------------- pyproject.toml | 2 +- 5 files changed, 213 insertions(+), 189 deletions(-) diff --git a/.gitignore b/.gitignore index b342fa5..884bb6e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ __pycache__/ .vscode/ .env +pylokid/temp_test.py test.py \ No newline at end of file diff --git a/pylokid/__init__.py b/pylokid/__init__.py index b68dbf1..266310e 100644 --- a/pylokid/__init__.py +++ b/pylokid/__init__.py @@ -1,7 +1,3 @@ """ Pylokid. From Mail to Lodur - all automated. """ - -__version__ = "3.0.2" -__git_version__ = "0" -__url__ = "https://github.com/tobru/pylokid" diff --git a/pylokid/library/emailhandling.py b/pylokid/library/emailhandling.py index 71b3006..1a2c0ac 100644 --- a/pylokid/library/emailhandling.py +++ b/pylokid/library/emailhandling.py @@ -36,6 +36,8 @@ class EmailHandling: def search_emails(self): """ searches for emails matching the configured subject """ + msg_ids = [] + self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS) try: typ, msg_ids = self.imap.search( @@ -49,61 +51,84 @@ class EmailHandling: self.logger.error("IMAP search aborted - exiting: %s", str(err)) raise SystemExit(1) - num_messages = len(msg_ids[0].split()) - self.logger.info("Found %s matching messages", str(num_messages)) - - return num_messages, msg_ids - - def store_attachments(self, msg_ids): - """ stores the attachments to filesystem """ - - data = {} - for msg_id in msg_ids[0].split(): - # download message from imap - typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])") + msg_list = msg_ids[0].split() + self.logger.info("Found %s matching messages", str(len(msg_list))) + # Retrieve subjects + msg_id_subject = {} + for msg in msg_list: + msg_id = msg.decode("utf-8") + typ, msg_data = self.imap.fetch(msg, "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])") if typ != "OK": - self.logger.error("Error fetching message") - continue + self.logger.error("Error fetching subject") + msg_id_subject[msg_id] = "unknown" + else: + mail = email.message_from_string(str(msg_data[0][1], "utf-8")) + subject = mail["subject"] + self.logger.info("Message ID %s has subject '%s'", msg_id, subject) + msg_id_subject[msg_id] = subject - # extract attachment - for response_part in msg_data: - if isinstance(response_part, tuple): - mail = email.message_from_string(str(response_part[1], "utf-8")) - subject = mail["subject"] - f_type, f_id = self.parse_subject(subject) - self.logger.info('[%s] Getting attachment from "%s"', f_id, subject) - for part in mail.walk(): - file_name = part.get_filename() - if not file_name: - self.logger.debug( - "Most probably not an attachment as no filename found" - ) - continue + # Deduplicate messages - usually the same message arrives multiple times + self.logger.info("Deduplicating messages") + temp = [] + msg_id_subject_deduplicated = dict() + for key, val in msg_id_subject.items(): + if val not in temp: + temp.append(val) + msg_id_subject_deduplicated[key] = val + self.logger.info( + "Adding Message ID %s '%s' to list to process", msg_id, subject + ) + else: + self.mark_seen(key, key) + + return msg_id_subject_deduplicated + + def store_attachment(self, msg_id): + """ stores the attachment to filesystem """ + + # download message from imap + typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])") + + if typ != "OK": + self.logger.error("Error fetching message") + return None, None + + # extract attachment + for response_part in msg_data: + if isinstance(response_part, tuple): + mail = email.message_from_string(str(response_part[1], "utf-8")) + subject = mail["subject"] + f_type, f_id = self.parse_subject(subject) + self.logger.info('[%s] Getting attachment from "%s"', f_id, subject) + for part in mail.walk(): + file_name = part.get_filename() + if not file_name: + self.logger.debug( + "Most probably not an attachment as no filename found" + ) + continue + + self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name) + + if bool(file_name): + f_type, _ = self.parse_subject(subject) + renamed_file_name = f_type + "_" + file_name + # save attachment to filesystem + file_path = os.path.join(self.tmp_dir, renamed_file_name) self.logger.info( - '[%s] Extracting attachment "%s"', f_id, file_name + '[%s] Saving attachment to "%s"', f_id, file_path ) + if not os.path.isfile(file_path): + file = open(file_path, "wb") + file.write(part.get_payload(decode=True)) + file.close() - if bool(file_name): - f_type, _ = self.parse_subject(subject) - renamed_file_name = f_type + "_" + file_name - # save attachment to filesystem - file_path = os.path.join(self.tmp_dir, renamed_file_name) + return renamed_file_name - self.logger.info( - '[%s] Saving attachment to "%s"', f_id, file_path - ) - if not os.path.isfile(file_path): - file = open(file_path, "wb") - file.write(part.get_payload(decode=True)) - file.close() - - data[subject] = renamed_file_name - - return data - - def mark_seen(self, msg_id): + def mark_seen(self, msg_id, f_id): + self.logger.info("[%s] Marking E-Mail message as seen", f_id) self.imap.store(msg_id, "+FLAGS", "(\\Seen)") def parse_subject(self, subject): diff --git a/pylokid/main.py b/pylokid/main.py index fb1c544..2048a3a 100644 --- a/pylokid/main.py +++ b/pylokid/main.py @@ -43,7 +43,7 @@ def main(): # Logging configuration logging.basicConfig( level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + format="%(asctime)s - %(levelname)s - %(message)s", ) logger = logging.getLogger("pylokid") logger.info("Starting pylokid version %s", version("pylokid")) @@ -80,170 +80,172 @@ def main(): pdf = PDFParsing() # Main Loop + logger.info("** Starting to process E-Mails **") while True: - attachments = {} - num_messages, msg_ids = imap_client.search_emails() - if num_messages: - attachments = imap_client.store_attachments(msg_ids) - if attachments: - for subject in attachments: - f_type, f_id = imap_client.parse_subject(subject) - file_name = attachments[subject] + # Search for matchting E-Mails + msg_ids = imap_client.search_emails() - # Upload file to cloud - webdav_client.upload(file_name, f_id) + for msg, subject in msg_ids.items(): + logger.info("Processing IMAP message ID %s", msg) + file_name = imap_client.store_attachment(msg) - # Take actions - depending on the type - if f_type == "Einsatzausdruck_FW": - logger.info("[%s] Processing type %s", f_id, f_type) + # If the message couldn't be parsed, skip to next message + if not file_name: + pass - # Check if the PDF isn't already parsed - if webdav_client.get_lodur_data(f_id, "_pdf.json"): - logger.info("[%s] PDF already parsed", f_id) - else: - # Extract information from PDF - pdf_data = pdf.extract_einsatzausdruck( - os.path.join(TMP_DIR, file_name), - f_id, - ) + # Figure out event type and F ID by parsing the subject + f_type, f_id = imap_client.parse_subject(subject) - # publish Einsatz on Pushover - logger.info("[%s] Publishing message on Pushover", f_id) - pushover.send_message( - "{}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format( - pdf_data["einsatz"], - pdf_data["ort"], - pdf_data["melder"].replace("\n", " "), - pdf_data["hinweis"], - pdf_data["sondersignal"], - pdf_data["bemerkungen"], - pdf_data["disponierteeinheiten"], - ), - title="Feuerwehr Einsatz - {}".format(f_id), - url="https://www.google.com/maps/search/?api=1&query={}".format( - pdf_data["ort"] - ), - url_title="Ort auf Karte suchen", - html=1, - ) + # Upload extracted attachment to cloud + webdav_client.upload(file_name, f_id) - # Upload extracted data to cloud - webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data) + # Take actions - depending on the type + if f_type == "Einsatzausdruck_FW": + logger.info("[%s] Processing type %s", f_id, f_type) - if webdav_client.get_lodur_data(f_id): - logger.info("[%s] Lodur data already retrieved", f_id) - else: - # Retrieve data from Lodur - lodur_id = lodur_client.get_einsatzrapport_id(f_id) - if lodur_id: - logger.info( - "[%s] Einsatzrapport available in Lodur with ID %s", - f_id, - lodur_id, - ) - logger.info( - "%s?modul=36&what=144&event=%s&edit=1", - LODUR_BASE_URL, - lodur_id, - ) - - lodur_data = lodur_client.retrieve_form_data(lodur_id) - webdav_client.store_data( - f_id, f_id + "_lodur.json", lodur_data - ) - - # upload Alarmdepesche PDF to Lodur - lodur_client.upload_alarmdepesche( - f_id, - os.path.join(TMP_DIR, file_name), - webdav_client, - ) - - # Marking message as seen, no need to reprocess again - for msg_id in msg_ids: - logger.info("[%s] Marking E-Mail message as seen", f_id) - imap_client.mark_seen(msg_id) - else: - logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id) - - elif f_type == "Einsatzprotokoll": - - lodur_id = webdav_client.get_lodur_data(f_id)["event_id"] - pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json") - logger.info( - "[%s] Processing type %s with Lodur ID %s", + # Check if the PDF isn't already parsed + if webdav_client.get_lodur_data(f_id, "_pdf.json"): + logger.info("[%s] PDF already parsed", f_id) + else: + # Extract information from PDF + pdf_data = pdf.extract_einsatzausdruck( + os.path.join(TMP_DIR, file_name), f_id, - f_type, - lodur_id, ) - # Retrieve Lodur data again and store it in Webdav - lodur_data = lodur_client.retrieve_form_data(lodur_id) - webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) + # publish Einsatz on Pushover + logger.info("[%s] Publishing message on Pushover", f_id) + pushover.send_message( + "{}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format( + pdf_data["einsatz"], + pdf_data["ort"], + pdf_data["melder"].replace("\n", " "), + pdf_data["hinweis"], + pdf_data["sondersignal"], + pdf_data["bemerkungen"], + pdf_data["disponierteeinheiten"], + ), + title="Feuerwehr Einsatz - {}".format(f_id), + url="https://www.google.com/maps/search/?api=1&query={}".format( + pdf_data["ort"] + ), + url_title="Ort auf Karte suchen", + html=1, + ) - if ( - "aut_created_report" in lodur_data - and lodur_data["aut_created_report"] == "finished" - ): - logger.info("[%s] Record in Lodur ready to be updated", f_id) + # Upload extracted data to cloud + webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data) - # Upload Einsatzprotokoll to Lodur + if webdav_client.get_lodur_data(f_id): + logger.info("[%s] Lodur data already retrieved", f_id) + # Marking message as seen, no need to reprocess again + imap_client.mark_seen(msg, f_id) + else: + # Retrieve data from Lodur + lodur_id = lodur_client.get_einsatzrapport_id(f_id) + if lodur_id: + logger.info( + "[%s] Einsatzrapport available in Lodur with ID %s", + f_id, + lodur_id, + ) + logger.info( + "%s?modul=36&what=144&event=%s&edit=1", + LODUR_BASE_URL, + lodur_id, + ) + + lodur_data = lodur_client.retrieve_form_data(lodur_id) + webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) + + # upload Alarmdepesche PDF to Lodur lodur_client.upload_alarmdepesche( f_id, os.path.join(TMP_DIR, file_name), webdav_client, ) - # Update entry in Lodur - lodur_client.einsatzprotokoll( - f_id, lodur_data, pdf_data, webdav_client - ) - - # Einsatz finished - publish on pushover - logger.info("[%s] Publishing message on Pushover", f_id) - pushover.send_message( - "Einsatz beendet", - title="Feuerwehr Einsatz beendet - {}".format(f_id), - ) - # Marking message as seen, no need to reprocess again - for msg_id in msg_ids: - logger.info("[%s] Marking E-Mail message as seen", f_id) - imap_client.mark_seen(msg_id) - + imap_client.mark_seen(msg, f_id) else: - logger.warn( - "[%s] Record in Lodur NOT ready yet to be updated", f_id - ) + logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id) - # This is usually a scan from the Depot printer - elif f_type == "Einsatzrapport": + elif f_type == "Einsatzprotokoll": - logger.info("[%s] Processing type %s", f_id, f_type) + lodur_id = webdav_client.get_lodur_data(f_id)["event_id"] + pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json") + logger.info( + "[%s] Processing type %s with Lodur ID %s", + f_id, + f_type, + lodur_id, + ) - # Attach scan in Lodur if f_id is available - # f_id can be empty when scan was misconfigured - if f_id != None: - lodur_id = webdav_client.get_lodur_data(f_id)["event_id"] - # Retrieve Lodur data again and store it in Webdav - lodur_data = lodur_client.retrieve_form_data(lodur_id) - webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) - lodur_client.einsatzrapport_scan( - f_id, - lodur_data, - os.path.join(TMP_DIR, file_name), - webdav_client, - ) + # Retrieve Lodur data again and store it in Webdav + lodur_data = lodur_client.retrieve_form_data(lodur_id) + webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) - logger.info("[%s] Publishing message on Pushover", f_id) + if ( + "aut_created_report" in lodur_data + and lodur_data["aut_created_report"] == "finished" + ): + logger.info("[%s] Record in Lodur ready to be updated", f_id) - pushover.send_message( - "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id), - title="Feuerwehr Scan bearbeitet - {}".format(f_id), + # Upload Einsatzprotokoll to Lodur + lodur_client.upload_alarmdepesche( + f_id, + os.path.join(TMP_DIR, file_name), + webdav_client, ) + + # Update entry in Lodur + lodur_client.einsatzprotokoll( + f_id, lodur_data, pdf_data, webdav_client + ) + + # Einsatz finished - publish on pushover + logger.info("[%s] Publishing message on Pushover", f_id) + pushover.send_message( + "Einsatz beendet", + title="Feuerwehr Einsatz beendet - {}".format(f_id), + ) + + # Marking message as seen, no need to reprocess again + imap_client.mark_seen(msg, f_id) + else: - logger.error("[%s] Unknown type: %s", f_id, f_type) + logger.warn( + "[%s] Record in Lodur NOT ready yet to be updated", f_id + ) + + # This is usually a scan from the Depot printer + elif f_type == "Einsatzrapport": + + logger.info("[%s] Processing type %s", f_id, f_type) + + # Attach scan in Lodur if f_id is available + # f_id can be empty when scan was misconfigured + if f_id != None: + lodur_id = webdav_client.get_lodur_data(f_id)["event_id"] + # Retrieve Lodur data again and store it in Webdav + lodur_data = lodur_client.retrieve_form_data(lodur_id) + webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) + lodur_client.einsatzrapport_scan( + f_id, + lodur_data, + os.path.join(TMP_DIR, file_name), + webdav_client, + ) + + logger.info("[%s] Publishing message on Pushover", f_id) + + pushover.send_message( + "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id), + title="Feuerwehr Scan bearbeitet - {}".format(f_id), + ) + else: + logger.error("[%s] Unknown type: %s", f_id, f_type) # send heartbeat requests.get(HEARTBEAT_URL) diff --git a/pyproject.toml b/pyproject.toml index a2f1ed8..2bbd35b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pylokid" -version = "3.0.3" +version = "3.1.0" description = "" authors = ["Tobias Brunner "] license = "MIT"