rewrite email handling

.gitignore

@@ -1,4 +1,5 @@
 __pycache__/
 .vscode/
 .env
+pylokid/temp_test.py
 test.py

pylokid/__init__.py

@@ -1,7 +1,3 @@
 """
 Pylokid. From Mail to Lodur - all automated.
 """
-
-__version__ = "3.0.2"
-__git_version__ = "0"
-__url__ = "https://github.com/tobru/pylokid"

pylokid/library/emailhandling.py

@@ -36,6 +36,8 @@ class EmailHandling:
     def search_emails(self):
         """ searches for emails matching the configured subject """
 
+        msg_ids = []
+
         self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
         try:
             typ, msg_ids = self.imap.search(
@@ -49,22 +51,48 @@ class EmailHandling:
             self.logger.error("IMAP search aborted - exiting: %s", str(err))
             raise SystemExit(1)
 
-        num_messages = len(msg_ids[0].split())
-        self.logger.info("Found %s matching messages", str(num_messages))
+        msg_list = msg_ids[0].split()
+        self.logger.info("Found %s matching messages", str(len(msg_list)))
 
-        return num_messages, msg_ids
+        # Retrieve subjects
+        msg_id_subject = {}
+        for msg in msg_list:
+            msg_id = msg.decode("utf-8")
+            typ, msg_data = self.imap.fetch(msg, "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
+            if typ != "OK":
+                self.logger.error("Error fetching subject")
+                msg_id_subject[msg_id] = "unknown"
+            else:
+                mail = email.message_from_string(str(msg_data[0][1], "utf-8"))
+                subject = mail["subject"]
+                self.logger.info("Message ID %s has subject '%s'", msg_id, subject)
+                msg_id_subject[msg_id] = subject
 
-    def store_attachments(self, msg_ids):
-        """ stores the attachments to filesystem """
+        # Deduplicate messages - usually the same message arrives multiple times
+        self.logger.info("Deduplicating messages")
+        temp = []
+        msg_id_subject_deduplicated = dict()
+        for key, val in msg_id_subject.items():
+            if val not in temp:
+                temp.append(val)
+                msg_id_subject_deduplicated[key] = val
+                self.logger.info(
+                    "Adding Message ID %s '%s' to list to process", msg_id, subject
+                )
+            else:
+                self.mark_seen(key, key)
+
+        return msg_id_subject_deduplicated
+
+    def store_attachment(self, msg_id):
+        """ stores the attachment to filesystem """
 
-        data = {}
-        for msg_id in msg_ids[0].split():
         # download message from imap
         typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
 
         if typ != "OK":
             self.logger.error("Error fetching message")
-                continue
+            return None, None
 
         # extract attachment
         for response_part in msg_data:
@@ -81,9 +109,7 @@ class EmailHandling:
                         )
                         continue
 
-                        self.logger.info(
-                            '[%s] Extracting attachment "%s"', f_id, file_name
-                        )
+                    self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
 
                     if bool(file_name):
                         f_type, _ = self.parse_subject(subject)
@@ -99,11 +125,10 @@ class EmailHandling:
                             file.write(part.get_payload(decode=True))
                             file.close()
 
-                            data[subject] = renamed_file_name
+        return renamed_file_name
 
-        return data
-
-    def mark_seen(self, msg_id):
+    def mark_seen(self, msg_id, f_id):
+        self.logger.info("[%s] Marking E-Mail message as seen", f_id)
         self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
 
     def parse_subject(self, subject):

pylokid/main.py

@@ -43,7 +43,7 @@ def main():
     # Logging configuration
     logging.basicConfig(
         level=logging.INFO,
-        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        format="%(asctime)s - %(levelname)s - %(message)s",
     )
     logger = logging.getLogger("pylokid")
     logger.info("Starting pylokid version %s", version("pylokid"))
@@ -80,18 +80,24 @@ def main():
     pdf = PDFParsing()
 
     # Main Loop
+    logger.info("** Starting to process E-Mails **")
     while True:
-        attachments = {}
-        num_messages, msg_ids = imap_client.search_emails()
-        if num_messages:
-            attachments = imap_client.store_attachments(msg_ids)
 
-        if attachments:
-            for subject in attachments:
+        # Search for matchting E-Mails
+        msg_ids = imap_client.search_emails()
+
+        for msg, subject in msg_ids.items():
+            logger.info("Processing IMAP message ID %s", msg)
+            file_name = imap_client.store_attachment(msg)
+
+            # If the message couldn't be parsed, skip to next message
+            if not file_name:
+                pass
+
+            # Figure out event type and F ID by parsing the subject
             f_type, f_id = imap_client.parse_subject(subject)
-                file_name = attachments[subject]
 
-                # Upload file to cloud
+            # Upload extracted attachment to cloud
             webdav_client.upload(file_name, f_id)
 
             # Take actions - depending on the type
@@ -133,6 +139,8 @@ def main():
 
                 if webdav_client.get_lodur_data(f_id):
                     logger.info("[%s] Lodur data already retrieved", f_id)
+                    # Marking message as seen, no need to reprocess again
+                    imap_client.mark_seen(msg, f_id)
                 else:
                     # Retrieve data from Lodur
                     lodur_id = lodur_client.get_einsatzrapport_id(f_id)
@@ -149,9 +157,7 @@ def main():
                         )
 
                         lodur_data = lodur_client.retrieve_form_data(lodur_id)
-                            webdav_client.store_data(
-                                f_id, f_id + "_lodur.json", lodur_data
-                            )
+                        webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
 
                         # upload Alarmdepesche PDF to Lodur
                         lodur_client.upload_alarmdepesche(
@@ -161,9 +167,7 @@ def main():
                         )
 
                         # Marking message as seen, no need to reprocess again
-                            for msg_id in msg_ids:
-                                logger.info("[%s] Marking E-Mail message as seen", f_id)
-                                imap_client.mark_seen(msg_id)
+                        imap_client.mark_seen(msg, f_id)
                     else:
                         logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
 
@@ -208,9 +212,7 @@ def main():
                     )
 
                     # Marking message as seen, no need to reprocess again
-                        for msg_id in msg_ids:
-                            logger.info("[%s] Marking E-Mail message as seen", f_id)
-                            imap_client.mark_seen(msg_id)
+                    imap_client.mark_seen(msg, f_id)
 
                 else:
                     logger.warn(

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pylokid"
-version = "3.0.3"
+version = "3.1.0"
 description = ""
 authors = ["Tobias Brunner <tobias@tobru.ch>"]
 license = "MIT"