add pdftk hint

By using BODY.PEEK[] messages are not implicitely marked seen which
allows to set them seen when it could be processed. Might help to
reiterate over mails until the record is ready in Lodur.

.drone.yml

@@ -1,4 +1,5 @@
 kind: pipeline
+type: kubernetes
 name: default
 
 steps:

.gitignore

@@ -1,4 +1,6 @@
 __pycache__/
 .vscode/
 .env
+pylokid/temp_test.py
 test.py
+tmp/

Dockerfile

@@ -39,13 +39,15 @@ COPY --from=builder \
       /app/dist /app/dist
 RUN pip install /app/dist/pylokid-*-py3-none-any.whl
 
-COPY hack/patches/*.patch /tmp
+COPY hack/patches/*.patch /tmp/
 
 # The ugliest possible way to workaround https://github.com/MechanicalSoup/MechanicalSoup/issues/356
 # For some unknown reasons Lodur now wants "Content-Type: application/pdf" set in the multipart
 # data section. And as I couln't figure out yet how to do that in MechanicalSoup and I only upload PDFs
 # I just patch it to hardcode it. YOLO
-RUN patch -p0 /usr/local/lib/python3.9/site-packages/mechanicalsoup/browser.py < /tmp/mechsoup-browser-content-type.patch
+RUN \
+  patch -p0 /usr/local/lib/python3.9/site-packages/mechanicalsoup/browser.py < /tmp/mechsoup-browser-content-type.patch && \
+  patch -p0 /usr/local/lib/python3.9/site-packages/mechanicalsoup/stateful_browser.py < /tmp/mechsoup-link-regex.patch
 
 ## ----------- Step 4
 FROM base AS runtime

hack/generate-einsatrapporte.py

@@ -0,0 +1,54 @@
+import uno
+from com.sun.star.beans import PropertyValue
+from com.sun.star.uno import RuntimeException
+
+def connect_to_libreoffice(port=2002):
+    localContext = uno.getComponentContext()
+    resolver = localContext.ServiceManager.createInstanceWithContext(
+        "com.sun.star.bridge.UnoUrlResolver", localContext)
+    try:
+        context = resolver.resolve(f"uno:socket,host=localhost,port={port};urp;StarOffice.ComponentContext")
+        return context.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", context)
+    except RuntimeException:
+        raise Exception("Make sure LibreOffice is running with a listening port (e.g., soffice --accept=\"socket,port=2002;urp;\" --norestore --nologo --nodefault)")
+
+def export_to_pdf(doc, file_name):
+    pdf_export_properties = tuple([
+        PropertyValue(Name="FilterName", Value="writer_pdf_Export")
+    ])
+    doc.storeToURL(f"file:///{file_name}", pdf_export_properties)
+
+def replace_text(doc, old_text, new_text):
+    replaceable = doc.createReplaceDescriptor()
+    replaceable.setSearchString(old_text)
+    replaceable.setReplaceString(new_text)
+
+    doc.replaceAll(replaceable)
+
+
+
+def main(start_counter, num_documents, document_path):
+    desktop = connect_to_libreoffice()
+
+    for i in range(num_documents):
+        counter = start_counter + i
+        load_props = PropertyValue(Name="Hidden", Value=True),
+        doc = desktop.loadComponentFromURL(f"file:///{document_path}", "_blank", 0, load_props)
+
+        replace_text(doc, "counter", str(counter))
+
+        file_name = f"/home/tobru/Documents/Feuerwehr/Stab/Fourier/EinsatzrapporteLeer/Einsatzrapport2024-{counter}.pdf"
+        export_to_pdf(doc, file_name)
+        doc.dispose()
+
+if __name__ == "__main__":
+    document_path = "/home/tobru/Documents/Feuerwehr/Stab/Fourier/FW-Einsatzrapport FWU.odt"
+    start_counter = 1
+    num_documents = 100
+
+    # Start libreoffice with:
+    # soffice --accept="socket,port=2002;urp;" --norestore --nologo --nodefault
+    main(start_counter, num_documents, document_path)
+
+    # after generation, run:
+    # pdftk $(ls -v *.pdf) cat output Einsatzrapporte2024.pdf

hack/patches/mechsoup-link-regex.patch

@@ -0,0 +1,21 @@
+--- /home/tobru/.cache/pypoetry/virtualenvs/pylokid-JpqZeVMm-py3.9/lib/python3.9/site-packages/mechanicalsoup/stateful_browser.py.orig	2021-02-27 14:21:53.979582533 +0100
++++ /home/tobru/.cache/pypoetry/virtualenvs/pylokid-JpqZeVMm-py3.9/lib/python3.9/site-packages/mechanicalsoup/stateful_browser.py	2021-02-27 14:23:17.680374365 +0100
+@@ -259,7 +259,7 @@
+         for link in self.links(*args, **kwargs):
+             print("    ", link)
+ 
+-    def links(self, url_regex=None, link_text=None, *args, **kwargs):
++    def links(self, url_regex=None, link_text=None, link_regex=None, *args, **kwargs):
+         """Return links in the page, as a list of bs4.element.Tag objects.
+ 
+         To return links matching specific criteria, specify ``url_regex``
+@@ -276,6 +276,9 @@
+         if link_text is not None:
+             all_links = [a for a in all_links
+                          if a.text == link_text]
++        if link_regex is not None:
++            all_links = [a for a in all_links
++                         if re.search(link_regex, a.text)]
+         return all_links
+ 
+     def find_link(self, *args, **kwargs):

pylokid/__init__.py

@@ -1,7 +1,3 @@
 """
 Pylokid. From Mail to Lodur - all automated.
 """
-
-__version__ = "3.0.1"
-__git_version__ = "0"
-__url__ = "https://github.com/tobru/pylokid"

pylokid/library/emailhandling.py

@@ -36,6 +36,8 @@ class EmailHandling:
     def search_emails(self):
         """searches for emails matching the configured subject"""
 
+        msg_ids = []
+
         self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
         try:
             typ, msg_ids = self.imap.search(
@@ -49,22 +51,51 @@ class EmailHandling:
             self.logger.error("IMAP search aborted - exiting: %s", str(err))
             raise SystemExit(1)
 
-        num_messages = len(msg_ids[0].split())
-        self.logger.info("Found %s matching messages", str(num_messages))
+        msg_list = msg_ids[0].split()
+        self.logger.info("Found %s matching messages", str(len(msg_list)))
 
-        return num_messages, msg_ids
+        # Retrieve subjects
+        msg_id_subject = {}
+        for msg in msg_list:
+            msg_id = msg.decode("utf-8")
+            typ, msg_data = self.imap.fetch(msg, "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
+            if typ != "OK":
+                self.logger.error("Error fetching subject")
+                msg_id_subject[msg_id] = "unknown"
+            else:
+                try:
+                    mail = email.message_from_string(str(msg_data[0][1], "utf-8"))
+                    subject = mail["subject"]
+                    self.logger.info("Message ID %s has subject '%s'", msg_id, subject)
+                    msg_id_subject[msg_id] = subject
+                except TypeError:
+                    self.logger.error("Could not decode mail - %s", msg_data[0][1])
 
-    def store_attachments(self, msg_ids):
-        """ stores the attachments to filesystem """
+        # Deduplicate messages - usually the same message arrives multiple times
+        self.logger.info("Deduplicating messages")
+        temp = []
+        msg_id_subject_deduplicated = dict()
+        for key, val in msg_id_subject.items():
+            if val not in temp:
+                temp.append(val)
+                msg_id_subject_deduplicated[key] = val
+                self.logger.info(
+                    "Adding Message ID %s '%s' to list to process", key, val
+                )
+            else:
+                self.mark_seen(key, key)
+
+        return msg_id_subject_deduplicated
+
+    def store_attachment(self, msg_id):
+        """stores the attachment to filesystem"""
 
-        data = {}
-        for msg_id in msg_ids[0].split():
         # download message from imap
-            typ, msg_data = self.imap.fetch(msg_id, "(RFC822)")
+        typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
 
         if typ != "OK":
             self.logger.error("Error fetching message")
-                continue
+            return None, None
 
         # extract attachment
         for response_part in msg_data:
@@ -81,9 +112,7 @@ class EmailHandling:
                         )
                         continue
 
-                        self.logger.info(
-                            '[%s] Extracting attachment "%s"', f_id, file_name
-                        )
+                    self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
 
                     if bool(file_name):
                         f_type, _ = self.parse_subject(subject)
@@ -99,11 +128,10 @@ class EmailHandling:
                             file.write(part.get_payload(decode=True))
                             file.close()
 
-                            data[subject] = renamed_file_name
+        return renamed_file_name
 
-        return data
-
-    def mark_seen(self, msg_id):
+    def mark_seen(self, msg_id, f_id):
+        self.logger.info("[%s] Marking E-Mail message as seen", f_id)
         self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
 
     def parse_subject(self, subject):

pylokid/library/lodur.py

@@ -127,7 +127,7 @@ class Lodur:
         else:
             return None
 
-    def einsatzprotokoll(self, f_id, lodur_data, webdav_client):
+    def einsatzprotokoll(self, f_id, lodur_data, pdf_data, webdav_client):
         """ Prepare Einsatzprotokoll to be sent to Lodur """
 
         self.logger.info("[%s] Updating Lodur entry", f_id)
@@ -138,9 +138,9 @@ class Lodur:
         lodur_data["ztb_m"] = lodur_data[
             "ztv_m"
         ]  # 05. Zeit (copy minute from start to round up to 1h)
-        lodur_data["eins_ereig"] = "{} - {} - {}".format(
-            f_id, lodur_data["ala_stich"], lodur_data["adr"]
-        )  # 07. Ereignis
+        lodur_data[
+            "eins_ereig"
+        ] = f"{f_id} - {pdf_data['einsatz']} - {lodur_data['adr']}"  # 07. Ereignis
         lodur_data["en_kr_feuwehr"] = "1"  # 21. Einsatzkräfte
         lodur_data["ali_io"] = "1"  # 24. Alarmierung
         lodur_data["keyword_els_zutreffend"] = "1"  # 25. Stichwort
@@ -192,7 +192,7 @@ class Lodur:
         self.submit_form_einsatzrapport(lodur_data)
 
         # Upload scan to Alarmdepesche
-        self.einsatzrapport_alarmdepesche(
+        self.upload_alarmdepesche(
             f_id,
             file_path,
             webdav_client,

pylokid/library/webdav.py

@@ -4,6 +4,7 @@
 
 import os
 import json
+import re
 from datetime import datetime
 import logging
 import asyncio
@@ -63,6 +64,10 @@ class WebDav:
             )
             self.logger.info('[%s] File "%s" uploaded', f_id, file_name)
 
+    def delete(self, file_name):
+        """delete file on webdav"""
+        self.loop.run_until_complete(self.webdav.delete(file_name))
+
     def einsatz_exists(self, f_id):
         """check if an einsatz is already created"""
 
@@ -75,6 +80,31 @@ class WebDav:
         else:
             return False
 
+    def einsatzrapport_inbox_check(self, tmp_dir):
+        """check if an einsatzrapport with an f_id exists in the WebDav Inbox and process it"""
+
+        rapporte_to_process = []
+
+        filelist = self.loop.run_until_complete(
+            self.webdav.ls(f"{self.webdav_basedir}/Inbox")
+        )
+        for file in filelist:
+            full_path = file[0]
+            parsed = re.search(".*Einsatzrapport_(F[0-9].*)\.pdf", full_path)
+            if parsed:
+                f_id = parsed.group(1)
+                self.logger.info("[%s] Found %s - Downloading", f_id, full_path)
+
+                # Download PDF for later processing
+                self.loop.run_until_complete(
+                    self.webdav.download(
+                        full_path, f"{tmp_dir}/Einsatzrapport_{f_id}.pdf"
+                    )
+                )
+                rapporte_to_process.append(f_id)
+
+        return rapporte_to_process
+
     def store_data(self, f_id, file_name, data):
         """stores data on webdav"""
 

pylokid/main.py

@@ -43,7 +43,7 @@ def main():
     # Logging configuration
     logging.basicConfig(
         level=logging.INFO,
-        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        format="%(asctime)s - %(levelname)s - %(message)s",
     )
     logger = logging.getLogger("pylokid")
     logger.info("Starting pylokid version %s", version("pylokid"))
@@ -80,18 +80,24 @@ def main():
     pdf = PDFParsing()
 
     # Main Loop
+    logger.info("** Starting to process E-Mails **")
     while True:
-        attachments = {}
-        num_messages, msg_ids = imap_client.search_emails()
-        if num_messages:
-            attachments = imap_client.store_attachments(msg_ids)
 
-        if attachments:
-            for subject in attachments:
+        # Search for matchting E-Mails
+        msg_ids = imap_client.search_emails()
+
+        for msg, subject in msg_ids.items():
+            logger.info("Processing IMAP message ID %s", msg)
+            file_name = imap_client.store_attachment(msg)
+
+            # If the message couldn't be parsed, skip to next message
+            if not file_name:
+                pass
+
+            # Figure out event type and F ID by parsing the subject
             f_type, f_id = imap_client.parse_subject(subject)
-                file_name = attachments[subject]
 
-                # Upload file to cloud
+            # Upload extracted attachment to cloud
             webdav_client.upload(file_name, f_id)
 
             # Take actions - depending on the type
@@ -133,6 +139,8 @@ def main():
 
                 if webdav_client.get_lodur_data(f_id):
                     logger.info("[%s] Lodur data already retrieved", f_id)
+                    # Marking message as seen, no need to reprocess again
+                    imap_client.mark_seen(msg, f_id)
                 else:
                     # Retrieve data from Lodur
                     lodur_id = lodur_client.get_einsatzrapport_id(f_id)
@@ -149,9 +157,7 @@ def main():
                         )
 
                         lodur_data = lodur_client.retrieve_form_data(lodur_id)
-                            webdav_client.store_data(
-                                f_id, f_id + "_lodur.json", lodur_data
-                            )
+                        webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
 
                         # upload Alarmdepesche PDF to Lodur
                         lodur_client.upload_alarmdepesche(
@@ -161,15 +167,14 @@ def main():
                         )
 
                         # Marking message as seen, no need to reprocess again
-                            for msg_id in msg_ids:
-                                logger.info("[%s] Marking E-Mail message as seen", f_id)
-                                imap_client.mark_seen(msg_id)
+                        imap_client.mark_seen(msg, f_id)
                     else:
                         logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
 
             elif f_type == "Einsatzprotokoll":
 
                 lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
+                pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json")
                 logger.info(
                     "[%s] Processing type %s with Lodur ID %s",
                     f_id,
@@ -195,19 +200,26 @@ def main():
                     )
 
                     # Update entry in Lodur
-                        lodur_client.einsatzprotokoll(f_id, lodur_data, webdav_client)
+                    lodur_client.einsatzprotokoll(
+                        f_id, lodur_data, pdf_data, webdav_client
+                    )
+
+                    # Correct time on AdF to round up to one hour
+                    # curl 'https://lodur-zh.ch/urdorf/tunnel.php?modul=36&what=1082'
+                    # -H 'Referer: https://lodur-zh.ch/urdorf/index.php?modul=36&what=145&event=3485&edit=1'
+                    # -H 'Cookie: PHPSESSID=85pnahp3q83apv7qsbi8hrj5g7'
+                    # --data-raw 'dtv_d=03&dtv_m=04&dtv_y=2021&dtb_d=03&dtb_m=04&dtb_y=2021&ztv_h=18&ztv_m=26&ztb_h=19&ztb_m=26'
+                    # ztb_m -> same as ztv_m
 
                     # Einsatz finished - publish on pushover
                     logger.info("[%s] Publishing message on Pushover", f_id)
                     pushover.send_message(
-                            "Einsatz {} beendet".format(f_id),
+                        "Einsatz beendet",
                         title="Feuerwehr Einsatz beendet - {}".format(f_id),
                     )
 
                     # Marking message as seen, no need to reprocess again
-                        for msg_id in msg_ids:
-                            logger.info("[%s] Marking E-Mail message as seen", f_id)
-                            imap_client.mark_seen(msg_id)
+                    imap_client.mark_seen(msg, f_id)
 
                 else:
                     logger.warn(
@@ -221,7 +233,7 @@ def main():
 
                 # Attach scan in Lodur if f_id is available
                 # f_id can be empty when scan was misconfigured
-                    if f_id != None:
+                if f_id != None and len(f_id) > 8:
                     lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
                     # Retrieve Lodur data again and store it in Webdav
                     lodur_data = lodur_client.retrieve_form_data(lodur_id)
@@ -232,6 +244,8 @@ def main():
                         os.path.join(TMP_DIR, file_name),
                         webdav_client,
                     )
+                else:
+                    f_id = f_type
 
                 logger.info("[%s] Publishing message on Pushover", f_id)
 
@@ -239,9 +253,46 @@ def main():
                     "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
                     title="Feuerwehr Scan bearbeitet - {}".format(f_id),
                 )
+
+                # Marking message as seen, no need to reprocess again
+                imap_client.mark_seen(msg, f_id)
+
             else:
                 logger.error("[%s] Unknown type: %s", f_id, f_type)
 
+        logger.info("Checking WebDav Inbox folder for Einsatzrapporte to process")
+        rapporte_to_process = webdav_client.einsatzrapport_inbox_check(TMP_DIR)
+
+        if rapporte_to_process:
+            for f_id_rapport in rapporte_to_process:
+                filename = f"Einsatzrapport_{f_id_rapport}.pdf"
+                local_file = os.path.join(TMP_DIR, filename)
+                # Upload to f_id folder
+                webdav_client.upload(filename, f_id_rapport)
+
+                # Process it for Lodur
+                lodur_id = webdav_client.get_lodur_data(f_id_rapport)["event_id"]
+                # Retrieve Lodur data again and store it in Webdav
+                lodur_data = lodur_client.retrieve_form_data(lodur_id)
+                webdav_client.store_data(
+                    f_id_rapport, f_id_rapport + "_lodur.json", lodur_data
+                )
+                lodur_client.einsatzrapport_scan(
+                    f_id_rapport,
+                    lodur_data,
+                    local_file,
+                    webdav_client,
+                )
+                # Delete processed Einsatzrapport from Inbox and local temp dir
+                logger.info("Einsatzrapport processed - deleting file in Inbox")
+                webdav_client.delete(f"{WEBDAV_BASEDIR}/Inbox/{filename}")
+                os.remove(local_file)
+
+                pushover.send_message(
+                    f"Einsatzrapport {f_id_rapport} wurde bearbeitet.",
+                    title=f"Feuerwehr Einsatzrapport bearbeitet - {f_id_rapport}",
+                )
+
         # send heartbeat
         requests.get(HEARTBEAT_URL)
         # repeat every

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pylokid"
-version = "3.0.1"
+version = "3.2.0"
 description = ""
 authors = ["Tobias Brunner <tobias@tobru.ch>"]
 license = "MIT"