diff --git a/.gitignore b/.gitignore
index b342fa5..884bb6e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
__pycache__/
.vscode/
.env
+pylokid/temp_test.py
test.py
\ No newline at end of file
diff --git a/pylokid/__init__.py b/pylokid/__init__.py
index b68dbf1..266310e 100644
--- a/pylokid/__init__.py
+++ b/pylokid/__init__.py
@@ -1,7 +1,3 @@
"""
Pylokid. From Mail to Lodur - all automated.
"""
-
-__version__ = "3.0.2"
-__git_version__ = "0"
-__url__ = "https://github.com/tobru/pylokid"
diff --git a/pylokid/library/emailhandling.py b/pylokid/library/emailhandling.py
index 71b3006..1a2c0ac 100644
--- a/pylokid/library/emailhandling.py
+++ b/pylokid/library/emailhandling.py
@@ -36,6 +36,8 @@ class EmailHandling:
def search_emails(self):
""" searches for emails matching the configured subject """
+ msg_ids = []
+
self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
try:
typ, msg_ids = self.imap.search(
@@ -49,61 +51,84 @@ class EmailHandling:
self.logger.error("IMAP search aborted - exiting: %s", str(err))
raise SystemExit(1)
- num_messages = len(msg_ids[0].split())
- self.logger.info("Found %s matching messages", str(num_messages))
-
- return num_messages, msg_ids
-
- def store_attachments(self, msg_ids):
- """ stores the attachments to filesystem """
-
- data = {}
- for msg_id in msg_ids[0].split():
- # download message from imap
- typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
+ msg_list = msg_ids[0].split()
+ self.logger.info("Found %s matching messages", str(len(msg_list)))
+ # Retrieve subjects
+ msg_id_subject = {}
+ for msg in msg_list:
+ msg_id = msg.decode("utf-8")
+ typ, msg_data = self.imap.fetch(msg, "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
if typ != "OK":
- self.logger.error("Error fetching message")
- continue
+ self.logger.error("Error fetching subject")
+ msg_id_subject[msg_id] = "unknown"
+ else:
+ mail = email.message_from_string(str(msg_data[0][1], "utf-8"))
+ subject = mail["subject"]
+ self.logger.info("Message ID %s has subject '%s'", msg_id, subject)
+ msg_id_subject[msg_id] = subject
- # extract attachment
- for response_part in msg_data:
- if isinstance(response_part, tuple):
- mail = email.message_from_string(str(response_part[1], "utf-8"))
- subject = mail["subject"]
- f_type, f_id = self.parse_subject(subject)
- self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
- for part in mail.walk():
- file_name = part.get_filename()
- if not file_name:
- self.logger.debug(
- "Most probably not an attachment as no filename found"
- )
- continue
+ # Deduplicate messages - usually the same message arrives multiple times
+ self.logger.info("Deduplicating messages")
+ temp = []
+ msg_id_subject_deduplicated = dict()
+ for key, val in msg_id_subject.items():
+ if val not in temp:
+ temp.append(val)
+ msg_id_subject_deduplicated[key] = val
+ self.logger.info(
+ "Adding Message ID %s '%s' to list to process", msg_id, subject
+ )
+ else:
+ self.mark_seen(key, key)
+
+ return msg_id_subject_deduplicated
+
+ def store_attachment(self, msg_id):
+ """ stores the attachment to filesystem """
+
+ # download message from imap
+ typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
+
+ if typ != "OK":
+ self.logger.error("Error fetching message")
+ return None, None
+
+ # extract attachment
+ for response_part in msg_data:
+ if isinstance(response_part, tuple):
+ mail = email.message_from_string(str(response_part[1], "utf-8"))
+ subject = mail["subject"]
+ f_type, f_id = self.parse_subject(subject)
+ self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
+ for part in mail.walk():
+ file_name = part.get_filename()
+ if not file_name:
+ self.logger.debug(
+ "Most probably not an attachment as no filename found"
+ )
+ continue
+
+ self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
+
+ if bool(file_name):
+ f_type, _ = self.parse_subject(subject)
+ renamed_file_name = f_type + "_" + file_name
+ # save attachment to filesystem
+ file_path = os.path.join(self.tmp_dir, renamed_file_name)
self.logger.info(
- '[%s] Extracting attachment "%s"', f_id, file_name
+ '[%s] Saving attachment to "%s"', f_id, file_path
)
+ if not os.path.isfile(file_path):
+ file = open(file_path, "wb")
+ file.write(part.get_payload(decode=True))
+ file.close()
- if bool(file_name):
- f_type, _ = self.parse_subject(subject)
- renamed_file_name = f_type + "_" + file_name
- # save attachment to filesystem
- file_path = os.path.join(self.tmp_dir, renamed_file_name)
+ return renamed_file_name
- self.logger.info(
- '[%s] Saving attachment to "%s"', f_id, file_path
- )
- if not os.path.isfile(file_path):
- file = open(file_path, "wb")
- file.write(part.get_payload(decode=True))
- file.close()
-
- data[subject] = renamed_file_name
-
- return data
-
- def mark_seen(self, msg_id):
+ def mark_seen(self, msg_id, f_id):
+ self.logger.info("[%s] Marking E-Mail message as seen", f_id)
self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
def parse_subject(self, subject):
diff --git a/pylokid/main.py b/pylokid/main.py
index fb1c544..2048a3a 100644
--- a/pylokid/main.py
+++ b/pylokid/main.py
@@ -43,7 +43,7 @@ def main():
# Logging configuration
logging.basicConfig(
level=logging.INFO,
- format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+ format="%(asctime)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger("pylokid")
logger.info("Starting pylokid version %s", version("pylokid"))
@@ -80,170 +80,172 @@ def main():
pdf = PDFParsing()
# Main Loop
+ logger.info("** Starting to process E-Mails **")
while True:
- attachments = {}
- num_messages, msg_ids = imap_client.search_emails()
- if num_messages:
- attachments = imap_client.store_attachments(msg_ids)
- if attachments:
- for subject in attachments:
- f_type, f_id = imap_client.parse_subject(subject)
- file_name = attachments[subject]
+ # Search for matchting E-Mails
+ msg_ids = imap_client.search_emails()
- # Upload file to cloud
- webdav_client.upload(file_name, f_id)
+ for msg, subject in msg_ids.items():
+ logger.info("Processing IMAP message ID %s", msg)
+ file_name = imap_client.store_attachment(msg)
- # Take actions - depending on the type
- if f_type == "Einsatzausdruck_FW":
- logger.info("[%s] Processing type %s", f_id, f_type)
+ # If the message couldn't be parsed, skip to next message
+ if not file_name:
+ pass
- # Check if the PDF isn't already parsed
- if webdav_client.get_lodur_data(f_id, "_pdf.json"):
- logger.info("[%s] PDF already parsed", f_id)
- else:
- # Extract information from PDF
- pdf_data = pdf.extract_einsatzausdruck(
- os.path.join(TMP_DIR, file_name),
- f_id,
- )
+ # Figure out event type and F ID by parsing the subject
+ f_type, f_id = imap_client.parse_subject(subject)
- # publish Einsatz on Pushover
- logger.info("[%s] Publishing message on Pushover", f_id)
- pushover.send_message(
- "{}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
- pdf_data["einsatz"],
- pdf_data["ort"],
- pdf_data["melder"].replace("\n", " "),
- pdf_data["hinweis"],
- pdf_data["sondersignal"],
- pdf_data["bemerkungen"],
- pdf_data["disponierteeinheiten"],
- ),
- title="Feuerwehr Einsatz - {}".format(f_id),
- url="https://www.google.com/maps/search/?api=1&query={}".format(
- pdf_data["ort"]
- ),
- url_title="Ort auf Karte suchen",
- html=1,
- )
+ # Upload extracted attachment to cloud
+ webdav_client.upload(file_name, f_id)
- # Upload extracted data to cloud
- webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data)
+ # Take actions - depending on the type
+ if f_type == "Einsatzausdruck_FW":
+ logger.info("[%s] Processing type %s", f_id, f_type)
- if webdav_client.get_lodur_data(f_id):
- logger.info("[%s] Lodur data already retrieved", f_id)
- else:
- # Retrieve data from Lodur
- lodur_id = lodur_client.get_einsatzrapport_id(f_id)
- if lodur_id:
- logger.info(
- "[%s] Einsatzrapport available in Lodur with ID %s",
- f_id,
- lodur_id,
- )
- logger.info(
- "%s?modul=36&what=144&event=%s&edit=1",
- LODUR_BASE_URL,
- lodur_id,
- )
-
- lodur_data = lodur_client.retrieve_form_data(lodur_id)
- webdav_client.store_data(
- f_id, f_id + "_lodur.json", lodur_data
- )
-
- # upload Alarmdepesche PDF to Lodur
- lodur_client.upload_alarmdepesche(
- f_id,
- os.path.join(TMP_DIR, file_name),
- webdav_client,
- )
-
- # Marking message as seen, no need to reprocess again
- for msg_id in msg_ids:
- logger.info("[%s] Marking E-Mail message as seen", f_id)
- imap_client.mark_seen(msg_id)
- else:
- logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
-
- elif f_type == "Einsatzprotokoll":
-
- lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
- pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json")
- logger.info(
- "[%s] Processing type %s with Lodur ID %s",
+ # Check if the PDF isn't already parsed
+ if webdav_client.get_lodur_data(f_id, "_pdf.json"):
+ logger.info("[%s] PDF already parsed", f_id)
+ else:
+ # Extract information from PDF
+ pdf_data = pdf.extract_einsatzausdruck(
+ os.path.join(TMP_DIR, file_name),
f_id,
- f_type,
- lodur_id,
)
- # Retrieve Lodur data again and store it in Webdav
- lodur_data = lodur_client.retrieve_form_data(lodur_id)
- webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
+ # publish Einsatz on Pushover
+ logger.info("[%s] Publishing message on Pushover", f_id)
+ pushover.send_message(
+ "{}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
+ pdf_data["einsatz"],
+ pdf_data["ort"],
+ pdf_data["melder"].replace("\n", " "),
+ pdf_data["hinweis"],
+ pdf_data["sondersignal"],
+ pdf_data["bemerkungen"],
+ pdf_data["disponierteeinheiten"],
+ ),
+ title="Feuerwehr Einsatz - {}".format(f_id),
+ url="https://www.google.com/maps/search/?api=1&query={}".format(
+ pdf_data["ort"]
+ ),
+ url_title="Ort auf Karte suchen",
+ html=1,
+ )
- if (
- "aut_created_report" in lodur_data
- and lodur_data["aut_created_report"] == "finished"
- ):
- logger.info("[%s] Record in Lodur ready to be updated", f_id)
+ # Upload extracted data to cloud
+ webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data)
- # Upload Einsatzprotokoll to Lodur
+ if webdav_client.get_lodur_data(f_id):
+ logger.info("[%s] Lodur data already retrieved", f_id)
+ # Marking message as seen, no need to reprocess again
+ imap_client.mark_seen(msg, f_id)
+ else:
+ # Retrieve data from Lodur
+ lodur_id = lodur_client.get_einsatzrapport_id(f_id)
+ if lodur_id:
+ logger.info(
+ "[%s] Einsatzrapport available in Lodur with ID %s",
+ f_id,
+ lodur_id,
+ )
+ logger.info(
+ "%s?modul=36&what=144&event=%s&edit=1",
+ LODUR_BASE_URL,
+ lodur_id,
+ )
+
+ lodur_data = lodur_client.retrieve_form_data(lodur_id)
+ webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
+
+ # upload Alarmdepesche PDF to Lodur
lodur_client.upload_alarmdepesche(
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
- # Update entry in Lodur
- lodur_client.einsatzprotokoll(
- f_id, lodur_data, pdf_data, webdav_client
- )
-
- # Einsatz finished - publish on pushover
- logger.info("[%s] Publishing message on Pushover", f_id)
- pushover.send_message(
- "Einsatz beendet",
- title="Feuerwehr Einsatz beendet - {}".format(f_id),
- )
-
# Marking message as seen, no need to reprocess again
- for msg_id in msg_ids:
- logger.info("[%s] Marking E-Mail message as seen", f_id)
- imap_client.mark_seen(msg_id)
-
+ imap_client.mark_seen(msg, f_id)
else:
- logger.warn(
- "[%s] Record in Lodur NOT ready yet to be updated", f_id
- )
+ logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
- # This is usually a scan from the Depot printer
- elif f_type == "Einsatzrapport":
+ elif f_type == "Einsatzprotokoll":
- logger.info("[%s] Processing type %s", f_id, f_type)
+ lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
+ pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json")
+ logger.info(
+ "[%s] Processing type %s with Lodur ID %s",
+ f_id,
+ f_type,
+ lodur_id,
+ )
- # Attach scan in Lodur if f_id is available
- # f_id can be empty when scan was misconfigured
- if f_id != None:
- lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
- # Retrieve Lodur data again and store it in Webdav
- lodur_data = lodur_client.retrieve_form_data(lodur_id)
- webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
- lodur_client.einsatzrapport_scan(
- f_id,
- lodur_data,
- os.path.join(TMP_DIR, file_name),
- webdav_client,
- )
+ # Retrieve Lodur data again and store it in Webdav
+ lodur_data = lodur_client.retrieve_form_data(lodur_id)
+ webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
- logger.info("[%s] Publishing message on Pushover", f_id)
+ if (
+ "aut_created_report" in lodur_data
+ and lodur_data["aut_created_report"] == "finished"
+ ):
+ logger.info("[%s] Record in Lodur ready to be updated", f_id)
- pushover.send_message(
- "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
- title="Feuerwehr Scan bearbeitet - {}".format(f_id),
+ # Upload Einsatzprotokoll to Lodur
+ lodur_client.upload_alarmdepesche(
+ f_id,
+ os.path.join(TMP_DIR, file_name),
+ webdav_client,
)
+
+ # Update entry in Lodur
+ lodur_client.einsatzprotokoll(
+ f_id, lodur_data, pdf_data, webdav_client
+ )
+
+ # Einsatz finished - publish on pushover
+ logger.info("[%s] Publishing message on Pushover", f_id)
+ pushover.send_message(
+ "Einsatz beendet",
+ title="Feuerwehr Einsatz beendet - {}".format(f_id),
+ )
+
+ # Marking message as seen, no need to reprocess again
+ imap_client.mark_seen(msg, f_id)
+
else:
- logger.error("[%s] Unknown type: %s", f_id, f_type)
+ logger.warn(
+ "[%s] Record in Lodur NOT ready yet to be updated", f_id
+ )
+
+ # This is usually a scan from the Depot printer
+ elif f_type == "Einsatzrapport":
+
+ logger.info("[%s] Processing type %s", f_id, f_type)
+
+ # Attach scan in Lodur if f_id is available
+ # f_id can be empty when scan was misconfigured
+ if f_id != None:
+ lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
+ # Retrieve Lodur data again and store it in Webdav
+ lodur_data = lodur_client.retrieve_form_data(lodur_id)
+ webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
+ lodur_client.einsatzrapport_scan(
+ f_id,
+ lodur_data,
+ os.path.join(TMP_DIR, file_name),
+ webdav_client,
+ )
+
+ logger.info("[%s] Publishing message on Pushover", f_id)
+
+ pushover.send_message(
+ "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
+ title="Feuerwehr Scan bearbeitet - {}".format(f_id),
+ )
+ else:
+ logger.error("[%s] Unknown type: %s", f_id, f_type)
# send heartbeat
requests.get(HEARTBEAT_URL)
diff --git a/pyproject.toml b/pyproject.toml
index a2f1ed8..2bbd35b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "pylokid"
-version = "3.0.3"
+version = "3.1.0"
description = ""
authors = ["Tobias Brunner "]
license = "MIT"