Browse Source

rewrite email handling

master v3.1.0
Tobias Brunner 2 months ago
parent
commit
5f8d2a7109
  1. 1
      .gitignore
  2. 4
      pylokid/__init__.py
  3. 119
      pylokid/library/emailhandling.py
  4. 282
      pylokid/main.py
  5. 2
      pyproject.toml

1
.gitignore

@ -1,4 +1,5 @@
__pycache__/
.vscode/
.env
pylokid/temp_test.py
test.py

4
pylokid/__init__.py

@ -1,7 +1,3 @@
"""
Pylokid. From Mail to Lodur - all automated.
"""
__version__ = "3.0.2"
__git_version__ = "0"
__url__ = "https://github.com/tobru/pylokid"

119
pylokid/library/emailhandling.py

@ -36,6 +36,8 @@ class EmailHandling:
def search_emails(self):
""" searches for emails matching the configured subject """
msg_ids = []
self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
try:
typ, msg_ids = self.imap.search(
@ -49,61 +51,84 @@ class EmailHandling:
self.logger.error("IMAP search aborted - exiting: %s", str(err))
raise SystemExit(1)
num_messages = len(msg_ids[0].split())
self.logger.info("Found %s matching messages", str(num_messages))
return num_messages, msg_ids
def store_attachments(self, msg_ids):
""" stores the attachments to filesystem """
data = {}
for msg_id in msg_ids[0].split():
# download message from imap
typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
msg_list = msg_ids[0].split()
self.logger.info("Found %s matching messages", str(len(msg_list)))
# Retrieve subjects
msg_id_subject = {}
for msg in msg_list:
msg_id = msg.decode("utf-8")
typ, msg_data = self.imap.fetch(msg, "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
if typ != "OK":
self.logger.error("Error fetching message")
continue
# extract attachment
for response_part in msg_data:
if isinstance(response_part, tuple):
mail = email.message_from_string(str(response_part[1], "utf-8"))
subject = mail["subject"]
f_type, f_id = self.parse_subject(subject)
self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
for part in mail.walk():
file_name = part.get_filename()
if not file_name:
self.logger.debug(
"Most probably not an attachment as no filename found"
)
continue
self.logger.info(
'[%s] Extracting attachment "%s"', f_id, file_name
self.logger.error("Error fetching subject")
msg_id_subject[msg_id] = "unknown"
else:
mail = email.message_from_string(str(msg_data[0][1], "utf-8"))
subject = mail["subject"]
self.logger.info("Message ID %s has subject '%s'", msg_id, subject)
msg_id_subject[msg_id] = subject
# Deduplicate messages - usually the same message arrives multiple times
self.logger.info("Deduplicating messages")
temp = []
msg_id_subject_deduplicated = dict()
for key, val in msg_id_subject.items():
if val not in temp:
temp.append(val)
msg_id_subject_deduplicated[key] = val
self.logger.info(
"Adding Message ID %s '%s' to list to process", msg_id, subject
)
else:
self.mark_seen(key, key)
return msg_id_subject_deduplicated
def store_attachment(self, msg_id):
""" stores the attachment to filesystem """
# download message from imap
typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
if typ != "OK":
self.logger.error("Error fetching message")
return None, None
# extract attachment
for response_part in msg_data:
if isinstance(response_part, tuple):
mail = email.message_from_string(str(response_part[1], "utf-8"))
subject = mail["subject"]
f_type, f_id = self.parse_subject(subject)
self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
for part in mail.walk():
file_name = part.get_filename()
if not file_name:
self.logger.debug(
"Most probably not an attachment as no filename found"
)
continue
if bool(file_name):
f_type, _ = self.parse_subject(subject)
renamed_file_name = f_type + "_" + file_name
# save attachment to filesystem
file_path = os.path.join(self.tmp_dir, renamed_file_name)
self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
self.logger.info(
'[%s] Saving attachment to "%s"', f_id, file_path
)
if not os.path.isfile(file_path):
file = open(file_path, "wb")
file.write(part.get_payload(decode=True))
file.close()
if bool(file_name):
f_type, _ = self.parse_subject(subject)
renamed_file_name = f_type + "_" + file_name
# save attachment to filesystem
file_path = os.path.join(self.tmp_dir, renamed_file_name)
data[subject] = renamed_file_name
self.logger.info(
'[%s] Saving attachment to "%s"', f_id, file_path
)
if not os.path.isfile(file_path):
file = open(file_path, "wb")
file.write(part.get_payload(decode=True))
file.close()
return data
return renamed_file_name
def mark_seen(self, msg_id):
def mark_seen(self, msg_id, f_id):
self.logger.info("[%s] Marking E-Mail message as seen", f_id)
self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
def parse_subject(self, subject):

282
pylokid/main.py

@ -43,7 +43,7 @@ def main():
# Logging configuration
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
format="%(asctime)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger("pylokid")
logger.info("Starting pylokid version %s", version("pylokid"))
@ -80,170 +80,172 @@ def main():
pdf = PDFParsing()
# Main Loop
logger.info("** Starting to process E-Mails **")
while True:
attachments = {}
num_messages, msg_ids = imap_client.search_emails()
if num_messages:
attachments = imap_client.store_attachments(msg_ids)
if attachments:
for subject in attachments:
f_type, f_id = imap_client.parse_subject(subject)
file_name = attachments[subject]
# Upload file to cloud
webdav_client.upload(file_name, f_id)
# Take actions - depending on the type
if f_type == "Einsatzausdruck_FW":
logger.info("[%s] Processing type %s", f_id, f_type)
# Check if the PDF isn't already parsed
if webdav_client.get_lodur_data(f_id, "_pdf.json"):
logger.info("[%s] PDF already parsed", f_id)
else:
# Extract information from PDF
pdf_data = pdf.extract_einsatzausdruck(
os.path.join(TMP_DIR, file_name),
f_id,
)
# publish Einsatz on Pushover
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"<b>{}</b>\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
pdf_data["einsatz"],
pdf_data["ort"],
pdf_data["melder"].replace("\n", " "),
pdf_data["hinweis"],
pdf_data["sondersignal"],
pdf_data["bemerkungen"],
pdf_data["disponierteeinheiten"],
),
title="Feuerwehr Einsatz - {}".format(f_id),
url="https://www.google.com/maps/search/?api=1&query={}".format(
pdf_data["ort"]
),
url_title="Ort auf Karte suchen",
html=1,
)
# Search for matchting E-Mails
msg_ids = imap_client.search_emails()
# Upload extracted data to cloud
webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data)
for msg, subject in msg_ids.items():
logger.info("Processing IMAP message ID %s", msg)
file_name = imap_client.store_attachment(msg)
if webdav_client.get_lodur_data(f_id):
logger.info("[%s] Lodur data already retrieved", f_id)
else:
# Retrieve data from Lodur
lodur_id = lodur_client.get_einsatzrapport_id(f_id)
if lodur_id:
logger.info(
"[%s] Einsatzrapport available in Lodur with ID %s",
f_id,
lodur_id,
)
logger.info(
"%s?modul=36&what=144&event=%s&edit=1",
LODUR_BASE_URL,
lodur_id,
)
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(
f_id, f_id + "_lodur.json", lodur_data
)
# upload Alarmdepesche PDF to Lodur
lodur_client.upload_alarmdepesche(
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
# Marking message as seen, no need to reprocess again
for msg_id in msg_ids:
logger.info("[%s] Marking E-Mail message as seen", f_id)
imap_client.mark_seen(msg_id)
else:
logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
elif f_type == "Einsatzprotokoll":
# If the message couldn't be parsed, skip to next message
if not file_name:
pass
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json")
logger.info(
"[%s] Processing type %s with Lodur ID %s",
# Figure out event type and F ID by parsing the subject
f_type, f_id = imap_client.parse_subject(subject)
# Upload extracted attachment to cloud
webdav_client.upload(file_name, f_id)
# Take actions - depending on the type
if f_type == "Einsatzausdruck_FW":
logger.info("[%s] Processing type %s", f_id, f_type)
# Check if the PDF isn't already parsed
if webdav_client.get_lodur_data(f_id, "_pdf.json"):
logger.info("[%s] PDF already parsed", f_id)
else:
# Extract information from PDF
pdf_data = pdf.extract_einsatzausdruck(
os.path.join(TMP_DIR, file_name),
f_id,
f_type,
lodur_id,
)
# Retrieve Lodur data again and store it in Webdav
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
# publish Einsatz on Pushover
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"<b>{}</b>\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
pdf_data["einsatz"],
pdf_data["ort"],
pdf_data["melder"].replace("\n", " "),
pdf_data["hinweis"],
pdf_data["sondersignal"],
pdf_data["bemerkungen"],
pdf_data["disponierteeinheiten"],
),
title="Feuerwehr Einsatz - {}".format(f_id),
url="https://www.google.com/maps/search/?api=1&query={}".format(
pdf_data["ort"]
),
url_title="Ort auf Karte suchen",
html=1,
)
if (
"aut_created_report" in lodur_data
and lodur_data["aut_created_report"] == "finished"
):
logger.info("[%s] Record in Lodur ready to be updated", f_id)
# Upload extracted data to cloud
webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data)
# Upload Einsatzprotokoll to Lodur
lodur_client.upload_alarmdepesche(
if webdav_client.get_lodur_data(f_id):
logger.info("[%s] Lodur data already retrieved", f_id)
# Marking message as seen, no need to reprocess again
imap_client.mark_seen(msg, f_id)
else:
# Retrieve data from Lodur
lodur_id = lodur_client.get_einsatzrapport_id(f_id)
if lodur_id:
logger.info(
"[%s] Einsatzrapport available in Lodur with ID %s",
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
lodur_id,
)
# Update entry in Lodur
lodur_client.einsatzprotokoll(
f_id, lodur_data, pdf_data, webdav_client
)
# Einsatz finished - publish on pushover
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"Einsatz beendet",
title="Feuerwehr Einsatz beendet - {}".format(f_id),
logger.info(
"%s?modul=36&what=144&event=%s&edit=1",
LODUR_BASE_URL,
lodur_id,
)
# Marking message as seen, no need to reprocess again
for msg_id in msg_ids:
logger.info("[%s] Marking E-Mail message as seen", f_id)
imap_client.mark_seen(msg_id)
else:
logger.warn(
"[%s] Record in Lodur NOT ready yet to be updated", f_id
)
# This is usually a scan from the Depot printer
elif f_type == "Einsatzrapport":
logger.info("[%s] Processing type %s", f_id, f_type)
# Attach scan in Lodur if f_id is available
# f_id can be empty when scan was misconfigured
if f_id != None:
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
# Retrieve Lodur data again and store it in Webdav
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
lodur_client.einsatzrapport_scan(
# upload Alarmdepesche PDF to Lodur
lodur_client.upload_alarmdepesche(
f_id,
lodur_data,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
logger.info("[%s] Publishing message on Pushover", f_id)
# Marking message as seen, no need to reprocess again
imap_client.mark_seen(msg, f_id)
else:
logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
elif f_type == "Einsatzprotokoll":
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json")
logger.info(
"[%s] Processing type %s with Lodur ID %s",
f_id,
f_type,
lodur_id,
)
# Retrieve Lodur data again and store it in Webdav
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
if (
"aut_created_report" in lodur_data
and lodur_data["aut_created_report"] == "finished"
):
logger.info("[%s] Record in Lodur ready to be updated", f_id)
# Upload Einsatzprotokoll to Lodur
lodur_client.upload_alarmdepesche(
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
# Update entry in Lodur
lodur_client.einsatzprotokoll(
f_id, lodur_data, pdf_data, webdav_client
)
# Einsatz finished - publish on pushover
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
title="Feuerwehr Scan bearbeitet - {}".format(f_id),
"Einsatz beendet",
title="Feuerwehr Einsatz beendet - {}".format(f_id),
)
# Marking message as seen, no need to reprocess again
imap_client.mark_seen(msg, f_id)
else:
logger.error("[%s] Unknown type: %s", f_id, f_type)
logger.warn(
"[%s] Record in Lodur NOT ready yet to be updated", f_id
)
# This is usually a scan from the Depot printer
elif f_type == "Einsatzrapport":
logger.info("[%s] Processing type %s", f_id, f_type)
# Attach scan in Lodur if f_id is available
# f_id can be empty when scan was misconfigured
if f_id != None:
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
# Retrieve Lodur data again and store it in Webdav
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
lodur_client.einsatzrapport_scan(
f_id,
lodur_data,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
title="Feuerwehr Scan bearbeitet - {}".format(f_id),
)
else:
logger.error("[%s] Unknown type: %s", f_id, f_type)
# send heartbeat
requests.get(HEARTBEAT_URL)

2
pyproject.toml

@ -1,6 +1,6 @@
[tool.poetry]
name = "pylokid"
version = "3.0.3"
version = "3.1.0"
description = ""
authors = ["Tobias Brunner <tobias@tobru.ch>"]
license = "MIT"

Loading…
Cancel
Save