tobru
/
pylokid
1
0
Fork 0
Code Pull Requests Releases Activity

the big refactoring

This commit is contained in:
Tobias Brunner 2017-12-30 19:19:40 +01:00
parent 625b8c7da3
commit 6516045293
8 changed files with 581 additions and 461 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

102
emailhandling.py Normal file
View File

@ -0,0 +1,102 @@
#!/usr/bin/env python3
""" E-Mail / IMAP handling """
import os
import logging
import re
import email
import email.parser
import imaplib
_EMAIL_SUBJECTS = '(OR SUBJECT "Einsatzausdruck_FW" SUBJECT "Einsatzprotokoll" UNSEEN)'
class EmailHandling:
""" Email handling """
def __init__(self, server, username, password, mailbox, tmp_dir):
self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to IMAP server ' + server)
self.tmp_dir = tmp_dir
try:
self.imap = imaplib.IMAP4_SSL(server)
self.imap.login(username, password)
self.imap.select(mailbox, readonly=False)
except Exception as err:
self.logger.error('IMAP connection failed - exiting: ' + str(err))
raise SystemExit(1)
self.logger.info('IMAP connection successfull')
def search_emails(self):
""" searches for emails matching the configured subject """
self.logger.info('Searching for messages matching the subject')
typ, msg_ids = self.imap.search(
None,
_EMAIL_SUBJECTS,
)
if typ != 'OK':
self.logger.error('Error searching for matching messages')
return False
num_messages = len(msg_ids[0].split())
self.logger.info('Found ' + str(num_messages) + ' matching messages')
return num_messages, msg_ids
def store_attachments(self, msg_ids):
""" stores the attachments to filesystem and marks message as read """
data = {}
for msg_id in msg_ids[0].split():
# download message from imap
typ, msg_data = self.imap.fetch(msg_id, '(RFC822)')
if typ != 'OK':
self.logger.error('Error fetching message')
continue
# extract attachment
for response_part in msg_data:
if isinstance(response_part, tuple):
mail = email.message_from_string(str(response_part[1], 'utf-8'))
subject = mail['subject']
self.logger.info('Getting attachment from: ' + subject)
for part in mail.walk():
file_name = part.get_filename()
if not file_name:
self.logger.debug('Most probably not an attachment as no filename found')
continue
self.logger.info('Extracting attachment: ' + file_name)
if bool(file_name):
f_type, _ = self.parse_subject(subject)
renamed_file_name = f_type + '_' + file_name
# save attachment to filesystem
file_path = os.path.join(self.tmp_dir, renamed_file_name)
self.logger.info('Saving attachment to ' + file_path)
if not os.path.isfile(file_path):
file = open(file_path, 'wb')
file.write(part.get_payload(decode=True))
file.close()
data[subject] = renamed_file_name
# mark as seen
self.logger.info('Marking message as seen ' + subject)
self.imap.store(msg_id, '+FLAGS', '(\\Seen)')
return data
def parse_subject(self, subject):
""" extract f id and type from subject """
parsed = re.search('(.*): (F[0-9].*)', subject)
f_type = parsed.group(1)
f_id = parsed.group(2)
return f_type, f_id

149
lodur.py Normal file
View File

@ -0,0 +1,149 @@
#!/usr/bin/env python3
""" Small Lodur Library for the Module 36 - Einsatzrapport """
import re
import logging
from datetime import datetime
import requests
class Lodur:
""" Lodur """
def __init__(self, url, username, password):
self.logger = logging.getLogger(__name__)
self.session = requests.session()
# Authenticate
self.logger.info('Connecting to Lodur')
answer = self.session.post(
url,
data={
'login_member_name': username,
'login_member_pwd': password,
}
)
# When login failed the form has the CSS class error
# This is the only way to tell if it worked or not
if re.search('"login error"', answer.text):
self.logger.fatal('Login to Lodur failed - exiting')
raise SystemExit(1)
else:
self.logger.info('Login to Lodur succeeded')
def create_einsatzrapport(self, f_id, pdf_data):
""" Create a new Einsatzrapport """
params = (
('modul', '36'),
('what', '144'),
('sp', '1'),
('event', ''),
('edit', ''),
('is_herznotfall', ''),
)
# when PDF parsing fails, pdf_data is false. fill with tbd when this happens
if pdf_data:
date = datetime.strptime(
pdf_data['datum'],
'%d.%m.%Y',
)
time = datetime.strptime(
pdf_data['zeit'],
'%H:%M',
)
eins_ereig = pdf_data['einsatz']
adr = pdf_data['strasse'] + ', ' + pdf_data['plzort']
else:
date = datetime.now()
eins_ereig = 'TBD'
adr = 'TBD'
data = {
'e_r_num': (None, f_id), # 01. Einsatzrapportnummer
'eins_stat_kantone': (None, '1'), # 02. Einsatzart FKS
'emergency_concept_id': (None, '2'), # 03. Verrechnungsart
'ver_sart': (None, 'ab'), # 03. Verrechnungsart internal: ab, th, uh, ak, tt
'dtv_d': (None, str(date.day)), # 04. Datum von
'dtv_m': (None, str(date.month)), # 04. Datum von
'dtv_y': (None, str(date.year)), # 04. Datum von
'dtb_d': (None, str(date.day)), # 04. Datum bis - we dont know yet the end date
'dtb_m': (None, str(date.month)), # 04. Datum bis - assume the same day
'dtb_y': (None, str(date.year)), # 04. Datum bis
'ztv_h': (None, str(time.hour)), # 05. Zeit von
'ztv_m': (None, str(time.minute)), # 05. Zeit von
'ztb_h': (None, str(time.hour + 1)), # 05. Zeit bis - we dont know yet the end time
'ztb_m': (None, str(time.minute)), # 05. Zeit bis - just add one hour and correct later
'e_ort_1': (None, '306'), # 06. Einsatzort: Urdorf 306, Birmensdorf 298
'eins_ereig': (None, eins_ereig.encode('iso-8859-1')), # 07. Ereignis
'adr': (None, adr.encode('iso-8859-1')), # 08. Adresse
#'zh_alarmierung_h': (None, 'UNKNOWN'), # 12. Alarmierung
#'zh_alarmierung_m': (None, 'UNKNOWN'), # 12. Alarmierung
#'zh_fw_ausg_h': (None, 'UNKNOWN'), # 13. FW ausgerückt
#'zh_fw_ausg_m': (None, 'UNKNOWN'), # 13. FW ausgerückt
#'zh_am_schad_h': (None, 'UNKNOWN'), # 14. Am Schadenplatz
#'zh_am_schad_m': (None, 'UNKNOWN'), # 14. Am Schadenplatz
#'zh_fw_einge_h': (None, 'UNKNOWN'), # 15. FW eingerückt
#'zh_fw_einge_m': (None, 'UNKNOWN'), # 15. FW eingerückt
#'eins_erst_h': (None, 'UNKNOWN'), # 16. Einsatzbereitschaft erstellt
#'eins_erst_m': (None, 'UNKNOWN'), # 16. Einsatzbereitschaft erstellt
'ang_sit': (None, 'TBD1'), # 17. Angetroffene Situation
'mn': (None, 'TBD2'), # 19. Massnahmen
'bk': (None, 'TBD3'), # 20. Bemerkungen
'en_kr_feuwehr': (None, '1'), # 21. Einsatzkräfte
'ali_io': (None, '1'), # 24. Alarmierung
'kopie_gvz': (None, '1'), # 31. Kopie innert 10 Tagen an GVZ
'mannschaftd_einsa': (None, '70'), # 32. Einsatzleiter|in
}
# post data to create new einsatzrapport
answer = self.session.post(
'https://lodur-zh.ch/urdorf/index.php',
params=params,
files=data,
)
# very ugly way to find the assigned event id by lodur
# lodur really adds a script element at the bottom of the returned html
# with the location to reload the page - containing the assigned event id
lodur_id = re.search('modul=36&event=([0-9].*)&edit=1&what=144', answer.text).group(1)
return lodur_id
def upload_alarmdepesche(self, lodur_id, file_path):
""" Upload a file to Alarmdepesche """
params = (
('modul', '36'),
('what', '828'),
('event', lodur_id),
)
data = {
'alarmdepesche': open(file_path, 'rb')
}
self.session.post(
'https://lodur-zh.ch/urdorf/index.php',
params=params,
files=data,
)
# TODO this doesnt work. We first have to fetch the current form with its
# data, update the fields we want to change and resubmit the form
def update_einsatzrapport(self, lodur_id, data):
""" Update the Einsatzrapport """
params = (
('modul', '36'),
('what', '144'),
('sp', '1'),
('event', lodur_id),
('edit', '1'),
('is_herznotfall', ''),
)
answer = self.session.post(
'https://lodur-zh.ch/urdorf/index.php',
params=params,
files=data,
)
print(answer.headers)

145
lodur_connect.py
View File

@ -1,145 +0,0 @@
#!/usr/bin/env python3
import re
from datetime import datetime
import requests
def create_einsatzrapport(username, password, base_url, f_id, pdf_data):
session = requests.session()
login_data = {
'login_member_name': username,
'login_member_pwd': password,
}
# Authenticate
session.post(base_url, data=login_data)
params = (
('modul', '36'),
('what', '144'),
('sp', '1'),
('event', ''),
('edit', ''),
('is_herznotfall', ''),
)
# when PDF parsing fails, pdf_data is false. fill with tbd when this happens
if pdf_data:
date = datetime.strptime(
pdf_data['datum'],
'%d.%m.%Y',
)
time = datetime.strptime(
pdf_data['zeit'],
'%H:%M',
)
eins_ereig = pdf_data['einsatz']
adr = pdf_data['strasse'] + ', ' + pdf_data['plzort']
else:
date = datetime.now()
eins_ereig = 'TBD'
adr = 'TBD'
data = {
'e_r_num': (None, f_id), # 01. Einsatzrapportnummer
'eins_stat_kantone': (None, '1'), # 02. Einsatzart FKS
'emergency_concept_id': (None, '2'), # 03. Verrechnungsart
'ver_sart': (None, 'ab'), # 03. Verrechnungsart internal: ab, th, uh, ak, tt
'dtv_d': (None, str(date.day)), # 04. Datum von
'dtv_m': (None, str(date.month)), # 04. Datum von
'dtv_y': (None, str(date.year)), # 04. Datum von
'dtb_d': (None, str(date.day)), # 04. Datum bis - we dont know yet the end date
'dtb_m': (None, str(date.month)), # 04. Datum bis - assume the same day
'dtb_y': (None, str(date.year)), # 04. Datum bis
'ztv_h': (None, str(time.hour)), # 05. Zeit von
'ztv_m': (None, str(time.minute)), # 05. Zeit von
'ztb_h': (None, str(time.hour + 1)), # 05. Zeit bis - we dont know yet the end time
'ztb_m': (None, str(time.minute)), # 05. Zeit bis - just add one hour and correct later
'e_ort_1': (None, '306'), # 06. Einsatzort: Urdorf 306, Birmensdorf 298
'eins_ereig': (None, eins_ereig.encode('iso-8859-1')), # 07. Ereignis
'adr': (None, adr.encode('iso-8859-1')), # 08. Adresse
#'zh_alarmierung_h': (None, 'UNKNOWN'), # 12. Alarmierung
#'zh_alarmierung_m': (None, 'UNKNOWN'), # 12. Alarmierung
#'zh_fw_ausg_h': (None, 'UNKNOWN'), # 13. FW ausgerückt
#'zh_fw_ausg_m': (None, 'UNKNOWN'), # 13. FW ausgerückt
#'zh_am_schad_h': (None, 'UNKNOWN'), # 14. Am Schadenplatz
#'zh_am_schad_m': (None, 'UNKNOWN'), # 14. Am Schadenplatz
#'zh_fw_einge_h': (None, 'UNKNOWN'), # 15. FW eingerückt
#'zh_fw_einge_m': (None, 'UNKNOWN'), # 15. FW eingerückt
#'eins_erst_h': (None, 'UNKNOWN'), # 16. Einsatzbereitschaft erstellt
#'eins_erst_m': (None, 'UNKNOWN'), # 16. Einsatzbereitschaft erstellt
'ang_sit': (None, 'TBD1'), # 17. Angetroffene Situation
'mn': (None, 'TBD2'), # 19. Massnahmen
'bk': (None, 'TBD3'), # 20. Bemerkungen
'en_kr_feuwehr': (None, '1'), # 21. Einsatzkräfte
'ali_io': (None, '1'), # 24. Alarmierung
'kopie_gvz': (None, '1'), # 31. Kopie innert 10 Tagen an GVZ
'mannschaftd_einsa': (None, '70'), # 32. Einsatzleiter|in
}
# post data to create new einsatzrapport
answer = session.post(
'https://lodur-zh.ch/urdorf/index.php',
params=params,
files=data,
)
# very ugly way to find the assigned event id by lodur
# lodur really adds a script element at the bottom of the returned html
# with the location to reload the page - containing the assigned event id
lodur_id = re.search('modul=36&event=([0-9].*)&edit=1&what=144', answer.text).group(1)
return lodur_id
def upload_alarmdepesche(username, password, base_url, lodur_id, file_name, file_path):
session = requests.session()
login_data = {
'login_member_name': username,
'login_member_pwd': password,
}
# Authenticate
session.post(base_url, data=login_data)
params = (
('modul', '36'),
('what', '828'),
('event', lodur_id),
)
data = {'alarmdepesche': open(file_path, 'rb')}
session.post(
'https://lodur-zh.ch/urdorf/index.php',
params=params,
files=data,
)
# TODO this doesnt work. We first have to fetch the current form with its
# data, update the fields we want to change and resubmit the form
def update_einsatzrapport(username, password, base_url, lodur_id, data):
""" Update the Einsatzrapport """
session = requests.session()
login_data = {
'login_member_name': username,
'login_member_pwd': password,
}
# Authenticate
session.post(base_url, data=login_data)
params = (
('modul', '36'),
('what', '144'),
('sp', '1'),
('event', lodur_id),
('edit', '1'),
('is_herznotfall', ''),
)
answer = session.post(
'https://lodur-zh.ch/urdorf/index.php',
params=params,
files=data,
)
print(answer.headers)

326
main.py
View File

@ -2,289 +2,152 @@
""" Thy pylokid main program """
import os
import re
from datetime import datetime
import asyncio
import logging
import os
import time
import requests
import email
import email.parser
import imaplib
import aioeasywebdav
from dotenv import load_dotenv, find_dotenv
import paho.mqtt.client as mqtt
from lodur_connect import create_einsatzrapport, upload_alarmdepesche, update_einsatzrapport
import pdf_extract
_EMAIL_SUBJECTS = '(OR SUBJECT "Einsatzausdruck_FW" SUBJECT "Einsatzprotokoll" UNSEEN)'
import requests
from dotenv import find_dotenv, load_dotenv
# local classes
from emailhandling import EmailHandling
from lodur import Lodur
from mqtt import MQTTClient
from pdf_extract import PDFHandling
from webdav import WebDav
# TODO replace by IMAP idle
_INTERVAL = 10
# Configuration
load_dotenv(find_dotenv())
imap_server = os.getenv("IMAP_SERVER")
imap_username = os.getenv("IMAP_USERNAME")
imap_password = os.getenv("IMAP_PASSWORD")
imap_mailbox = os.getenv("IMAP_MAILBOX", "INBOX")
imap_mailbox_archive = os.getenv("IMAP_MAILBOX_ARCHIVE", "Archive")
webdav_url = os.getenv("WEBDAV_URL")
webdav_username = os.getenv("WEBDAV_USERNAME")
webdav_password = os.getenv("WEBDAV_PASSWORD")
webdav_basedir = os.getenv("WEBDAV_BASEDIR")
tmp_dir = os.getenv("TMP_DIR", "/tmp")
mqtt_server = os.getenv("MQTT_SERVER")
mqtt_user = os.getenv("MQTT_USER")
mqtt_password = os.getenv("MQTT_PASSWORD")
lodur_user = os.getenv("LODUR_USER")
lodur_password = os.getenv("LODUR_PASSWORD")
lodur_base_url = os.getenv("LODUR_BASE_URL")
heartbeat_url = os.getenv("HEARTBEAT_URL")
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)
def search_emails(imap):
""" searches for emails matching the configured subject """
# search for matching messages
logger.info('Searching for messages matching the subject')
typ, msg_ids = imap.search(
None,
_EMAIL_SUBJECTS,
)
if typ != 'OK':
logger.error('Error searching for matching messages')
raise
num_messages = len(msg_ids[0].split())
logger.info('Found ' + str(num_messages) + ' matching messages')
return num_messages, msg_ids
def store_attachments(imap, msg_ids):
""" stores the attachments to filesystem and marks message as read """
data = {}
for msg_id in msg_ids[0].split():
# download message from imap
typ, msg_data = imap.fetch(msg_id, '(RFC822)')
if typ != 'OK':
logger.error('Error fetching message')
raise
# extract attachment
for response_part in msg_data:
if isinstance(response_part, tuple):
mail = email.message_from_string(str(response_part[1], 'utf-8'))
subject = mail['subject']
logger.info('Getting attachment from: ' + subject)
for part in mail.walk():
file_name = part.get_filename()
if not file_name:
logger.debug('Most probably not an attachment as no filename found')
continue
logger.info('Extracting attachment: ' + file_name)
if bool(file_name):
f_type, f_id = parse_subject(subject)
renamed_file_name = f_type + '_' + file_name
# save attachment to filesystem
file_path = os.path.join(tmp_dir, renamed_file_name)
logger.info('Saving attachment to ' + file_path)
if not os.path.isfile(file_path):
file = open(file_path, 'wb')
file.write(part.get_payload(decode=True))
file.close()
data[subject] = renamed_file_name
# mark as seen
imap.store(msg_id, '+FLAGS', '(\\Seen)')
return data
def upload_webdav(loop, webdav, file_name, f_id):
""" uploads a file to webdav - checks for existence before doing so """
# upload with webdav
remote_upload_dir = webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
logger.info('Uploading file to WebDAV:' + remote_upload_dir)
# create directory if not yet there
if not loop.run_until_complete(webdav.exists(remote_upload_dir)):
logger.info('Creating directory ' + remote_upload_dir)
loop.run_until_complete(webdav.mkdir(remote_upload_dir))
remote_file_path = remote_upload_dir + "/" + file_name
if loop.run_until_complete(webdav.exists(remote_file_path)):
logger.info('File ' + file_name + ' already exists on webdav')
else:
loop.run_until_complete(
webdav.upload(os.path.join(tmp_dir, file_name), remote_file_path)
)
logger.info('File ' + file_name + ' uploaded')
def einsatz_exists(loop, webdav, f_id):
""" check if an einsatz is already created """
remote_upload_dir = webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
if loop.run_until_complete(webdav.exists(remote_upload_dir)):
logger.info('Einsatz exists ' + f_id)
return True
else:
return False
def parse_subject(subject):
""" extract f id and type from subject """
parsed = re.search('(.*): (F[0-9].*)', subject)
f_type = parsed.group(1)
f_id = parsed.group(2)
return f_type, f_id
def store_lodur_id(loop, webdav, lodur_id, f_id):
""" stores assigned lodur_id on webdav """
file_name = f_id + '_lodurid.txt'
file_path = os.path.join(tmp_dir, file_name)
if not os.path.isfile(file_path):
file = open(file_path, 'w')
file.write(str(lodur_id))
file.close()
logger.info('Stored Lodur ID locally in: ' + file_path)
upload_webdav(loop, webdav, file_name, f_id)
else:
logger.info('Lodur ID already available locally in: ' + file_path)
def get_lodur_id(loop, webdav, f_id):
""" gets lodur_id if it exists """
file_name = f_id + '_lodurid.txt'
file_path = os.path.join(tmp_dir, file_name)
# first check if we already have it locally - then check on webdav
if os.path.isfile(file_path):
with open(file_path, 'r') as content:
lodur_id = content.read()
logger.info('Found Lodur ID for ' + f_id + ' locally: ' + lodur_id)
return lodur_id
else:
remote_upload_dir = webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
remote_file_path = remote_upload_dir + '/' + file_name
if loop.run_until_complete(webdav.exists(remote_file_path)):
loop.run_until_complete(webdav.download(remote_file_path, file_path))
with open(file_path, 'r') as content:
lodur_id = content.read()
logger.info('Found Lodur ID for ' + f_id + ' on WebDAV: ' + lodur_id)
return lodur_id
else:
logger.info('No Lodur ID found for ' + f_id)
return False
IMAP_SERVER = os.getenv("IMAP_SERVER")
IMAP_USERNAME = os.getenv("IMAP_USERNAME")
IMAP_PASSWORD = os.getenv("IMAP_PASSWORD")
IMAP_MAILBOX = os.getenv("IMAP_MAILBOX", "INBOX")
WEBDAV_URL = os.getenv("WEBDAV_URL")
WEBDAV_USERNAME = os.getenv("WEBDAV_USERNAME")
WEBDAV_PASSWORD = os.getenv("WEBDAV_PASSWORD")
WEBDAV_BASEDIR = os.getenv("WEBDAV_BASEDIR")
TMP_DIR = os.getenv("TMP_DIR", "/tmp")
MQTT_SERVER = os.getenv("MQTT_SERVER")
MQTT_USER = os.getenv("MQTT_USER")
MQTT_PASSWORD = os.getenv("MQTT_PASSWORD")
LODUR_USER = os.getenv("LODUR_USER")
LODUR_PASSWORD = os.getenv("LODUR_PASSWORD")
LODUR_BASE_URL = os.getenv("LODUR_BASE_URL")
HEARTBEAT_URL = os.getenv("HEARTBEAT_URL")
def main():
""" main """
# MQTT connection
logger.info('Connecting to MQTT broker ' + mqtt_server)
mqtt_client = mqtt.Client('pylokid')
mqtt_client.username_pw_set(mqtt_user, password=mqtt_password)
mqtt_client.tls_set()
mqtt_client.connect(mqtt_server, 8883, 60)
mqtt_client.loop_start()
# imap connection
logger.info('Connecting to IMAP server ' + imap_server)
imap = imaplib.IMAP4_SSL(imap_server)
imap.login(imap_username, imap_password)
imap.select(imap_mailbox, readonly=False)
# webdav connection
logger.info('Connecting to WebDAV server ' + webdav_url)
loop = asyncio.get_event_loop()
webdav = aioeasywebdav.connect(
webdav_url,
username=webdav_username,
password=webdav_password,
# Logging configuration
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger('pylokid')
# Initialize IMAP Session
imap_client = EmailHandling(
IMAP_SERVER,
IMAP_USERNAME,
IMAP_PASSWORD,
IMAP_MAILBOX,
TMP_DIR,
)
# Initialize Lodur Session
lodur_client = Lodur(
LODUR_BASE_URL,
LODUR_USER,
LODUR_PASSWORD,
)
# Initialize WebDav Session
webdav_client = WebDav(
WEBDAV_URL,
WEBDAV_USERNAME,
WEBDAV_PASSWORD,
WEBDAV_BASEDIR,
TMP_DIR,
)
# Initialize MQTT Sessions
mqtt_client = MQTTClient(
MQTT_SERVER,
MQTT_USER,
MQTT_PASSWORD,
)
# Initialize PDF Parser
pdf = PDFHandling()
while True:
attachments = {}
num_messages, msg_ids = search_emails(imap)
if num_messages > 0:
attachments = store_attachments(imap, msg_ids)
num_messages, msg_ids = imap_client.search_emails()
if num_messages:
attachments = imap_client.store_attachments(msg_ids)
if len(attachments) > 0:
if attachments:
for subject in attachments:
f_type, f_id = parse_subject(subject)
f_type, f_id = imap_client.parse_subject(subject)
file_name = attachments[subject]
upload_webdav(loop, webdav, file_name, f_id)
webdav_client.upload(file_name, f_id)
# Take actions - depending on the type
if f_type == 'Einsatzausdruck_FW':
lodur_id = get_lodur_id(loop, webdav, f_id)
lodur_id = webdav_client.get_lodur_id(f_id)
if lodur_id:
logger.info(
'Einsatzrapport ' + f_id + ' already created in Lodur: ' + lodur_id
)
# Upload Alarmdepesche as it could contain more information than the first one
upload_alarmdepesche(
lodur_user,
lodur_password,
lodur_base_url,
lodur_client.upload_alarmdepesche(
lodur_id,
file_name,
os.path.join(tmp_dir, file_name),
os.path.join(TMP_DIR, file_name),
)
else:
# this is real - publish Einsatz on MQTT
# TODO publish more information about the einsatz - coming from the PDF
mqtt_client.publish('pylokid/' + f_type, f_id)
mqtt_client.send_message(f_type, f_id)
# get as many information from PDF as possible
pdf_data = pdf_extract.extract_einsatzausdruck(
os.path.join(tmp_dir, file_name),
pdf_data = pdf.extract_einsatzausdruck(
os.path.join(TMP_DIR, file_name),
f_id,
)
# create new Einsatzrapport in Lodur
logger.info('Creating Einsatzrapport in Lodur for ' + f_id)
lodur_id = create_einsatzrapport(
lodur_user,
lodur_password,
lodur_base_url,
lodur_id = lodur_client.create_einsatzrapport(
f_id,
pdf_data,
)
logger.info('Sent data to Lodur. Assigned Lodur ID: ' + lodur_id)
# store lodur id in webdav
store_lodur_id(loop, webdav, lodur_id, f_id)
webdav_client.store_lodur_id(lodur_id, f_id)
logger.info(
'Uploading PDF for ' + f_id + ' to Lodur Einsatzrapport ' + lodur_id
)
upload_alarmdepesche(
lodur_user,
lodur_password,
lodur_base_url,
lodur_client.upload_alarmdepesche(
lodur_id,
file_name,
os.path.join(tmp_dir, file_name),
os.path.join(TMP_DIR, file_name),
)
elif f_type == 'Einsatzprotokoll':
# Einsatz finished - publish on MQTT
mqtt_client.publish('pylokid/' + f_type, f_id)
mqtt_client.send_message(f_type, f_id)
lodur_id = get_lodur_id(loop, webdav, f_id)
lodur_id = webdav_client.get_lodur_id(f_id)
if lodur_id:
logger.info('Uploading Einsatzprotokoll to Lodur')
upload_alarmdepesche(
lodur_user,
lodur_password,
lodur_base_url,
lodur_client.upload_alarmdepesche(
lodur_id,
file_name,
os.path.join(tmp_dir, file_name),
os.path.join(TMP_DIR, file_name),
)
pdf_data = pdf_extract.extract_einsatzprotokoll(
os.path.join(tmp_dir, file_name),
pdf_data = pdf.extract_einsatzprotokoll(
os.path.join(TMP_DIR, file_name),
f_id,
)
# only update when parsing was successfull
@ -298,9 +161,12 @@ def main():
logger.error('Unknown type: ' + f_type)
# send heartbeat
requests.get(heartbeat_url)
requests.get(HEARTBEAT_URL)
# repeat every
time.sleep(_INTERVAL)
if __name__ == '__main__':
main()
try:
main()
except KeyboardInterrupt:
print("Byebye")

29
mqtt.py Normal file
View File

@ -0,0 +1,29 @@
#!/usr/bin/env python3
""" MQTT Functions """
import logging
import paho.mqtt.client as mqtt
class MQTTClient:
""" MQTT Client """
def __init__(self, server, username, password):
self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to MQTT broker ' + server)
try:
self.mqtt_client = mqtt.Client('pylokid')
self.mqtt_client.username_pw_set(username, password=password)
self.mqtt_client.tls_set()
self.mqtt_client.connect(server, 8883, 60)
self.mqtt_client.loop_start()
except Exception as err:
self.logger.error('MQTT connection failed - exiting: ' + str(err))
raise SystemExit(1)
self.logger.info('MQTT connection successfull')
def send_message(self, f_type, f_id):
""" Publish a message over MQTT """
self.mqtt_client.publish('pylokid/' + f_type, f_id)

180
pdf_extract.py
View File

@ -9,101 +9,117 @@ from pdfminer.converter import TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfpage import PDFPage
def concatenate_to_multiline_string(data, start, end):
""" concatenates multiple lines to a single multiline string """
res = ''
counter = start
while counter <= end:
res += data[counter] + '\n'
counter += 1
return res
class PDFHandling:
""" PDF handling like parsing """
def convert(file):
""" converts the PDF to a multiline string """
pagenums = set()
manager = PDFResourceManager()
codec = 'utf-8'
caching = True
def __init__(self):
self.logger = logging.getLogger(__name__)
output = io.StringIO()
converter = TextConverter(manager, output, codec=codec, laparams=LAParams())
# less logging for pdfminer - more is not needed
logger_doc = logging.getLogger('pdfminer.pdfdocument')
logger_doc.setLevel(logging.WARNING)
logger_page = logging.getLogger('pdfminer.pdfpage')
logger_page.setLevel(logging.WARNING)
logger_interp = logging.getLogger('pdfminer.pdfinterp')
logger_interp.setLevel(logging.WARNING)
interpreter = PDFPageInterpreter(manager, converter)
infile = open(file, 'rb')
def concatenate_to_multiline_string(self, data, start, end):
""" concatenates multiple lines to a single multiline string """
for page in PDFPage.get_pages(infile, pagenums, caching=caching, check_extractable=True):
interpreter.process_page(page)
res = ''
counter = start
while counter <= end:
res += data[counter] + '\n'
counter += 1
return res
converted_pdf = output.getvalue()
def convert(self, file):
""" converts the PDF to a multiline string """
infile.close()
converter.close()
output.close()
return converted_pdf
pagenums = set()
manager = PDFResourceManager()
codec = 'utf-8'
caching = True
def extract_einsatzausdruck(file, f_id):
""" extracts as many information from the parsed Einsatzausdruck as possible """
output = io.StringIO()
converter = TextConverter(manager, output, codec=codec, laparams=LAParams())
splited = convert(file).splitlines()
interpreter = PDFPageInterpreter(manager, converter)
infile = open(file, 'rb')
# sanity check to see if we can correlate the f_id
if f_id == splited[14]:
logging.info('PDF parsing: f_id matches line 14')
else:
logging.error('PDF parsing: f_id doesn\'t match line 14')
return False
for page in PDFPage.get_pages(infile, pagenums, caching=caching, check_extractable=True):
interpreter.process_page(page)
try:
# search some well-known words for later positional computation
index_bemerkungen = splited.index('Bemerkungen')
index_dispo = splited.index('Disponierte Einheiten')
index_hinweis = splited.index('Hinweis')
except:
loggin.error('PDF file doesn\'t look like a Einsatzausdruck')
return False
converted_pdf = output.getvalue()
# get length of bemerkungen field
# it lives between the line which contains 'Bemerkungen' and
# the line 'Disponierte Einheiten'
length_bemerkungen = index_dispo - index_bemerkungen - 1
infile.close()
converter.close()
output.close()
return converted_pdf
data = {
'auftrag': splited[14],
'datum': splited[15],
'zeit': splited[16],
'melder': concatenate_to_multiline_string(splited, 18, 19),
'erfasser': splited[20],
'bemerkungen': concatenate_to_multiline_string(
splited,
index_bemerkungen,
index_bemerkungen + length_bemerkungen
),
'einsatz': splited[index_dispo+5],
'plzort': splited[index_dispo+8].title(),
'strasse': splited[index_dispo+9].title(),
#'objekt': splited[],
'hinweis': splited[index_hinweis+2]
}
return data
def extract_einsatzausdruck(self, file, f_id):
""" extracts as many information from the parsed Einsatzausdruck as possible """
def extract_einsatzprotokoll(file, f_id):
""" extracts as many information from the parsed Einsatzprotokoll as possible """
splited = self.convert(file).splitlines()
splited = convert(file).splitlines()
# sanity check to see if we can correlate the f_id
if f_id == splited[14]:
self.logger.info('PDF parsing: f_id matches line 14')
else:
self.logger.error('PDF parsing: f_id doesn\'t match line 14')
return False
# sanity check to see if we can correlate the f_id
if f_id == splited[26]:
logging.info('PDF parsing: f_id matches line 26')
else:
logging.error('PDF parsing: f_id doesn\'t match line 26')
return False
try:
# search some well-known words for later positional computation
index_bemerkungen = splited.index('Bemerkungen')
index_dispo = splited.index('Disponierte Einheiten')
index_hinweis = splited.index('Hinweis')
except:
self.logger.error('PDF file doesn\'t look like a Einsatzausdruck')
return False
data = {
'auftrag': splited[26],
'datum': splited[25],
'angelegt': splited[28],
'disposition': splited[30],
'ausgerueckt': splited[32],
'anort': splited[33],
}
return data
# get length of bemerkungen field
# it lives between the line which contains 'Bemerkungen' and
# the line 'Disponierte Einheiten'
length_bemerkungen = index_dispo - index_bemerkungen - 1
data = {
'auftrag': splited[14],
'datum': splited[15],
'zeit': splited[16],
'melder': self.concatenate_to_multiline_string(splited, 18, 19),
'erfasser': splited[20],
'bemerkungen': self.concatenate_to_multiline_string(
splited,
index_bemerkungen,
index_bemerkungen + length_bemerkungen
),
'einsatz': splited[index_dispo+5],
'plzort': splited[index_dispo+8].title(),
'strasse': splited[index_dispo+9].title(),
#'objekt': splited[],
'hinweis': splited[index_hinweis+2]
}
return data
def extract_einsatzprotokoll(self, file, f_id):
""" extracts as many information from the parsed Einsatzprotokoll as possible """
splited = self.convert(file).splitlines()
# sanity check to see if we can correlate the f_id
if f_id == splited[26]:
self.logger.info('PDF parsing: f_id matches line 26')
else:
self.logger.error('PDF parsing: f_id doesn\'t match line 26')
return False
data = {
'auftrag': splited[26],
'datum': splited[25],
'angelegt': splited[28],
'disposition': splited[30],
'ausgerueckt': splited[32],
'anort': splited[33],
}
return data

8
requirements.txt
View File

@ -1,4 +1,4 @@
aioeasywebdav
python-dotenv
paho-mqtt
requests
aioeasywebdav==2.4.0
python-dotenv==0.7.1
paho-mqtt==1.3.1
requests==2.18.4

103
webdav.py Normal file
View File

@ -0,0 +1,103 @@
#!/usr/bin/env python3
""" WebDav Functions """
import os
from datetime import datetime
import logging
import asyncio
import aioeasywebdav
class WebDav:
""" WebDav Client """
def __init__(self, url, username, password, webdav_basedir, tmp_dir):
self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to WebDAV server: ' + url)
self.loop = asyncio.get_event_loop()
self.webdav_basedir = webdav_basedir
self.tmp_dir = tmp_dir
try:
self.webdav = aioeasywebdav.connect(
url,
username=username,
password=password,
)
except:
self.logger.error('WebDav connection failed - exiting')
self.logger.info('WebDav connection successfull')
def upload(self, file_name, f_id):
""" uploads a file to webdav - checks for existence before doing so """
# upload with webdav
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
self.logger.info('Uploading file to WebDAV:' + remote_upload_dir)
# create directory if not yet there
if not self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)):
self.logger.info('Creating directory ' + remote_upload_dir)
self.loop.run_until_complete(self.webdav.mkdir(remote_upload_dir))
remote_file_path = remote_upload_dir + "/" + file_name
if self.loop.run_until_complete(self.webdav.exists(remote_file_path)):
self.logger.info('File ' + file_name + ' already exists on webdav')
else:
self.loop.run_until_complete(
self.webdav.upload(
os.path.join(self.tmp_dir, file_name),
remote_file_path,
)
)
self.logger.info('File ' + file_name + ' uploaded')
def einsatz_exists(self, f_id):
""" check if an einsatz is already created """
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
if self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)):
self.logger.info('Einsatz exists ' + f_id)
return True
else:
return False
def store_lodur_id(self, lodur_id, f_id):
""" stores assigned lodur_id on webdav """
file_name = f_id + '_lodurid.txt'
file_path = os.path.join(self.tmp_dir, file_name)
if not os.path.isfile(file_path):
file = open(file_path, 'w')
file.write(str(lodur_id))
file.close()
self.logger.info('Stored Lodur ID locally in: ' + file_path)
self.upload(file_name, f_id)
else:
self.logger.info('Lodur ID already available locally in: ' + file_path)
def get_lodur_id(self, f_id):
""" gets lodur_id if it exists """
file_name = f_id + '_lodurid.txt'
file_path = os.path.join(self.tmp_dir, file_name)
# first check if we already have it locally - then check on webdav
if os.path.isfile(file_path):
with open(file_path, 'r') as content:
lodur_id = content.read()
self.logger.info('Found Lodur ID for ' + f_id + ' locally: ' + lodur_id)
return lodur_id
else:
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
remote_file_path = remote_upload_dir + '/' + file_name
if self.loop.run_until_complete(self.webdav.exists(remote_file_path)):
self.loop.run_until_complete(self.webdav.download(remote_file_path, file_path))
with open(file_path, 'r') as content:
lodur_id = content.read()
self.logger.info('Found Lodur ID for ' + f_id + ' on WebDAV: ' + lodur_id)
return lodur_id
else:
self.logger.info('No Lodur ID found for ' + f_id)
return False