tobru
/
pylokid
1
0
Fork 0
Code Pull Requests Releases Activity

improve logging - create lib dir

This commit is contained in:
Tobias Brunner 2018-01-02 16:45:37 +01:00
parent 6b93ae3dfa
commit a663cd084e
7 changed files with 44 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -20,13 +20,9 @@ Einsätze correctly into [Lodur](https://www.lodur.ch/lodur.html).
### Version 1
* Much more error handling
* Parse PDF
* Store parsed data in Lodur text fields for copy/paste
* Cleanup code into proper functions and classes
* Lodur "API" class
* Proper exit
* Healthchecks for Kubernetes probes
* IMAP IDLE
* MQTT Message Format / Send PDF payload
* Parse Form
Before version 1 can be tagged, it must have processed at least 5 real
Einsätze!
@ -39,7 +35,6 @@ Einsätze!
* Generalize
* Documentation
* IMAP idle
* Display PDF on Dashboard
* Send statistics to InfluxDB
* Webapp to see what's going on

19
emailhandling.py → library/emailhandling.py
View File

@ -16,7 +16,7 @@ class EmailHandling:
def __init__(self, server, username, password, mailbox, tmp_dir):
self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to IMAP server ' + server)
self.logger.info('Connecting to IMAP server %s', server)
self.tmp_dir = tmp_dir
try:
@ -24,7 +24,7 @@ class EmailHandling:
self.imap.login(username, password)
self.imap.select(mailbox, readonly=False)
except Exception as err:
self.logger.error('IMAP connection failed - exiting: ' + str(err))
self.logger.error('IMAP connection failed - exiting: %s', str(err))
raise SystemExit(1)
self.logger.info('IMAP connection successfull')
@ -42,7 +42,7 @@ class EmailHandling:
return False
num_messages = len(msg_ids[0].split())
self.logger.info('Found ' + str(num_messages) + ' matching messages')
self.logger.info('Found %s matching messages', str(num_messages))
return num_messages, msg_ids
@ -63,14 +63,17 @@ class EmailHandling:
if isinstance(response_part, tuple):
mail = email.message_from_string(str(response_part[1], 'utf-8'))
subject = mail['subject']
self.logger.info('Getting attachment from: ' + subject)
f_type, f_id = self.parse_subject(subject)
self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
for part in mail.walk():
file_name = part.get_filename()
if not file_name:
self.logger.debug('Most probably not an attachment as no filename found')
self.logger.debug(
'Most probably not an attachment as no filename found'
)
continue
self.logger.info('Extracting attachment: ' + file_name)
self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
if bool(file_name):
f_type, _ = self.parse_subject(subject)
@ -78,7 +81,7 @@ class EmailHandling:
# save attachment to filesystem
file_path = os.path.join(self.tmp_dir, renamed_file_name)
self.logger.info('Saving attachment to ' + file_path)
self.logger.info('[%s] Saving attachment to "%s"', f_id, file_path)
if not os.path.isfile(file_path):
file = open(file_path, 'wb')
file.write(part.get_payload(decode=True))
@ -87,7 +90,7 @@ class EmailHandling:
data[subject] = renamed_file_name
# mark as seen
self.logger.info('Marking message as seen ' + subject)
self.logger.info('[%s] Marking message "%s" as seen', f_id, subject)
self.imap.store(msg_id, '+FLAGS', '(\\Seen)')
return data

14
lodur.py → library/lodur.py
View File

@ -37,14 +37,7 @@ class Lodur:
raise SystemExit(1)
def einsatzprotokoll(self, f_id, pdf_data, webdav_client):
""" Prepare Einsatzprotokoll to be sent to Lodur
TODO This doesn't work as Lodur doesn't add the values directly in to HTML but
uses JavaScript to dynamically populate the form data.
To be able to update the form we would need to have access to the existing data
or else it won't work.
Ideas: Somehow store the RAW data in a JSON file and reuse this
to update the form in Lodur
"""
""" Prepare Einsatzprotokoll to be sent to Lodur """
# check if data is already sent to lodur - data contains lodur_id
lodur_data = webdav_client.get_lodur_data(f_id)
@ -153,6 +146,8 @@ class Lodur:
def einsatzrapport_alarmdepesche(self, f_id, file_path, webdav_client):
""" Upload a file to Alarmdepesche """
self.logger.info('[%s] Submitting Alarmdepesche to Lodur', f_id)
# check if data is already sent to lodur - data contains lodur_id
lodur_id = webdav_client.get_lodur_data(f_id)['event_id']
@ -161,7 +156,6 @@ class Lodur:
self.browser.select_form('#frm_alarmdepesche')
# Fill in form data
self.logger.info('[%s] Submitting Alarmdepesche to Lodur', f_id)
self.browser['alarmdepesche'] = open(file_path, 'rb')
# Submit the form
@ -224,7 +218,7 @@ class Lodur:
self.url +
'?modul=36&edit=1&what=144&event=' + lodur_id
).text
auto_num = re.search("fdata\['auto_num'\]\[2\]='(.*)';", content).group(1)
auto_num = re.search(r"fdata\['auto_num'\]\[2\]='(.*)';", content).group(1)
self.logger.info('[%s] Lodur assigned the auto_num %s', lodur_data['e_r_num'], auto_num)
return lodur_id, auto_num

0
mqtt.py → library/mqtt.py
View File

16
pdf_extract.py → library/pdf_extract.py
View File

@ -64,18 +64,18 @@ class PDFHandling:
# sanity check to see if we can correlate the f_id
if f_id == splited[14]:
self.logger.info('PDF parsing: f_id matches line 14')
self.logger.info('[%s] ID found in PDF', f_id)
else:
self.logger.error('PDF parsing: f_id doesn\'t match line 14')
self.logger.error('[%s] ID not found in PDF', f_id)
return False
try:
# search some well-known words for later positional computation
# search some well-known words for later positional computation
try:
index_bemerkungen = splited.index('Bemerkungen')
index_dispo = splited.index('Disponierte Einheiten')
index_hinweis = splited.index('Hinweis')
except:
self.logger.error('PDF file doesn\'t look like a Einsatzausdruck')
except IndexError:
self.logger.error('[%s] PDF file doesn\'t look like a Einsatzausdruck', f_id)
return False
# get length of bemerkungen field
@ -109,9 +109,9 @@ class PDFHandling:
# sanity check to see if we can correlate the f_id
if f_id == splited[26]:
self.logger.info('PDF parsing: f_id matches line 26')
self.logger.info('[%s] ID found in PDF', f_id)
else:
self.logger.error('PDF parsing: f_id doesn\'t match line 26')
self.logger.error('[%s] ID not found in PDF', f_id)
return False
data = {

18
webdav.py → library/webdav.py
View File

@ -14,7 +14,7 @@ class WebDav:
def __init__(self, url, username, password, webdav_basedir, tmp_dir):
self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to WebDAV server: ' + url)
self.logger.info('Connecting to WebDAV server %s', url)
self.loop = asyncio.get_event_loop()
self.webdav_basedir = webdav_basedir
@ -26,25 +26,25 @@ class WebDav:
password=password,
)
except:
self.logger.error('WebDav connection failed - exiting')
self.logger.error('WebDAV connection failed - exiting')
self.logger.info('WebDav connection successfull')
self.logger.info('WebDAV connection successfull')
def upload(self, file_name, f_id):
""" uploads a file to webdav - checks for existence before doing so """
# upload with webdav
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
self.logger.info('Uploading file to WebDAV:' + remote_upload_dir)
self.logger.info('[%s] Uploading file to WebDAV [%s]', f_id, remote_upload_dir)
# create directory if not yet there
if not self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)):
self.logger.info('Creating directory ' + remote_upload_dir)
self.logger.info('[%s] Creating directory [%s]', f_id, remote_upload_dir)
self.loop.run_until_complete(self.webdav.mkdir(remote_upload_dir))
remote_file_path = remote_upload_dir + "/" + file_name
if self.loop.run_until_complete(self.webdav.exists(remote_file_path)):
self.logger.info('File ' + file_name + ' already exists on webdav')
self.logger.info('[%s] File %s already exists on WebDAV', f_id, file_name)
else:
self.loop.run_until_complete(
self.webdav.upload(
@ -52,14 +52,14 @@ class WebDav:
remote_file_path,
)
)
self.logger.info('File ' + file_name + ' uploaded')
self.logger.info('[%s] File %s uploaded', f_id, file_name)
def einsatz_exists(self, f_id):
""" check if an einsatz is already created """
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
if self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)):
self.logger.info('Einsatz exists ' + f_id)
self.logger.info('[%s] Einsatz exists on WebDAV', f_id)
return True
else:
return False
@ -74,7 +74,7 @@ class WebDav:
file.write(json.dumps(lodur_data))
file.close()
self.logger.info('Stored Lodur data locally in: ' + file_path)
self.logger.info('[%s] Stored Lodur data locally in %s', f_id, file_path)
self.upload(file_name, f_id)
def get_lodur_data(self, f_id):

17
main.py
View File

@ -10,11 +10,11 @@ import requests
from dotenv import find_dotenv, load_dotenv
# local classes
from emailhandling import EmailHandling
from lodur import Lodur
from mqtt import MQTTClient
from pdf_extract import PDFHandling
from webdav import WebDav
from library.emailhandling import EmailHandling
from library.lodur import Lodur
from library.mqtt import MQTTClient
from library.pdf_extract import PDFHandling
from library.webdav import WebDav
# TODO replace by IMAP idle
_INTERVAL = 10
@ -83,6 +83,7 @@ def main():
# Initialize PDF Parser
pdf = PDFHandling()
# Main Loop
while True:
attachments = {}
num_messages, msg_ids = imap_client.search_emails()
@ -97,7 +98,9 @@ def main():
# Take actions - depending on the type
if f_type == 'Einsatzausdruck_FW':
logger.info('[%s] Processing type %s', f_id, f_type)
lodur_data = webdav_client.get_lodur_data(f_id)
if lodur_data:
logger.info(
'[%s] Einsatzrapport already created in Lodur', f_id
@ -136,13 +139,12 @@ def main():
)
elif f_type == 'Einsatzprotokoll':
logger.info('[%s] Processing type %s', f_id, f_type)
# Einsatz finished - publish on MQTT
mqtt_client.send_message(f_type, f_id)
lodur_data = webdav_client.get_lodur_data(f_id)
if lodur_data:
logger.info('[%s] Uploading Einsatzprotokoll to Lodur', f_id)
# Upload Einsatzprotokoll to Lodur
lodur_client.einsatzrapport_alarmdepesche(
f_id,
@ -157,7 +159,6 @@ def main():
)
# Update entry in Lodur with parse PDF data
logger.info('[%s] Updating Einsatzrapport with data from PDF', f_id)
lodur_client.einsatzprotokoll(f_id, pdf_data, webdav_client)
else: