add pdftk hint

By using BODY.PEEK[] messages are not implicitely marked seen which
allows to set them seen when it could be processed. Might help to
reiterate over mails until the record is ready in Lodur.

.drone.yml

@@ -1,4 +1,5 @@
 kind: pipeline
+type: kubernetes
 name: default
 
 steps:

.gitignore

@@ -1,4 +1,6 @@
 __pycache__/
 .vscode/
 .env
-test.py
+pylokid/temp_test.py
+test.py
+tmp/

Dockerfile

@@ -39,13 +39,15 @@ COPY --from=builder \
       /app/dist /app/dist
 RUN pip install /app/dist/pylokid-*-py3-none-any.whl
 
-COPY hack/patches/*.patch /tmp
+COPY hack/patches/*.patch /tmp/
 
 # The ugliest possible way to workaround https://github.com/MechanicalSoup/MechanicalSoup/issues/356
 # For some unknown reasons Lodur now wants "Content-Type: application/pdf" set in the multipart
 # data section. And as I couln't figure out yet how to do that in MechanicalSoup and I only upload PDFs
 # I just patch it to hardcode it. YOLO
-RUN patch -p0 /usr/local/lib/python3.9/site-packages/mechanicalsoup/browser.py < /tmp/mechsoup-browser-content-type.patch
+RUN \
+  patch -p0 /usr/local/lib/python3.9/site-packages/mechanicalsoup/browser.py < /tmp/mechsoup-browser-content-type.patch && \
+  patch -p0 /usr/local/lib/python3.9/site-packages/mechanicalsoup/stateful_browser.py < /tmp/mechsoup-link-regex.patch
 
 ## ----------- Step 4
 FROM base AS runtime

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Tobias Brunner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -32,7 +32,7 @@ pylokid funktioniert so:
   "Einsatzausdruck_FW" oder "Einsatzprotokoll".
 * Wird ein passendes E-Mail gefunden, wird der Anhang (das PDF)
   heruntergeladen, in die Cloud gespeichert (WebDAV) und im Lodur
-  ein entsprechender Einsatzrapport eröffnet und vorausgefüllt.
+  der entsprechende Einsatzrapport gesucht.
   Das PDF wird sinnvoll umbenannt und als Alarmdepesche ins Lodur
   geladen.
 * Kommen weitere E-Mails mit dem Betreff "Einsatzausdruck_FW" werden
@@ -42,6 +42,7 @@ pylokid funktioniert so:
 * Wird der von Hand ausgefüllte Einsatzrapport via Scanner per E-Mail
   an das E-Mail Postfach gesendet (Betreff "Attached Image FXXXXXXXX")
   wird das PDF in der Cloud und im Lodur gespeichert.
+* Nach jedem Durchgang wird ein Heartbeat an den konfigurierten Healthcheck Service gesendet, z.B. https://healthchecks.io/
 
 Desweiteren wird über Pushover eine Nachricht mit möglichst vielen
 Informationen publiziert.
@@ -72,6 +73,38 @@ an Lodur gesendet werden, in einem JSON File im WebDAV neben den
 PDFs abgelegt. So lässt sich im Nachhinein ein Datensatz bearbeiten
 und eine Zuordnung des Einsatzes im WebDAV und in Lodur herstellen.
 
+## Detailierter Ablauf
+
+### Einsatzausdruck_FW
+
+1. PDF extrahieren und in Cloud hochladen
+2. Falls PDF noch nicht geparst wurde wird davon ausgegangen, dass dies die initiale Meldung ist:
+   1. PDF parsen
+   2. Push Nachricht mit Infos aus PDF senden
+   3. Geparste Daten als JSON in Cloud speichern
+3. Falls Einsatz im Lodur noch nicht ausgelesen:
+   1. Einsatz Datensatz ID im Lodur suchen
+   2. Ganzer Datensatz auslesen
+   3. Datensatz als JSON in Cloud speichern
+   4. PDF in Lodur speichern
+   5. E-Mail als gelesen markieren - wird somit nicht nochmals bearbeitet
+### Einsatzprotokoll
+
+1. Lodur Datensatz ID aus Cloud laden (JSON Datei)
+2. Ganzer Datensatz aus Lodur auslesen und als JSON in Cloud speichern
+3. Falls Datensatz zur Bearbeitung freigegeben ist (`aut_created_report == finished`)
+   1. PDF in Lodur speichern
+   2. Einsatzprotokoll Daten ergänzen und in Lodur speichern
+   3. Push Nachricht senden (Einsatz beendet)
+   4. E-Mail als gelesen markieren - wird somit nicht nochmals bearbeitet
+
+### Einsatzrapport
+
+1. Prüfen, ob F-Nummer aus Scan E-Mail Betreff gefunden
+2. Lodur Datensatz ID aus Cloud laden (JSON Datei)
+3. Ganzer Datensatz aus Lodur auslesen und als JSON in Cloud speichern
+4. PDF in Lodur speichern und Datensatz ergänzen
+5. Push Nachricht senden (Rapport bearbeitet)
 ## Installation and Configuration
 
 The application is written in Python and runs perfectly on Kubernetes.

hack/generate-einsatrapporte.py

@@ -0,0 +1,54 @@
+import uno
+from com.sun.star.beans import PropertyValue
+from com.sun.star.uno import RuntimeException
+
+def connect_to_libreoffice(port=2002):
+    localContext = uno.getComponentContext()
+    resolver = localContext.ServiceManager.createInstanceWithContext(
+        "com.sun.star.bridge.UnoUrlResolver", localContext)
+    try:
+        context = resolver.resolve(f"uno:socket,host=localhost,port={port};urp;StarOffice.ComponentContext")
+        return context.ServiceManager.createInstanceWithContext("com.sun.star.frame.Desktop", context)
+    except RuntimeException:
+        raise Exception("Make sure LibreOffice is running with a listening port (e.g., soffice --accept=\"socket,port=2002;urp;\" --norestore --nologo --nodefault)")
+
+def export_to_pdf(doc, file_name):
+    pdf_export_properties = tuple([
+        PropertyValue(Name="FilterName", Value="writer_pdf_Export")
+    ])
+    doc.storeToURL(f"file:///{file_name}", pdf_export_properties)
+
+def replace_text(doc, old_text, new_text):
+    replaceable = doc.createReplaceDescriptor()
+    replaceable.setSearchString(old_text)
+    replaceable.setReplaceString(new_text)
+
+    doc.replaceAll(replaceable)
+
+
+
+def main(start_counter, num_documents, document_path):
+    desktop = connect_to_libreoffice()
+
+    for i in range(num_documents):
+        counter = start_counter + i
+        load_props = PropertyValue(Name="Hidden", Value=True),
+        doc = desktop.loadComponentFromURL(f"file:///{document_path}", "_blank", 0, load_props)
+
+        replace_text(doc, "counter", str(counter))
+
+        file_name = f"/home/tobru/Documents/Feuerwehr/Stab/Fourier/EinsatzrapporteLeer/Einsatzrapport2024-{counter}.pdf"
+        export_to_pdf(doc, file_name)
+        doc.dispose()
+
+if __name__ == "__main__":
+    document_path = "/home/tobru/Documents/Feuerwehr/Stab/Fourier/FW-Einsatzrapport FWU.odt"
+    start_counter = 1
+    num_documents = 100
+
+    # Start libreoffice with:
+    # soffice --accept="socket,port=2002;urp;" --norestore --nologo --nodefault
+    main(start_counter, num_documents, document_path)
+
+    # after generation, run:
+    # pdftk $(ls -v *.pdf) cat output Einsatzrapporte2024.pdf

hack/patches/mechsoup-link-regex.patch

@@ -0,0 +1,21 @@
+--- /home/tobru/.cache/pypoetry/virtualenvs/pylokid-JpqZeVMm-py3.9/lib/python3.9/site-packages/mechanicalsoup/stateful_browser.py.orig	2021-02-27 14:21:53.979582533 +0100
++++ /home/tobru/.cache/pypoetry/virtualenvs/pylokid-JpqZeVMm-py3.9/lib/python3.9/site-packages/mechanicalsoup/stateful_browser.py	2021-02-27 14:23:17.680374365 +0100
+@@ -259,7 +259,7 @@
+         for link in self.links(*args, **kwargs):
+             print("    ", link)
+ 
+-    def links(self, url_regex=None, link_text=None, *args, **kwargs):
++    def links(self, url_regex=None, link_text=None, link_regex=None, *args, **kwargs):
+         """Return links in the page, as a list of bs4.element.Tag objects.
+ 
+         To return links matching specific criteria, specify ``url_regex``
+@@ -276,6 +276,9 @@
+         if link_text is not None:
+             all_links = [a for a in all_links
+                          if a.text == link_text]
++        if link_regex is not None:
++            all_links = [a for a in all_links
++                         if re.search(link_regex, a.text)]
+         return all_links
+ 
+     def find_link(self, *args, **kwargs):

poetry.lock

@@ -30,6 +30,14 @@ yarl = ">=1.0,<2.0"
 [package.extras]
 speedups = ["aiodns", "brotlipy", "cchardet"]
 
+[[package]]
+name = "appdirs"
+version = "1.4.4"
+description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
+category = "dev"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "async-timeout"
 version = "3.0.1"
@@ -67,6 +75,28 @@ soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""}
 html5lib = ["html5lib"]
 lxml = ["lxml"]
 
+[[package]]
+name = "black"
+version = "20.8b1"
+description = "The uncompromising code formatter."
+category = "dev"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+appdirs = "*"
+click = ">=7.1.2"
+mypy-extensions = ">=0.4.3"
+pathspec = ">=0.6,<1"
+regex = ">=2020.1.8"
+toml = ">=0.10.1"
+typed-ast = ">=1.4.0"
+typing-extensions = ">=3.7.4"
+
+[package.extras]
+colorama = ["colorama (>=0.4.3)"]
+d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
+
 [[package]]
 name = "certifi"
 version = "2020.12.5"
@@ -83,6 +113,27 @@ category = "main"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "click"
+version = "7.1.2"
+description = "Composable command line interface toolkit"
+category = "dev"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[[package]]
+name = "flake8"
+version = "3.8.4"
+description = "the modular source code checker: pep8 pyflakes and co"
+category = "dev"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
+
+[package.dependencies]
+mccabe = ">=0.6.0,<0.7.0"
+pycodestyle = ">=2.6.0a1,<2.7.0"
+pyflakes = ">=2.2.0,<2.3.0"
+
 [[package]]
 name = "idna"
 version = "2.10"
@@ -105,6 +156,14 @@ html5 = ["html5lib"]
 htmlsoup = ["beautifulsoup4"]
 source = ["Cython (>=0.29.7)"]
 
+[[package]]
+name = "mccabe"
+version = "0.6.1"
+description = "McCabe checker, plugin for flake8"
+category = "dev"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "mechanicalsoup"
 version = "1.0.0"
@@ -127,6 +186,38 @@ category = "main"
 optional = false
 python-versions = ">=3.6"
 
+[[package]]
+name = "mypy-extensions"
+version = "0.4.3"
+description = "Experimental type system extensions for programs checked with the mypy typechecker."
+category = "dev"
+optional = false
+python-versions = "*"
+
+[[package]]
+name = "pathspec"
+version = "0.8.1"
+description = "Utility library for gitignore style pattern matching of file paths."
+category = "dev"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+
+[[package]]
+name = "pycodestyle"
+version = "2.6.0"
+description = "Python style guide checker"
+category = "dev"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
+[[package]]
+name = "pyflakes"
+version = "2.2.0"
+description = "passive checker of Python programs"
+category = "dev"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+
 [[package]]
 name = "python-dotenv"
 version = "0.15.0"
@@ -149,6 +240,14 @@ python-versions = "*"
 [package.dependencies]
 requests = ">=1.0"
 
+[[package]]
+name = "regex"
+version = "2020.11.13"
+description = "Alternative regular expression module, to replace re."
+category = "dev"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "requests"
 version = "2.25.1"
@@ -194,6 +293,22 @@ category = "main"
 optional = false
 python-versions = ">=3.6"
 
+[[package]]
+name = "toml"
+version = "0.10.2"
+description = "Python Library for Tom's Obvious, Minimal Language"
+category = "dev"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+
+[[package]]
+name = "typed-ast"
+version = "1.4.2"
+description = "a fork of Python 2 and 3 ast modules with type comment support"
+category = "dev"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "typing-extensions"
 version = "3.7.4.3"
@@ -230,7 +345,7 @@ multidict = ">=4.0"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.9"
-content-hash = "c5fa2a2589b88adb6a9bf94ec4a88e0e6b9528e40837cf6b2cf045a3e072d6d4"
+content-hash = "87029b7a633e874d04d308355076802812613a9b93a03770d10ab6b9fcbc5b44"
 
 [metadata.files]
 aioeasywebdav = [
@@ -276,6 +391,10 @@ aiohttp = [
     {file = "aiohttp-3.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7"},
     {file = "aiohttp-3.7.3.tar.gz", hash = "sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4"},
 ]
+appdirs = [
+    {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
+    {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
+]
 async-timeout = [
     {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"},
     {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"},
@@ -289,6 +408,9 @@ beautifulsoup4 = [
     {file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"},
     {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"},
 ]
+black = [
+    {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"},
+]
 certifi = [
     {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"},
     {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"},
@@ -297,6 +419,14 @@ chardet = [
     {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
     {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
 ]
+click = [
+    {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
+    {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
+]
+flake8 = [
+    {file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"},
+    {file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"},
+]
 idna = [
     {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
     {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
@@ -340,6 +470,10 @@ lxml = [
     {file = "lxml-4.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf"},
     {file = "lxml-4.6.2.tar.gz", hash = "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc"},
 ]
+mccabe = [
+    {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
+    {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
+]
 mechanicalsoup = [
     {file = "MechanicalSoup-1.0.0-py2.py3-none-any.whl", hash = "sha256:2ed9a494c144fb2c262408dcbd5c79e1ef325a7426f1fa3a3443eaef133b5f77"},
     {file = "MechanicalSoup-1.0.0.tar.gz", hash = "sha256:37d3b15c1957917d3ae171561e77f4dd4c08c35eb4500b8781f6e7e1bb6c4d07"},
@@ -383,6 +517,22 @@ multidict = [
     {file = "multidict-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359"},
     {file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"},
 ]
+mypy-extensions = [
+    {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
+    {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
+]
+pathspec = [
+    {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"},
+    {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"},
+]
+pycodestyle = [
+    {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"},
+    {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"},
+]
+pyflakes = [
+    {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"},
+    {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"},
+]
 python-dotenv = [
     {file = "python-dotenv-0.15.0.tar.gz", hash = "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0"},
     {file = "python_dotenv-0.15.0-py2.py3-none-any.whl", hash = "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e"},
@@ -390,6 +540,49 @@ python-dotenv = [
 python-pushover = [
     {file = "python-pushover-0.4.tar.gz", hash = "sha256:dee1b1344fb8a5874365fc9f886d9cbc7775536629999be54dfa60177cf80810"},
 ]
+regex = [
+    {file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"},
+    {file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"},
+    {file = "regex-2020.11.13-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee"},
+    {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5"},
+    {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7"},
+    {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31"},
+    {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa"},
+    {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6"},
+    {file = "regex-2020.11.13-cp36-cp36m-win32.whl", hash = "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e"},
+    {file = "regex-2020.11.13-cp36-cp36m-win_amd64.whl", hash = "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884"},
+    {file = "regex-2020.11.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b"},
+    {file = "regex-2020.11.13-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88"},
+    {file = "regex-2020.11.13-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0"},
+    {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1"},
+    {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0"},
+    {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512"},
+    {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba"},
+    {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538"},
+    {file = "regex-2020.11.13-cp37-cp37m-win32.whl", hash = "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4"},
+    {file = "regex-2020.11.13-cp37-cp37m-win_amd64.whl", hash = "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444"},
+    {file = "regex-2020.11.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f"},
+    {file = "regex-2020.11.13-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d"},
+    {file = "regex-2020.11.13-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af"},
+    {file = "regex-2020.11.13-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f"},
+    {file = "regex-2020.11.13-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b"},
+    {file = "regex-2020.11.13-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8"},
+    {file = "regex-2020.11.13-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5"},
+    {file = "regex-2020.11.13-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b"},
+    {file = "regex-2020.11.13-cp38-cp38-win32.whl", hash = "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c"},
+    {file = "regex-2020.11.13-cp38-cp38-win_amd64.whl", hash = "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683"},
+    {file = "regex-2020.11.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc"},
+    {file = "regex-2020.11.13-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364"},
+    {file = "regex-2020.11.13-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e"},
+    {file = "regex-2020.11.13-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e"},
+    {file = "regex-2020.11.13-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917"},
+    {file = "regex-2020.11.13-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b"},
+    {file = "regex-2020.11.13-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9"},
+    {file = "regex-2020.11.13-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c"},
+    {file = "regex-2020.11.13-cp39-cp39-win32.whl", hash = "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"},
+    {file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"},
+    {file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"},
+]
 requests = [
     {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"},
     {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"},
@@ -412,6 +605,42 @@ soupsieve = [
     {file = "soupsieve-2.2-py3-none-any.whl", hash = "sha256:d3a5ea5b350423f47d07639f74475afedad48cf41c0ad7a82ca13a3928af34f6"},
     {file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"},
 ]
+toml = [
+    {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
+    {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
+]
+typed-ast = [
+    {file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70"},
+    {file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487"},
+    {file = "typed_ast-1.4.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412"},
+    {file = "typed_ast-1.4.2-cp35-cp35m-win32.whl", hash = "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400"},
+    {file = "typed_ast-1.4.2-cp35-cp35m-win_amd64.whl", hash = "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606"},
+    {file = "typed_ast-1.4.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64"},
+    {file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07"},
+    {file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc"},
+    {file = "typed_ast-1.4.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a"},
+    {file = "typed_ast-1.4.2-cp36-cp36m-win32.whl", hash = "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151"},
+    {file = "typed_ast-1.4.2-cp36-cp36m-win_amd64.whl", hash = "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3"},
+    {file = "typed_ast-1.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41"},
+    {file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f"},
+    {file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581"},
+    {file = "typed_ast-1.4.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37"},
+    {file = "typed_ast-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd"},
+    {file = "typed_ast-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496"},
+    {file = "typed_ast-1.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc"},
+    {file = "typed_ast-1.4.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10"},
+    {file = "typed_ast-1.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea"},
+    {file = "typed_ast-1.4.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787"},
+    {file = "typed_ast-1.4.2-cp38-cp38-win32.whl", hash = "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2"},
+    {file = "typed_ast-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937"},
+    {file = "typed_ast-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1"},
+    {file = "typed_ast-1.4.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6"},
+    {file = "typed_ast-1.4.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166"},
+    {file = "typed_ast-1.4.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d"},
+    {file = "typed_ast-1.4.2-cp39-cp39-win32.whl", hash = "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b"},
+    {file = "typed_ast-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440"},
+    {file = "typed_ast-1.4.2.tar.gz", hash = "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a"},
+]
 typing-extensions = [
     {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"},
     {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"},

pylokid/__init__.py

@@ -1,7 +1,3 @@
 """
 Pylokid. From Mail to Lodur - all automated.
 """
-
-__version__ = "2.2.0"
-__git_version__ = "0"
-__url__ = "https://github.com/tobru/pylokid"

pylokid/__main__.py

@@ -4,4 +4,7 @@ Helper module to run not-installed version (via ``python3 -m pylokid``)
 from pylokid.main import main
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except KeyboardInterrupt:
+        print("Byebye")

pylokid/library/emailhandling.py

@@ -12,12 +12,13 @@ import imaplib
 
 _EMAIL_SUBJECTS = '(OR OR SUBJECT "Einsatzausdruck_FW" SUBJECT "Einsatzprotokoll" SUBJECT "Einsatzrapport" UNSEEN)'
 
+
 class EmailHandling:
-    """ Email handling """
+    """Email handling"""
 
     def __init__(self, server, username, password, mailbox, tmp_dir):
         self.logger = logging.getLogger(__name__)
-        self.logger.info('Connecting to IMAP server %s', server)
+        self.logger.info("Connecting to IMAP server %s", server)
 
         self.tmp_dir = tmp_dir
         socket.setdefaulttimeout(60)
@@ -27,86 +28,117 @@ class EmailHandling:
             self.imap.login(username, password)
             self.imap.select(mailbox, readonly=False)
         except Exception as err:
-            self.logger.error('IMAP connection failed - exiting: %s', str(err))
+            self.logger.error("IMAP connection failed - exiting: %s", str(err))
             raise SystemExit(1)
 
-        self.logger.info('IMAP connection successfull')
+        self.logger.info("IMAP connection successful")
 
     def search_emails(self):
-        """ searches for emails matching the configured subject """
+        """searches for emails matching the configured subject"""
 
-        self.logger.info('Searching for messages matching: %s', _EMAIL_SUBJECTS)
+        msg_ids = []
+
+        self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
         try:
             typ, msg_ids = self.imap.search(
                 None,
                 _EMAIL_SUBJECTS,
             )
-            if typ != 'OK':
-                self.logger.error('Error searching for matching messages')
+            if typ != "OK":
+                self.logger.error("Error searching for matching messages")
                 return False
         except imaplib.IMAP4.abort as err:
-            self.logger.error('IMAP search aborted - exiting: %s', str(err))
+            self.logger.error("IMAP search aborted - exiting: %s", str(err))
             raise SystemExit(1)
 
-        num_messages = len(msg_ids[0].split())
-        self.logger.info('Found %s matching messages', str(num_messages))
+        msg_list = msg_ids[0].split()
+        self.logger.info("Found %s matching messages", str(len(msg_list)))
 
-        return num_messages, msg_ids
+        # Retrieve subjects
+        msg_id_subject = {}
+        for msg in msg_list:
+            msg_id = msg.decode("utf-8")
+            typ, msg_data = self.imap.fetch(msg, "(BODY.PEEK[HEADER.FIELDS (SUBJECT)])")
+            if typ != "OK":
+                self.logger.error("Error fetching subject")
+                msg_id_subject[msg_id] = "unknown"
+            else:
+                try:
+                    mail = email.message_from_string(str(msg_data[0][1], "utf-8"))
+                    subject = mail["subject"]
+                    self.logger.info("Message ID %s has subject '%s'", msg_id, subject)
+                    msg_id_subject[msg_id] = subject
+                except TypeError:
+                    self.logger.error("Could not decode mail - %s", msg_data[0][1])
 
-    def store_attachments(self, msg_ids):
-        """ stores the attachments to filesystem and marks message as read """
+        # Deduplicate messages - usually the same message arrives multiple times
+        self.logger.info("Deduplicating messages")
+        temp = []
+        msg_id_subject_deduplicated = dict()
+        for key, val in msg_id_subject.items():
+            if val not in temp:
+                temp.append(val)
+                msg_id_subject_deduplicated[key] = val
+                self.logger.info(
+                    "Adding Message ID %s '%s' to list to process", key, val
+                )
+            else:
+                self.mark_seen(key, key)
 
-        data = {}
-        for msg_id in msg_ids[0].split():
-            # download message from imap
-            typ, msg_data = self.imap.fetch(msg_id, '(RFC822)')
+        return msg_id_subject_deduplicated
 
-            if typ != 'OK':
-                self.logger.error('Error fetching message')
-                continue
+    def store_attachment(self, msg_id):
+        """stores the attachment to filesystem"""
 
-            # extract attachment
-            for response_part in msg_data:
-                if isinstance(response_part, tuple):
-                    mail = email.message_from_string(str(response_part[1], 'utf-8'))
-                    subject = mail['subject']
-                    f_type, f_id = self.parse_subject(subject)
-                    self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
-                    for part in mail.walk():
-                        file_name = part.get_filename()
-                        if not file_name:
-                            self.logger.debug(
-                                'Most probably not an attachment as no filename found'
-                            )
-                            continue
+        # download message from imap
+        typ, msg_data = self.imap.fetch(msg_id, "(BODY.PEEK[])")
 
-                        self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
+        if typ != "OK":
+            self.logger.error("Error fetching message")
+            return None, None
 
-                        if bool(file_name):
-                            f_type, _ = self.parse_subject(subject)
-                            renamed_file_name = f_type + '_' + file_name
-                            # save attachment to filesystem
-                            file_path = os.path.join(self.tmp_dir, renamed_file_name)
+        # extract attachment
+        for response_part in msg_data:
+            if isinstance(response_part, tuple):
+                mail = email.message_from_string(str(response_part[1], "utf-8"))
+                subject = mail["subject"]
+                f_type, f_id = self.parse_subject(subject)
+                self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
+                for part in mail.walk():
+                    file_name = part.get_filename()
+                    if not file_name:
+                        self.logger.debug(
+                            "Most probably not an attachment as no filename found"
+                        )
+                        continue
 
-                            self.logger.info('[%s] Saving attachment to "%s"', f_id, file_path)
-                            if not os.path.isfile(file_path):
-                                file = open(file_path, 'wb')
-                                file.write(part.get_payload(decode=True))
-                                file.close()
+                    self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
 
-                            data[subject] = renamed_file_name
+                    if bool(file_name):
+                        f_type, _ = self.parse_subject(subject)
+                        renamed_file_name = f_type + "_" + file_name
+                        # save attachment to filesystem
+                        file_path = os.path.join(self.tmp_dir, renamed_file_name)
 
-            # mark as seen
-            self.logger.info('[%s] Marking message "%s" as seen', f_id, subject)
-            self.imap.store(msg_id, '+FLAGS', '(\\Seen)')
+                        self.logger.info(
+                            '[%s] Saving attachment to "%s"', f_id, file_path
+                        )
+                        if not os.path.isfile(file_path):
+                            file = open(file_path, "wb")
+                            file.write(part.get_payload(decode=True))
+                            file.close()
 
-        return data
+        return renamed_file_name
+
+    def mark_seen(self, msg_id, f_id):
+        self.logger.info("[%s] Marking E-Mail message as seen", f_id)
+        self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
 
     def parse_subject(self, subject):
-        """ extract f id and type from subject """
+        """extract f id and type from subject"""
 
         # This regex matches the subjects filtered already in IMAP search
-        parsed = re.search('([a-zA-Z_]*):? ?(F[0-9].*)?', subject)
+        parsed = re.search("([a-zA-Z_]*):? ?(F[0-9].*)?", subject)
         f_type = parsed.group(1)
         f_id = parsed.group(2)
 

pylokid/library/lodur.py

@@ -4,16 +4,18 @@
 
 import re
 import logging
-from datetime import datetime
-from datetime import timedelta
+import json
 import mechanicalsoup
+import pprint
+from datetime import datetime, timedelta
+
 
 class Lodur:
     """ Lodur """
 
     def __init__(self, url, username, password):
         self.logger = logging.getLogger(__name__)
-        self.logger.info('Connecting to Lodur')
+        self.logger.info("Connecting to Lodur")
 
         self.url = url
         self.username = username
@@ -24,219 +26,177 @@ class Lodur:
 
         self.login()
         if self.logged_in():
-            self.logger.info('Login to Lodur succeeded')
+            self.logger.info("Login to Lodur succeeded")
         else:
-            self.logger.fatal('Login to Lodur failed - exiting')
+            self.logger.fatal("Login to Lodur failed - exiting")
             raise SystemExit(1)
 
     def login(self):
         """ Login to lodur """
 
         # The login form is located in module number 9
-        self.browser.open(self.url + '?modul=9')
+        self.browser.open(self.url + "?modul=9")
         # only log in when not yed logged in
         if not self.logged_in():
             # open login page again as the logged_in function has navigated to another page
-            self.browser.open(self.url + '?modul=9')
+            self.browser.open(self.url + "?modul=9")
             self.browser.select_form()
 
-            self.browser['login_member_name'] = self.username
-            self.browser['login_member_pwd'] = self.password
+            self.browser["login_member_name"] = self.username
+            self.browser["login_member_pwd"] = self.password
             self.browser.submit_selected()
 
     def logged_in(self):
         """ check if logged in to lodur - session is valid """
         # Check if login succeeded by finding the img with
         # alt text LOGOUT on dashboard
-        self.browser.open(self.url + '?modul=16')
+        self.browser.open(self.url + "?modul=16")
         page = self.browser.get_current_page()
-        if page.find(alt='LOGOUT'):
-            self.logger.debug('Logged in')
+        if page.find(alt="LOGOUT"):
+            self.logger.debug("Logged in")
             return True
         else:
-            self.logger.debug('Not logged in')
+            self.logger.debug("Not logged in")
             return False
 
-    def einsatzprotokoll(self, f_id, pdf_data, webdav_client):
+    def get_einsatzrapport_id(self, f_id, state="open"):
+        """ Find ID of automatically created Einsatzrapport """
+
+        # Login to lodur
+        self.login()
+
+        # Browse to Einsatzrapport page
+        if state == "open":
+            self.browser.open("{}?modul=36".format(self.url))
+
+        try:
+            einsatzrapport_url = self.browser.find_link(link_regex=re.compile(f_id))
+        except mechanicalsoup.LinkNotFoundError:
+            self.logger.error("[%s] No Einsatzrapport found in Lodur", f_id)
+            return None
+
+        if einsatzrapport_url:
+            lodur_id = re.search(
+                ".*event=([0-9]{1,})&.*", einsatzrapport_url["href"]
+            ).group(1)
+            return lodur_id
+        else:
+            return None
+
+    def retrieve_form_data(self, lodur_id):
+        """ Retrieve all fields from an Einsatzrapport in Lodur """
+
+        # Login to lodur
+        self.login()
+
+        # Browse to Einsatzrapport page
+        self.browser.open(
+            "{}?modul=36&what=144&event={}&edit=1".format(self.url, lodur_id)
+        )
+
+        # Lodur doesn't simply store form field values in the form value field
+        # LOLNOPE - it is stored in javascript in the variable fdata
+        # And the data format used is just crap - including mixing of different data types
+        # WHAT DO THEY ACTUALLY THINK ABOUT THIS!!
+
+        # Retrieve all <script></script> tags from page
+        json_string = None
+        all_scripts = self.browser.page.find_all("script", type="text/javascript")
+        # Iterate over all tags to find the one containing fdata
+        for script in all_scripts:
+            # Some scripts don't have content - we're not interested in them
+            if script.contents:
+                # Search for "var fdata" in all scripts - if found, that's what we're looking for
+                content = script.contents[0]
+                if "var fdata" in content:
+                    # Cut out unnecessary "var fdata"
+                    json_string = content.replace("var fdata = ", "")
+
+        # Now let's parse that data into a data structure which helps
+        # in filling out the form and make it usable in Python
+        if json_string:
+            # Remove the last character which is a ;
+            usable = {}
+            for key, value in json.loads(json_string[:-1]).items():
+                # WHY DO THEY MIX DIFFERENT TYPES!
+                if isinstance(value, list):
+                    usable[key] = value[2]
+                elif isinstance(value, dict):
+                    usable[key] = value["2"]
+            return usable
+        else:
+            return None
+
+    def einsatzprotokoll(self, f_id, lodur_data, pdf_data, webdav_client):
         """ Prepare Einsatzprotokoll to be sent to Lodur """
 
-        # check if data is already sent to lodur - data contains lodur_id
-        lodur_data = webdav_client.get_lodur_data(f_id)
+        self.logger.info("[%s] Updating Lodur entry", f_id)
 
-        if lodur_data:
-            # einsatz available in Lodur - updating existing entry
-            self.logger.info('[%s] Lodur data found - updating entry', f_id)
+        # Complement existing form data
+        self.logger.info("[%s] Preparing form data for Einsatzprotokoll", f_id)
 
-            # when PDF parsing fails, pdf_data is false. fill with tbd when this happens
-            if pdf_data:
-                try:
-                    zh_fw_ausg = datetime.strptime(
-                        pdf_data['ausgerueckt'],
-                        '%H:%M:%S',
-                    )
-                    zh_am_schad = datetime.strptime(
-                        pdf_data['vorort'],
-                        '%H:%M:%S',
-                    )
-                except ValueError as err:
-                    self.logger.error('[%s] Date parsing failed: %s', f_id, err)
-                    zh_fw_ausg = datetime.now()
-                    zh_am_schad = datetime.now()
-            else:
-                # Do nothing when no PDF data - we don't have anything to do then
-                self.logger.error('[%s] No PDF data found - filling in dummy data', f_id)
-                zh_fw_ausg = datetime.now()
-                zh_am_schad = datetime.now()
-
-            # Complement existing form data
-            self.logger.info('[%s] Preparing form data for Einsatzprotokoll', f_id)
-            lodur_data['zh_fw_ausg_h'] = zh_fw_ausg.hour # 13. FW ausgerückt
-            lodur_data['zh_fw_ausg_m'] = zh_fw_ausg.minute # 13. FW ausgerückt
-            lodur_data['zh_am_schad_h'] = zh_am_schad.hour # 14. Am Schadenplatz
-            lodur_data['zh_am_schad_m'] = zh_am_schad.minute # 14. Am Schadenplatz
-            # The following fields are currently unknown as PDF parsing is hard for these
-            #lodur_data['zh_fw_einge_h'] = UNKNOWN, # 15. FW eingerückt
-            #lodur_data['zh_fw_einge_m'] = 'UNKNOWN' # 15. FW eingerückt
-            #lodur_data['eins_erst_h'] = 'UNKNOWN' # 16. Einsatzbereitschaft erstellt
-            #lodur_data['eins_erst_m'] = 'UNKNOWN' # 16. Einsatzbereitschaft erstellt
-
-            # Submit the form
-            self.submit_form_einsatzrapport(lodur_data)
-
-            # save lodur data to webdav
-            webdav_client.store_lodur_data(f_id, lodur_data)
-
-        else:
-            # einsatz not available in Lodur
-            self.logger.error('[%s] No lodur_id found')
-            return False
-
-    def einsatzrapport(self, f_id, pdf_data, webdav_client):
-        """ Prepare form in module 36 - Einsatzrapport """
-
-        # when PDF parsing fails, pdf_data is false. fill with placeholder when this happens
-        if pdf_data:
-            date = datetime.strptime(
-                pdf_data['datum'],
-                '%d.%m.%Y',
-            )
-            time = datetime.strptime(
-                pdf_data['zeit'],
-                '%H:%M',
-            )
-            eins_ereig = pdf_data['einsatz']
-            bemerkungen = pdf_data['bemerkungen'] + '\n' + pdf_data['disponierteeinheiten']
-            wer_ala = pdf_data['melder']
-            adr = pdf_data['ort']
-        else:
-            date = datetime.now()
-            time = datetime.now()
-            eins_ereig = 'UNKNOWN'
-            bemerkungen = 'UNKNOWN'
-            wer_ala = 'UNKNOWN'
-            adr = 'UNKNOWN'
-
-        # Prepare end date and time, can cross midnight
-        # We blindly add 1 hours - that's the usual length of an Einsatz
-        time_end = time + timedelta(hours=1)
-        # check if date is higher after adding 1 hour, this means we crossed midnight
-        if datetime.date(time_end) > datetime.date(time):
-            date_end = date + timedelta(days=1)
-        else:
-            date_end = date
-
-        # Fill in form data
-        self.logger.info('[%s] Preparing form data for Einsatzrapport', f_id)
-        lodur_data = {
-            'e_r_num': f_id, # 01. Einsatzrapportnummer
-            'eins_stat_kantone': '1', # 02. Einsatzart FKS
-            'emergency_concept_id': '2', # 03. Verrechnungsart
-            'ver_sart': 'ab', # 03. Verrechnungsart internal: ab, th, uh, ak, tt
-            'dtv_d': str(date.day), # 04. Datum von
-            'dtv_m': str(date.month), # 04. Datum von
-            'dtv_y': str(date.year), # 04. Datum von
-            'dtb_d': str(date_end.day), # 04. Datum bis
-            'dtb_m': str(date_end.month), # 04. Datum bis
-            'dtb_y': str(date_end.year), # 04. Datum bis
-            'ztv_h': str(time.hour), # 05. Zeit von
-            'ztv_m': str(time.minute), # 05. Zeit von
-            'ztb_h': str(time_end.hour), # 05. Zeit bis - we dont know yet the end time
-            'ztb_m': str(time_end.minute), # 05. Zeit bis - just add 1 hour and correct later
-            'e_ort_1': '306', # 06. Einsatzort: Urdorf 306, Birmensdorf 298
-            'eins_ereig': eins_ereig, # 07. Ereignis
-            'adr': adr, # 08. Adresse
-            'wer_ala': wer_ala, # 10. Wer hat alarmiert
-            'zh_alarmierung_h': str(time.hour), # 12. Alarmierung
-            'zh_alarmierung_m': str(time.minute), # 12. Alarmierung
-            'ang_sit': 'TBD1', # 17. Angetroffene Situation
-            'mn': 'TBD2', # 19. Massnahmen
-            'bk': bemerkungen, # 20. Bemerkungen
-            'en_kr_feuwehr': '1', # 21. Einsatzkräfte
-            'ali_io': '1', # 24. Alarmierung
-            'kopie_gvz': '1', # 31. Kopie innert 10 Tagen an GVZ
-            'mannschaftd_einsa': '88', # 32. Einsatzleiter|in
-        }
+        lodur_data["ztb_m"] = lodur_data[
+            "ztv_m"
+        ]  # 05. Zeit (copy minute from start to round up to 1h)
+        lodur_data[
+            "eins_ereig"
+        ] = f"{f_id} - {pdf_data['einsatz']} - {lodur_data['adr']}"  # 07. Ereignis
+        lodur_data["en_kr_feuwehr"] = "1"  # 21. Einsatzkräfte
+        lodur_data["ali_io"] = "1"  # 24. Alarmierung
+        lodur_data["keyword_els_zutreffend"] = "1"  # 25. Stichwort
+        lodur_data["address_zutreffend"] = "1"  # 26. Adresse zutreffend
+        lodur_data["kopie_gvz"] = "1"  # 31. Kopie innert 10 Tagen an GVZ
+        lodur_data["mannschaftd_einsa"] = "88"  # 32. Einsatzleiter|in
 
         # Submit the form
-        lodur_id, auto_num = self.submit_form_einsatzrapport(lodur_data)
+        self.submit_form_einsatzrapport(lodur_data)
 
-        # save lodur id and data to webdav
-        lodur_data['event_id'] = lodur_id
-        lodur_data['auto_num'] = auto_num
-        webdav_client.store_lodur_data(f_id, lodur_data)
+        # save lodur data to webdav
+        webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
 
-        return lodur_id
-
-    def einsatzrapport_alarmdepesche(self, f_id, file_path, webdav_client):
+    def upload_alarmdepesche(self, f_id, file_path, webdav_client):
         """ Upload a file to Alarmdepesche """
 
-        self.logger.info('[%s] Submitting file %s to Lodur "Alarmdepesche"', f_id, file_path)
+        self.logger.info(
+            '[%s] Submitting file %s to Lodur "Alarmdepesche"', f_id, file_path
+        )
 
         # Login to lodur
         self.login()
 
         # check if data is already sent to lodur - data contains lodur_id
-        lodur_id = webdav_client.get_lodur_data(f_id)['event_id']
+        lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
 
         # Prepare the form
-        self.browser.open('{}?modul=36&event={}&what=828'.format(self.url,lodur_id ))
-        frm_alarmdepesche = self.browser.select_form('#frm_alarmdepesche')
+        self.browser.open("{}?modul=36&event={}&what=828".format(self.url, lodur_id))
+        frm_alarmdepesche = self.browser.select_form("#frm_alarmdepesche")
 
         # Fill in form data
-        frm_alarmdepesche.set('alarmdepesche', file_path)
+        frm_alarmdepesche.set("alarmdepesche", file_path)
 
         # Submit the form
         self.browser.submit_selected()
-        self.logger.info('[%s] File uploaded to Lodur', f_id)
+        self.logger.info("[%s] File uploaded to Lodur", f_id)
 
-    def einsatzrapport_scan(self, f_id, file_path, webdav_client):
+    def einsatzrapport_scan(self, f_id, lodur_data, file_path, webdav_client):
         """ Prepare Einsatzrapport Scan to be sent to Lodur """
 
-        # check if data is already sent to lodur - data contains lodur_id
-        lodur_data = webdav_client.get_lodur_data(f_id)
+        # Complement existing form data
+        self.logger.info("[%s] Updating Lodur entry", f_id)
+        lodur_data[
+            "ang_sit"
+        ] = "Siehe Alarmdepesche - Einsatzrapport"  # 17. Angetroffene Situation
+        lodur_data["mn"] = "Siehe Alarmdepesche - Einsatzrapport"  # 19. Massnahmen
 
-        if lodur_data:
-            # einsatz available in Lodur - updating existing entry
-            self.logger.info('[%s] Lodur data found - updating entry', f_id)
+        # Submit the form
+        self.submit_form_einsatzrapport(lodur_data)
 
-            # Complement existing form data
-            self.logger.info('[%s] Preparing form data for Einsatzprotokoll', f_id)
-            lodur_data['ang_sit'] = 'Siehe Alarmdepesche - Einsatzrapport' # 17. Angetroffene Situation
-            lodur_data['mn'] = 'Siehe Alarmdepesche - Einsatzrapport' # 19. Massnahmen
-
-            # Submit the form
-            self.submit_form_einsatzrapport(lodur_data)
-
-            # Upload scan to Alarmdepesche
-            self.einsatzrapport_alarmdepesche(
-                f_id,
-                file_path,
-                webdav_client,
-            )
-        else:
-            # einsatz not available in Lodur
-            self.logger.error('[%s] No lodur_id found')
-            return False
+        # Upload scan to Alarmdepesche
+        self.upload_alarmdepesche(
+            f_id,
+            file_path,
+            webdav_client,
+        )
 
     def submit_form_einsatzrapport(self, lodur_data):
         """ Form in module 36 - Einsatzrapport """
@@ -244,65 +204,46 @@ class Lodur:
         # Login to lodur
         self.login()
 
-        # Prepare the form
-        if 'event_id' in lodur_data:
-            # existing entry to update
-            self.logger.info(
-                '[%s] Updating existing entry with ID %s',
-                lodur_data['e_r_num'],
-                lodur_data['event_id'],
-            )
-            self.browser.open(
-                self.url +
-                '?modul=36&what=144&edit=1&event=' +
-                lodur_data['event_id']
-            )
-        else:
-            self.logger.info('[%s] Creating new entry in Lodur', lodur_data['e_r_num'])
-            self.browser.open(
-                self.url +
-                '?modul=36'
-            )
+        f_id = lodur_data["e_r_num"]
 
-        self.browser.select_form('#einsatzrapport_main_form')
+        self.logger.info(
+            "[%s] Updating existing entry with ID %s",
+            f_id,
+            lodur_data["event_id"],
+        )
+
+        self.browser.open(
+            self.url + "?modul=36&what=144&edit=1&event=" + lodur_data["event_id"]
+        )
+
+        form = self.browser.select_form("#einsatzrapport_main_form")
 
         # Prepare the form data to be submitted
         for key, value in lodur_data.items():
+            # Not all keys in the parsed Lodur data are actually part of the form
             # Encode some of the fields so they are sent in correct format
             # Encoding bk causes some troubles - therefore we skip that - but it
             # would be good if it would be encoded as it can / will contain f.e.abs
             # Umlauts
             # AttributeError: 'bytes' object has no attribute 'parent'
-            self.logger.info('Form data: %s = %s', key, value)
-            if key in ('eins_ereig', 'adr', 'wer_ala'):
-                self.browser[key] = value.encode('iso-8859-1')
-            else:
-                self.browser[key] = value
+            try:
+                if key in ("eins_ereig", "adr", "wer_ala"):
+                    form.set(key, value.encode("iso-8859-1"))
+                else:
+                    form.set(key, value)
+                self.logger.debug("[%s] Set field %s to %s", f_id, key, value)
+            except mechanicalsoup.LinkNotFoundError as e:
+                self.logger.debug(
+                    "[%s] Could not set field %s to %s. Reason: %s",
+                    f_id,
+                    key,
+                    value,
+                    str(e),
+                )
 
         # Submit the form
-        self.logger.info('[%s] Submitting form Einsatzrapport', lodur_data['e_r_num'])
+        self.logger.info("[%s] Submitting form Einsatzrapport", lodur_data["e_r_num"])
         response = self.browser.submit_selected()
-        self.logger.info('[%s] Form Einsatzrapport submitted', lodur_data['e_r_num'])
+        self.logger.info("[%s] Form Einsatzrapport submitted", lodur_data["e_r_num"])
 
-        if 'event_id' in lodur_data:
-            return True
-        else:
-            # very ugly way to find the assigned event id by lodur
-            # lodur adds a script element at the bottom of the returned html
-            # with the location to reload the page - containing the assigned event id
-            # print(response.text)
-            lodur_id = re.search('modul=36&event=([0-9].*)&edit=1&what=144', response.text).group(1)
-            self.logger.info('[%s] Lodur assigned the event_id %s', lodur_data['e_r_num'], lodur_id)
-
-            # The hidden field auto_num is also needed for updating the form
-            # and it's written somewhere in javascript code - but not on the page
-            # delivered after the submission which contains the redirect URL
-            # It's only delivered in the next page. So we browse to this page now
-            content = self.browser.open(
-                self.url +
-                '?modul=36&edit=1&what=144&event=' + lodur_id
-            ).text
-            auto_num = re.search(r"\"([0-9]{4}\|[0-9]{1,3})\"", content).group(1)
-            self.logger.info('[%s] Lodur assigned the auto_num %s', lodur_data['e_r_num'], auto_num)
-
-            return lodur_id, auto_num
+        return True

pylokid/library/pdftotext.py

@@ -5,109 +5,157 @@
 import subprocess
 import logging
 
+
 class PDFParsing:
     """ PDF parsing """
 
     def __init__(self):
         self.logger = logging.getLogger(__name__)
-        self.logger.info('PDF parsing based on pdftotext loaded')
+        self.logger.info("PDF parsing based on pdftotext loaded")
 
     def extract(self, f_id, file, datafields):
 
-        self.logger.info('[%s] parsing PDF file %s', f_id, file)
+        self.logger.info("[%s] parsing PDF file %s", f_id, file)
 
         data = {}
 
         for field, coordinate in datafields.items():
 
             # x-coordinate of the crop area top left corner
-            x = coordinate['xMin']
+            x = coordinate["xMin"]
 
             # y-coordinate of the crop area top left corner
-            y = coordinate['yMin']
+            y = coordinate["yMin"]
 
             # width of crop area in pixels
-            w = coordinate['xMax'] - coordinate['xMin']
+            w = coordinate["xMax"] - coordinate["xMin"]
 
             # height of crop area in pixels
-            h = coordinate['yMax'] - coordinate['yMin']
+            h = coordinate["yMax"] - coordinate["yMin"]
 
-            self.logger.debug('[%s] Computed command for field %s: %s', f_id, field,
-              'pdftotext -f 1 -l 1 -x {} -y {} -W {} -H {}'.format(x,y,w,h)
+            self.logger.debug(
+                "[%s] Computed command for field %s: %s",
+                f_id,
+                field,
+                "pdftotext -f 1 -l 1 -x {} -y {} -W {} -H {}".format(x, y, w, h),
             )
 
-            scrapeddata = subprocess.Popen([
-                  '/usr/bin/pdftotext',
-                  '-f', '1',
-                  '-l', '1',
-                  '-x', str(x),
-                  '-y', str(y),
-                  '-W', str(w),
-                  '-H', str(h),
-                  file,
-                  '-'
+            scrapeddata = subprocess.Popen(
+                [
+                    "/usr/bin/pdftotext",
+                    "-f",
+                    "1",
+                    "-l",
+                    "1",
+                    "-x",
+                    str(x),
+                    "-y",
+                    str(y),
+                    "-W",
+                    str(w),
+                    "-H",
+                    str(h),
+                    file,
+                    "-",
                 ],
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
-                text=True)
+                text=True,
+            )
             stdout, _ = scrapeddata.communicate()
 
             ## TODO: fixup some fields (lowercase, remove unnecessary \n)
-            if 'edit' in coordinate and coordinate['edit'] == 'title':
+            if "edit" in coordinate and coordinate["edit"] == "title":
                 data[field] = stdout.rstrip().title()
             else:
                 data[field] = stdout.rstrip()
 
         # sanity check to see if we can correlate the f_id
-        if f_id == data['auftrag']:
-            self.logger.debug('[%s] ID matches in PDF', f_id)
+        if f_id == data["auftrag"]:
+            self.logger.debug("[%s] ID matches in PDF", f_id)
             return data
         else:
-            self.logger.error('[%s] ID does not match in PDF: "%s"', f_id, data['auftrag'])
+            self.logger.error(
+                '[%s] ID does not match in PDF: "%s"', f_id, data["auftrag"]
+            )
             return False
 
     def extract_einsatzausdruck(self, file, f_id):
         """ extracts information from Einsatzausdruck using external pdftotext """
 
-        self.logger.debug('[%s] Parsing PDF: %s', f_id, file)
+        self.logger.debug("[%s] Parsing PDF: %s", f_id, file)
 
         # Get them using 'pdftotext -bbox'
         # y = row
         # x = column: xMax 450 / 590 means full width
         coordinates = {
-            'auftrag': {
-                'xMin': 70, 'yMin': 47, 'xMax': 120,'yMax': 58,
+            "auftrag": {
+                "xMin": 70,
+                "yMin": 47,
+                "xMax": 120,
+                "yMax": 58,
             },
-            'datum': {
-                'xMin': 190, 'yMin': 47, 'xMax': 239, 'yMax': 58,
+            "datum": {
+                "xMin": 190,
+                "yMin": 47,
+                "xMax": 239,
+                "yMax": 58,
             },
-            'zeit': {
-                'xMin': 190, 'yMin': 59, 'xMax': 215, 'yMax': 70,
+            "zeit": {
+                "xMin": 190,
+                "yMin": 59,
+                "xMax": 215,
+                "yMax": 70,
             },
-            'melder': {
-                'xMin': 304, 'yMin': 47, 'xMax': 446, 'yMax': 70, 'edit': 'title'
+            "melder": {
+                "xMin": 304,
+                "yMin": 47,
+                "xMax": 446,
+                "yMax": 70,
+                "edit": "title",
             },
-            'erfasser':{
-                'xMin': 448, 'yMin': 59, 'xMax': 478, 'yMax': 70,
+            "erfasser": {
+                "xMin": 448,
+                "yMin": 59,
+                "xMax": 478,
+                "yMax": 70,
             },
             # big field until "Disponierte Einheiten"
-            'bemerkungen': {
-                'xMin': 28, 'yMin': 112, 'xMax': 590, 'yMax': 350,
+            "bemerkungen": {
+                "xMin": 28,
+                "yMin": 112,
+                "xMax": 590,
+                "yMax": 350,
             },
-            'disponierteeinheiten': {
-                'xMin': 28, 'yMin': 366, 'xMax': 450, 'yMax': 376,
+            "disponierteeinheiten": {
+                "xMin": 28,
+                "yMin": 366,
+                "xMax": 450,
+                "yMax": 376,
             },
-            'einsatz': {
-                'xMin': 76, 'yMin': 690, 'xMax': 450, 'yMax': 703,
+            "einsatz": {
+                "xMin": 76,
+                "yMin": 690,
+                "xMax": 450,
+                "yMax": 703,
             },
-            'sondersignal': {
-                'xMin': 76, 'yMin': 707, 'xMax': 450, 'yMax': 721,
+            "sondersignal": {
+                "xMin": 76,
+                "yMin": 707,
+                "xMax": 450,
+                "yMax": 721,
             },
-            'ort': {
-                'xMin': 76, 'yMin': 732, 'xMax': 590, 'yMax': 745,
+            "ort": {
+                "xMin": 76,
+                "yMin": 732,
+                "xMax": 590,
+                "yMax": 745,
             },
-            'hinweis': {
-                'xMin': 76, 'yMin': 773, 'xMax': 450, 'yMax': 787,
+            "hinweis": {
+                "xMin": 76,
+                "yMin": 773,
+                "xMax": 450,
+                "yMax": 787,
             },
         }
 
@@ -116,27 +164,42 @@ class PDFParsing:
     def extract_einsatzprotokoll(self, file, f_id):
         """ extracts information from Einsatzprotokoll using external pdftotext """
 
-        self.logger.debug('[%s] Parsing PDF: %s', f_id, file)
+        self.logger.debug("[%s] Parsing PDF: %s", f_id, file)
 
         # Get them using 'pdftotext -bbox'
         # y = row
         # x = column: xMax 450 / 590 means full width
         coordinates = {
-            'auftrag': {
-                'xMin': 192, 'yMin': 132, 'xMax': 238,'yMax': 142,
+            "auftrag": {
+                "xMin": 192,
+                "yMin": 132,
+                "xMax": 238,
+                "yMax": 142,
             },
-            'angelegt': {
-                'xMin': 192, 'yMin': 294, 'xMax': 226, 'yMax': 304,
+            "angelegt": {
+                "xMin": 192,
+                "yMin": 294,
+                "xMax": 226,
+                "yMax": 304,
             },
-            'dispo': {
-                'xMin': 192, 'yMin': 312, 'xMax': 226, 'yMax': 322,
+            "dispo": {
+                "xMin": 192,
+                "yMin": 312,
+                "xMax": 226,
+                "yMax": 322,
             },
-            'ausgerueckt': {
-                'xMin': 192, 'yMin': 331, 'xMax': 226, 'yMax': 341,
+            "ausgerueckt": {
+                "xMin": 192,
+                "yMin": 331,
+                "xMax": 226,
+                "yMax": 341,
             },
-            'vorort':{
-                'xMin': 192, 'yMin': 348, 'xMax': 226, 'yMax': 358,
+            "vorort": {
+                "xMin": 192,
+                "yMin": 348,
+                "xMax": 226,
+                "yMax": 358,
             },
         }
 
-        return self.extract(f_id, file, coordinates)
+        return self.extract(f_id, file, coordinates)

pylokid/library/webdav.py

@@ -4,17 +4,19 @@
 
 import os
 import json
+import re
 from datetime import datetime
 import logging
 import asyncio
 import aioeasywebdav
 
+
 class WebDav:
-    """ WebDav Client """
+    """WebDav Client"""
 
     def __init__(self, url, username, password, webdav_basedir, tmp_dir):
         self.logger = logging.getLogger(__name__)
-        self.logger.info('Connecting to WebDAV server %s', url)
+        self.logger.info("Connecting to WebDAV server %s", url)
 
         self.loop = asyncio.get_event_loop()
         self.webdav_basedir = webdav_basedir
@@ -26,18 +28,20 @@ class WebDav:
                 password=password,
             )
         except:
-            self.logger.error('WebDAV connection failed - exiting')
+            self.logger.error("WebDAV connection failed - exiting")
 
-        self.logger.info('WebDAV connection successfull')
+        self.logger.info("WebDAV connection successfull")
 
-    def upload(self, file_name, f_id, check_exists = True):
-        """ uploads a file to webdav - checks for existence before doing so """
+    def upload(self, file_name, f_id, check_exists=True):
+        """uploads a file to webdav - checks for existence before doing so"""
 
         # upload with webdav
         if f_id == None:
             remote_upload_dir = self.webdav_basedir + "/Inbox"
         else:
-            remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
+            remote_upload_dir = (
+                self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
+            )
 
         self.logger.info('[%s] Uploading file to WebDAV "%s"', f_id, remote_upload_dir)
 
@@ -47,7 +51,9 @@ class WebDav:
             self.loop.run_until_complete(self.webdav.mkdir(remote_upload_dir))
 
         remote_file_path = remote_upload_dir + "/" + file_name
-        if check_exists and self.loop.run_until_complete(self.webdav.exists(remote_file_path)):
+        if check_exists and self.loop.run_until_complete(
+            self.webdav.exists(remote_file_path)
+        ):
             self.logger.info('[%s] File "%s" already exists on WebDAV', f_id, file_name)
         else:
             self.loop.run_until_complete(
@@ -58,50 +64,84 @@ class WebDav:
             )
             self.logger.info('[%s] File "%s" uploaded', f_id, file_name)
 
-    def einsatz_exists(self, f_id):
-        """ check if an einsatz is already created """
+    def delete(self, file_name):
+        """delete file on webdav"""
+        self.loop.run_until_complete(self.webdav.delete(file_name))
 
-        remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
+    def einsatz_exists(self, f_id):
+        """check if an einsatz is already created"""
+
+        remote_upload_dir = (
+            self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
+        )
         if self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)):
-            self.logger.info('[%s] Einsatz exists on WebDAV', f_id)
+            self.logger.info("[%s] Einsatz exists on WebDAV", f_id)
             return True
         else:
             return False
 
-    def store_lodur_data(self, f_id, lodur_data):
-        """ stores lodur data on webdav """
+    def einsatzrapport_inbox_check(self, tmp_dir):
+        """check if an einsatzrapport with an f_id exists in the WebDav Inbox and process it"""
+
+        rapporte_to_process = []
+
+        filelist = self.loop.run_until_complete(
+            self.webdav.ls(f"{self.webdav_basedir}/Inbox")
+        )
+        for file in filelist:
+            full_path = file[0]
+            parsed = re.search(".*Einsatzrapport_(F[0-9].*)\.pdf", full_path)
+            if parsed:
+                f_id = parsed.group(1)
+                self.logger.info("[%s] Found %s - Downloading", f_id, full_path)
+
+                # Download PDF for later processing
+                self.loop.run_until_complete(
+                    self.webdav.download(
+                        full_path, f"{tmp_dir}/Einsatzrapport_{f_id}.pdf"
+                    )
+                )
+                rapporte_to_process.append(f_id)
+
+        return rapporte_to_process
+
+    def store_data(self, f_id, file_name, data):
+        """stores data on webdav"""
 
-        file_name = f_id + '_lodur.json'
         file_path = os.path.join(self.tmp_dir, file_name)
 
-        file = open(file_path, 'w')
-        file.write(json.dumps(lodur_data))
+        file = open(file_path, "w")
+        file.write(json.dumps(data))
         file.close()
 
-        self.logger.info('[%s] Stored Lodur data locally in %s', f_id, file_path)
+        self.logger.info("[%s] Stored data locally in %s", f_id, file_path)
         self.upload(file_name, f_id, False)
 
-    def get_lodur_data(self, f_id):
-        """ gets lodur data if it exists """
+    def get_lodur_data(self, f_id, filetype="_lodur.json"):
+        """gets lodur data if it exists"""
 
-        file_name = f_id + '_lodur.json'
+        file_name = f_id + filetype
         file_path = os.path.join(self.tmp_dir, file_name)
 
         # first check if we already have it locally - then check on webdav
         if os.path.isfile(file_path):
-            with open(file_path, 'r') as content:
+            with open(file_path, "r") as content:
                 lodur_data = json.loads(content.read())
-                self.logger.info('[%s] Found Lodur data locally', f_id)
+                self.logger.info("[%s] Found Lodur data locally", f_id)
                 return lodur_data
         else:
-            remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
-            remote_file_path = remote_upload_dir + '/' + file_name
+            remote_upload_dir = (
+                self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
+            )
+            remote_file_path = remote_upload_dir + "/" + file_name
             if self.loop.run_until_complete(self.webdav.exists(remote_file_path)):
-                self.loop.run_until_complete(self.webdav.download(remote_file_path, file_path))
-                with open(file_path, 'r') as content:
+                self.loop.run_until_complete(
+                    self.webdav.download(remote_file_path, file_path)
+                )
+                with open(file_path, "r") as content:
                     lodur_data = json.loads(content.read())
-                    self.logger.info('[%s] Found Lodur data on WebDAV', f_id)
+                    self.logger.info("[%s] Found Lodur data on WebDAV", f_id)
                     return lodur_data
             else:
-                self.logger.info('[%s] No existing Lodur data found', f_id)
+                self.logger.info("[%s] No existing Lodur data found", f_id)
                 return False

pylokid/main.py

@@ -7,6 +7,7 @@ import os
 import time
 
 import requests
+from importlib.metadata import version
 from dotenv import find_dotenv, load_dotenv
 from pushover import Client
 
@@ -34,18 +35,18 @@ LODUR_BASE_URL = os.getenv("LODUR_BASE_URL")
 HEARTBEAT_URL = os.getenv("HEARTBEAT_URL")
 PUSHOVER_API_TOKEN = os.getenv("PUSHOVER_API_TOKEN")
 PUSHOVER_USER_KEY = os.getenv("PUSHOVER_USER_KEY")
-PYLOKID_VERSION = "2.2.0"
+
 
 def main():
-    """ main """
+    """main"""
 
     # Logging configuration
     logging.basicConfig(
         level=logging.INFO,
-        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        format="%(asctime)s - %(levelname)s - %(message)s",
     )
-    logger = logging.getLogger('pylokid')
-    logger.info('Starting pylokid version %s', PYLOKID_VERSION)
+    logger = logging.getLogger("pylokid")
+    logger.info("Starting pylokid version %s", version("pylokid"))
 
     # Initialize IMAP Session
     imap_client = EmailHandling(
@@ -73,154 +74,227 @@ def main():
     )
 
     # Initialize Pushover
-    pushover = Client(
-        user_key=PUSHOVER_USER_KEY,
-        api_token=PUSHOVER_API_TOKEN
-    )
+    pushover = Client(user_key=PUSHOVER_USER_KEY, api_token=PUSHOVER_API_TOKEN)
 
     # Initialize PDF Parser
     pdf = PDFParsing()
 
     # Main Loop
+    logger.info("** Starting to process E-Mails **")
     while True:
-        attachments = {}
-        num_messages, msg_ids = imap_client.search_emails()
-        if num_messages:
-            attachments = imap_client.store_attachments(msg_ids)
 
-        if attachments:
-            for subject in attachments:
-                f_type, f_id = imap_client.parse_subject(subject)
-                file_name = attachments[subject]
+        # Search for matchting E-Mails
+        msg_ids = imap_client.search_emails()
 
-                # Upload file to cloud
-                webdav_client.upload(file_name, f_id)
+        for msg, subject in msg_ids.items():
+            logger.info("Processing IMAP message ID %s", msg)
+            file_name = imap_client.store_attachment(msg)
 
-                # Take actions - depending on the type
-                if f_type == 'Einsatzausdruck_FW':
-                    logger.info('[%s] Processing type %s', f_id, f_type)
-                    lodur_data = webdav_client.get_lodur_data(f_id)
+            # If the message couldn't be parsed, skip to next message
+            if not file_name:
+                pass
 
-                    if lodur_data:
+            # Figure out event type and F ID by parsing the subject
+            f_type, f_id = imap_client.parse_subject(subject)
+
+            # Upload extracted attachment to cloud
+            webdav_client.upload(file_name, f_id)
+
+            # Take actions - depending on the type
+            if f_type == "Einsatzausdruck_FW":
+                logger.info("[%s] Processing type %s", f_id, f_type)
+
+                # Check if the PDF isn't already parsed
+                if webdav_client.get_lodur_data(f_id, "_pdf.json"):
+                    logger.info("[%s] PDF already parsed", f_id)
+                else:
+                    # Extract information from PDF
+                    pdf_data = pdf.extract_einsatzausdruck(
+                        os.path.join(TMP_DIR, file_name),
+                        f_id,
+                    )
+
+                    # publish Einsatz on Pushover
+                    logger.info("[%s] Publishing message on Pushover", f_id)
+                    pushover.send_message(
+                        "<b>{}</b>\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
+                            pdf_data["einsatz"],
+                            pdf_data["ort"],
+                            pdf_data["melder"].replace("\n", " "),
+                            pdf_data["hinweis"],
+                            pdf_data["sondersignal"],
+                            pdf_data["bemerkungen"],
+                            pdf_data["disponierteeinheiten"],
+                        ),
+                        title="Feuerwehr Einsatz - {}".format(f_id),
+                        url="https://www.google.com/maps/search/?api=1&query={}".format(
+                            pdf_data["ort"]
+                        ),
+                        url_title="Ort auf Karte suchen",
+                        html=1,
+                    )
+
+                    # Upload extracted data to cloud
+                    webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data)
+
+                if webdav_client.get_lodur_data(f_id):
+                    logger.info("[%s] Lodur data already retrieved", f_id)
+                    # Marking message as seen, no need to reprocess again
+                    imap_client.mark_seen(msg, f_id)
+                else:
+                    # Retrieve data from Lodur
+                    lodur_id = lodur_client.get_einsatzrapport_id(f_id)
+                    if lodur_id:
                         logger.info(
-                            '[%s] Einsatzrapport already created in Lodur', f_id
-                        )
-                        # Upload Alarmdepesche as it could contain more information
-                        # than the first one
-                        lodur_client.einsatzrapport_alarmdepesche(
+                            "[%s] Einsatzrapport available in Lodur with ID %s",
                             f_id,
-                            os.path.join(TMP_DIR, file_name),
-                            webdav_client,
+                            lodur_id,
                         )
-
-                    else:
-                        ## Here we get the initial Einsatzauftrag - Time to run
-                        # get as many information from PDF as possible
-                        pdf_file = os.path.join(TMP_DIR, file_name)
-                        pdf_data = pdf.extract_einsatzausdruck(
-                            pdf_file,
-                            f_id,
-                        )
-
-                        # publish Einsatz on Pushover
                         logger.info(
-                            '[%s] Publishing message on Pushover', f_id
-                        )
-                        pushover.send_message(
-                            "Einsatz {} eröffnet: {}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
-                                f_id,
-                                pdf_data['einsatz'],
-                                pdf_data['ort'],
-                                pdf_data['melder'].replace('\n',' '),
-                                pdf_data['hinweis'],
-                                pdf_data['sondersignal'],
-                                pdf_data['disponierteeinheiten'],
-                                pdf_data['bemerkungen'],
-                            ),
-                            title="Feuerwehr Einsatz",
-                            url="https://www.google.com/maps/search/?api=1&query={}".format(pdf_data['ort']),
-                            url_title="Ort auf Karte suchen"
+                            "%s?modul=36&what=144&event=%s&edit=1",
+                            LODUR_BASE_URL,
+                            lodur_id,
                         )
 
-                        # create new Einsatzrapport in Lodur
-                        lodur_client.einsatzrapport(
-                            f_id,
-                            pdf_data,
-                            webdav_client,
-                        )
+                        lodur_data = lodur_client.retrieve_form_data(lodur_id)
+                        webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
 
                         # upload Alarmdepesche PDF to Lodur
-                        lodur_client.einsatzrapport_alarmdepesche(
+                        lodur_client.upload_alarmdepesche(
                             f_id,
                             os.path.join(TMP_DIR, file_name),
                             webdav_client,
                         )
 
-                elif f_type == 'Einsatzprotokoll':
-                    logger.info('[%s] Processing type %s', f_id, f_type)
-
-                    lodur_data = webdav_client.get_lodur_data(f_id)
-                    if lodur_data:
-                        # Upload Einsatzprotokoll to Lodur
-                        lodur_client.einsatzrapport_alarmdepesche(
-                            f_id,
-                            os.path.join(TMP_DIR, file_name),
-                            webdav_client,
-                        )
-
-                        # Parse the Einsatzprotokoll PDF
-                        pdf_file = os.path.join(TMP_DIR, file_name)
-                        pdf_data = pdf.extract_einsatzprotokoll(
-                            pdf_file,
-                            f_id,
-                        )
-
-                        # Update entry in Lodur with parsed PDF data
-                        lodur_client.einsatzprotokoll(f_id, pdf_data, webdav_client)
-
-                        # Einsatz finished - publish on pushover
-                        logger.info(
-                            '[%s] Publishing message on Pushover', f_id
-                        )
-                        pushover.send_message(
-                            "Einsatz {} beendet".format(f_id),
-                            title="Feuerwehr Einsatz beendet",
-                        )
-
+                        # Marking message as seen, no need to reprocess again
+                        imap_client.mark_seen(msg, f_id)
                     else:
-                        logger.error(
-                            '[%s] Cannot process Einsatzprotokoll as there is no Lodur ID',
-                            f_id
-                        )
+                        logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
 
-                # This is usually a scan from the Depot printer
-                elif f_type == 'Einsatzrapport':
-                    logger.info('[%s] Processing type %s', f_id, f_type)
+            elif f_type == "Einsatzprotokoll":
 
-                    # Attach scan in Lodur if f_id is available
-                    if f_id != None:
-                        pdf_file = os.path.join(TMP_DIR, file_name)
-                        lodur_client.einsatzrapport_scan(f_id, pdf_file, webdav_client)
+                lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
+                pdf_data = webdav_client.get_lodur_data(f_id, "_pdf.json")
+                logger.info(
+                    "[%s] Processing type %s with Lodur ID %s",
+                    f_id,
+                    f_type,
+                    lodur_id,
+                )
 
-                    logger.info(
-                        '[%s] Publishing message on Pushover', f_id
+                # Retrieve Lodur data again and store it in Webdav
+                lodur_data = lodur_client.retrieve_form_data(lodur_id)
+                webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
+
+                if (
+                    "aut_created_report" in lodur_data
+                    and lodur_data["aut_created_report"] == "finished"
+                ):
+                    logger.info("[%s] Record in Lodur ready to be updated", f_id)
+
+                    # Upload Einsatzprotokoll to Lodur
+                    lodur_client.upload_alarmdepesche(
+                        f_id,
+                        os.path.join(TMP_DIR, file_name),
+                        webdav_client,
                     )
 
+                    # Update entry in Lodur
+                    lodur_client.einsatzprotokoll(
+                        f_id, lodur_data, pdf_data, webdav_client
+                    )
+
+                    # Correct time on AdF to round up to one hour
+                    # curl 'https://lodur-zh.ch/urdorf/tunnel.php?modul=36&what=1082'
+                    # -H 'Referer: https://lodur-zh.ch/urdorf/index.php?modul=36&what=145&event=3485&edit=1'
+                    # -H 'Cookie: PHPSESSID=85pnahp3q83apv7qsbi8hrj5g7'
+                    # --data-raw 'dtv_d=03&dtv_m=04&dtv_y=2021&dtb_d=03&dtb_m=04&dtb_y=2021&ztv_h=18&ztv_m=26&ztb_h=19&ztb_m=26'
+                    # ztb_m -> same as ztv_m
+
+                    # Einsatz finished - publish on pushover
+                    logger.info("[%s] Publishing message on Pushover", f_id)
                     pushover.send_message(
-                        "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
-                        title="Feuerwehr Scan bearbeitet",
+                        "Einsatz beendet",
+                        title="Feuerwehr Einsatz beendet - {}".format(f_id),
+                    )
+
+                    # Marking message as seen, no need to reprocess again
+                    imap_client.mark_seen(msg, f_id)
+
+                else:
+                    logger.warn(
+                        "[%s] Record in Lodur NOT ready yet to be updated", f_id
+                    )
+
+            # This is usually a scan from the Depot printer
+            elif f_type == "Einsatzrapport":
+
+                logger.info("[%s] Processing type %s", f_id, f_type)
+
+                # Attach scan in Lodur if f_id is available
+                # f_id can be empty when scan was misconfigured
+                if f_id != None and len(f_id) > 8:
+                    lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
+                    # Retrieve Lodur data again and store it in Webdav
+                    lodur_data = lodur_client.retrieve_form_data(lodur_id)
+                    webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
+                    lodur_client.einsatzrapport_scan(
+                        f_id,
+                        lodur_data,
+                        os.path.join(TMP_DIR, file_name),
+                        webdav_client,
                     )
                 else:
-                    logger.error('[%s] Unknown type: %s', f_id, f_type)
+                    f_id = f_type
+
+                logger.info("[%s] Publishing message on Pushover", f_id)
+
+                pushover.send_message(
+                    "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
+                    title="Feuerwehr Scan bearbeitet - {}".format(f_id),
+                )
+
+                # Marking message as seen, no need to reprocess again
+                imap_client.mark_seen(msg, f_id)
+
+            else:
+                logger.error("[%s] Unknown type: %s", f_id, f_type)
+
+        logger.info("Checking WebDav Inbox folder for Einsatzrapporte to process")
+        rapporte_to_process = webdav_client.einsatzrapport_inbox_check(TMP_DIR)
+
+        if rapporte_to_process:
+            for f_id_rapport in rapporte_to_process:
+                filename = f"Einsatzrapport_{f_id_rapport}.pdf"
+                local_file = os.path.join(TMP_DIR, filename)
+                # Upload to f_id folder
+                webdav_client.upload(filename, f_id_rapport)
+
+                # Process it for Lodur
+                lodur_id = webdav_client.get_lodur_data(f_id_rapport)["event_id"]
+                # Retrieve Lodur data again and store it in Webdav
+                lodur_data = lodur_client.retrieve_form_data(lodur_id)
+                webdav_client.store_data(
+                    f_id_rapport, f_id_rapport + "_lodur.json", lodur_data
+                )
+                lodur_client.einsatzrapport_scan(
+                    f_id_rapport,
+                    lodur_data,
+                    local_file,
+                    webdav_client,
+                )
+                # Delete processed Einsatzrapport from Inbox and local temp dir
+                logger.info("Einsatzrapport processed - deleting file in Inbox")
+                webdav_client.delete(f"{WEBDAV_BASEDIR}/Inbox/{filename}")
+                os.remove(local_file)
+
+                pushover.send_message(
+                    f"Einsatzrapport {f_id_rapport} wurde bearbeitet.",
+                    title=f"Feuerwehr Einsatzrapport bearbeitet - {f_id_rapport}",
+                )
 
         # send heartbeat
         requests.get(HEARTBEAT_URL)
         # repeat every
-        logger.info('Waiting %s seconds until next check', IMAP_CHECK_INTERVAL)
+        logger.info("Waiting %s seconds until next check", IMAP_CHECK_INTERVAL)
         time.sleep(int(IMAP_CHECK_INTERVAL))
-
-if __name__ == '__main__':
-    try:
-        main()
-    except KeyboardInterrupt:
-        print("Byebye")

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pylokid"
-version = "2.2.0"
+version = "3.2.0"
 description = ""
 authors = ["Tobias Brunner <tobias@tobru.ch>"]
 license = "MIT"
@@ -14,6 +14,8 @@ python-pushover = "^0.4"
 MechanicalSoup = "^1.0.0"
 
 [tool.poetry.dev-dependencies]
+flake8 = "^3.8.4"
+black = "^20.8b1"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

test_pdftotext.py

@@ -1,30 +0,0 @@
-import re
-import logging
-from pprint import pprint
-from pathlib import Path
-from library.pdftotext import PDFParsing
-
-PATH = '/home/tobru/Documents/Feuerwehr/Stab/Fourier/Einsatzdepeschen/2019'
-
-logging.basicConfig(
-    level=logging.INFO,
-    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-)
-
-PDF = PDFParsing()
-
-for path in Path(PATH).glob('**/Einsatzausdruck*.pdf'):
-    file = str(path)
-    print(file)
-    f_id = re.search('.*(F[0-9]{8})_.*', file).group(1)
-    print(f_id)
-    pprint(PDF.extract_einsatzausdruck(file, f_id))
-
-"""
-for path in Path(PATH).glob('**/Einsatzprotokoll*.pdf'):
-    file = str(path)
-    print(file)
-    f_id = re.search('.*(F[0-9]{8})_.*', file).group(1)
-    print(f_id)
-    pprint(PDF.extract_einsatzprotokoll(file, f_id))
-"""