Browse Source

major update to reuse already existing records

Lodur since some time automatically creates Einsatzrapporte via an API
from SRZ/GVZ. One of the main features of Pylokid was to exactly do
that. With that new change this isn't necessary anymore. Pylokid has
been amended to find the pre-existing entry and work with that -
enhancing it with any additional information missing and uploads PDFs to
the right place.

While at it a very small modernization has been made and the project
moved to use Poetry and Black formatting. But it's still the same ugly
code - to reflect Lodur.
pull/2/head
Tobias Brunner 2 months ago
parent
commit
9c8c5396c8
  1. 231
      poetry.lock
  2. 2
      pylokid/__init__.py
  3. 52
      pylokid/library/emailhandling.py
  4. 366
      pylokid/library/lodur.py
  5. 205
      pylokid/library/pdftotext.py
  6. 58
      pylokid/library/webdav.py
  7. 202
      pylokid/main.py
  8. 4
      pyproject.toml
  9. 30
      test_pdftotext.py

231
poetry.lock

@ -30,6 +30,14 @@ yarl = ">=1.0,<2.0"
[package.extras]
speedups = ["aiodns", "brotlipy", "cchardet"]
[[package]]
name = "appdirs"
version = "1.4.4"
description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "async-timeout"
version = "3.0.1"
@ -67,6 +75,28 @@ soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""}
html5lib = ["html5lib"]
lxml = ["lxml"]
[[package]]
name = "black"
version = "20.8b1"
description = "The uncompromising code formatter."
category = "dev"
optional = false
python-versions = ">=3.6"
[package.dependencies]
appdirs = "*"
click = ">=7.1.2"
mypy-extensions = ">=0.4.3"
pathspec = ">=0.6,<1"
regex = ">=2020.1.8"
toml = ">=0.10.1"
typed-ast = ">=1.4.0"
typing-extensions = ">=3.7.4"
[package.extras]
colorama = ["colorama (>=0.4.3)"]
d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
[[package]]
name = "certifi"
version = "2020.12.5"
@ -83,6 +113,27 @@ category = "main"
optional = false
python-versions = "*"
[[package]]
name = "click"
version = "7.1.2"
description = "Composable command line interface toolkit"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[[package]]
name = "flake8"
version = "3.8.4"
description = "the modular source code checker: pep8 pyflakes and co"
category = "dev"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
[package.dependencies]
mccabe = ">=0.6.0,<0.7.0"
pycodestyle = ">=2.6.0a1,<2.7.0"
pyflakes = ">=2.2.0,<2.3.0"
[[package]]
name = "idna"
version = "2.10"
@ -105,6 +156,14 @@ html5 = ["html5lib"]
htmlsoup = ["beautifulsoup4"]
source = ["Cython (>=0.29.7)"]
[[package]]
name = "mccabe"
version = "0.6.1"
description = "McCabe checker, plugin for flake8"
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "mechanicalsoup"
version = "1.0.0"
@ -127,6 +186,38 @@ category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "mypy-extensions"
version = "0.4.3"
description = "Experimental type system extensions for programs checked with the mypy typechecker."
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "pathspec"
version = "0.8.1"
description = "Utility library for gitignore style pattern matching of file paths."
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[[package]]
name = "pycodestyle"
version = "2.6.0"
description = "Python style guide checker"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "pyflakes"
version = "2.2.0"
description = "passive checker of Python programs"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "python-dotenv"
version = "0.15.0"
@ -149,6 +240,14 @@ python-versions = "*"
[package.dependencies]
requests = ">=1.0"
[[package]]
name = "regex"
version = "2020.11.13"
description = "Alternative regular expression module, to replace re."
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "requests"
version = "2.25.1"
@ -194,6 +293,22 @@ category = "main"
optional = false
python-versions = ">=3.6"
[[package]]
name = "toml"
version = "0.10.2"
description = "Python Library for Tom's Obvious, Minimal Language"
category = "dev"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "typed-ast"
version = "1.4.2"
description = "a fork of Python 2 and 3 ast modules with type comment support"
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "typing-extensions"
version = "3.7.4.3"
@ -230,7 +345,7 @@ multidict = ">=4.0"
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "c5fa2a2589b88adb6a9bf94ec4a88e0e6b9528e40837cf6b2cf045a3e072d6d4"
content-hash = "87029b7a633e874d04d308355076802812613a9b93a03770d10ab6b9fcbc5b44"
[metadata.files]
aioeasywebdav = [
@ -276,6 +391,10 @@ aiohttp = [
{file = "aiohttp-3.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7"},
{file = "aiohttp-3.7.3.tar.gz", hash = "sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4"},
]
appdirs = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
]
async-timeout = [
{file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"},
{file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"},
@ -289,6 +408,9 @@ beautifulsoup4 = [
{file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"},
{file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"},
]
black = [
{file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"},
]
certifi = [
{file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"},
{file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"},
@ -297,6 +419,14 @@ chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
{file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
]
click = [
{file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
{file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
]
flake8 = [
{file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"},
{file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"},
]
idna = [
{file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
{file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
@ -340,6 +470,10 @@ lxml = [
{file = "lxml-4.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf"},
{file = "lxml-4.6.2.tar.gz", hash = "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc"},
]
mccabe = [
{file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
{file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
]
mechanicalsoup = [
{file = "MechanicalSoup-1.0.0-py2.py3-none-any.whl", hash = "sha256:2ed9a494c144fb2c262408dcbd5c79e1ef325a7426f1fa3a3443eaef133b5f77"},
{file = "MechanicalSoup-1.0.0.tar.gz", hash = "sha256:37d3b15c1957917d3ae171561e77f4dd4c08c35eb4500b8781f6e7e1bb6c4d07"},
@ -383,6 +517,22 @@ multidict = [
{file = "multidict-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359"},
{file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"},
]
mypy-extensions = [
{file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
{file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
]
pathspec = [
{file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"},
{file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"},
]
pycodestyle = [
{file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"},
{file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"},
]
pyflakes = [
{file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"},
{file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"},
]
python-dotenv = [
{file = "python-dotenv-0.15.0.tar.gz", hash = "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0"},
{file = "python_dotenv-0.15.0-py2.py3-none-any.whl", hash = "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e"},
@ -390,6 +540,49 @@ python-dotenv = [
python-pushover = [
{file = "python-pushover-0.4.tar.gz", hash = "sha256:dee1b1344fb8a5874365fc9f886d9cbc7775536629999be54dfa60177cf80810"},
]
regex = [
{file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6"},
{file = "regex-2020.11.13-cp36-cp36m-win32.whl", hash = "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e"},
{file = "regex-2020.11.13-cp36-cp36m-win_amd64.whl", hash = "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884"},
{file = "regex-2020.11.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538"},
{file = "regex-2020.11.13-cp37-cp37m-win32.whl", hash = "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4"},
{file = "regex-2020.11.13-cp37-cp37m-win_amd64.whl", hash = "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444"},
{file = "regex-2020.11.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f"},
{file = "regex-2020.11.13-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d"},
{file = "regex-2020.11.13-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b"},
{file = "regex-2020.11.13-cp38-cp38-win32.whl", hash = "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c"},
{file = "regex-2020.11.13-cp38-cp38-win_amd64.whl", hash = "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683"},
{file = "regex-2020.11.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc"},
{file = "regex-2020.11.13-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364"},
{file = "regex-2020.11.13-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c"},
{file = "regex-2020.11.13-cp39-cp39-win32.whl", hash = "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"},
{file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"},
{file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"},
]
requests = [
{file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"},
{file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"},
@ -412,6 +605,42 @@ soupsieve = [
{file = "soupsieve-2.2-py3-none-any.whl", hash = "sha256:d3a5ea5b350423f47d07639f74475afedad48cf41c0ad7a82ca13a3928af34f6"},
{file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"},
]
toml = [
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
]
typed-ast = [
{file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70"},
{file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487"},
{file = "typed_ast-1.4.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412"},
{file = "typed_ast-1.4.2-cp35-cp35m-win32.whl", hash = "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400"},
{file = "typed_ast-1.4.2-cp35-cp35m-win_amd64.whl", hash = "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606"},
{file = "typed_ast-1.4.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64"},
{file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07"},
{file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc"},
{file = "typed_ast-1.4.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a"},
{file = "typed_ast-1.4.2-cp36-cp36m-win32.whl", hash = "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151"},
{file = "typed_ast-1.4.2-cp36-cp36m-win_amd64.whl", hash = "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3"},
{file = "typed_ast-1.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41"},
{file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f"},
{file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581"},
{file = "typed_ast-1.4.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37"},
{file = "typed_ast-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd"},
{file = "typed_ast-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496"},
{file = "typed_ast-1.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc"},
{file = "typed_ast-1.4.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10"},
{file = "typed_ast-1.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea"},
{file = "typed_ast-1.4.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787"},
{file = "typed_ast-1.4.2-cp38-cp38-win32.whl", hash = "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2"},
{file = "typed_ast-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937"},
{file = "typed_ast-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1"},
{file = "typed_ast-1.4.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6"},
{file = "typed_ast-1.4.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166"},
{file = "typed_ast-1.4.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d"},
{file = "typed_ast-1.4.2-cp39-cp39-win32.whl", hash = "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b"},
{file = "typed_ast-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440"},
{file = "typed_ast-1.4.2.tar.gz", hash = "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a"},
]
typing-extensions = [
{file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"},
{file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"},

2
pylokid/__init__.py

@ -2,6 +2,6 @@
Pylokid. From Mail to Lodur - all automated.
"""
__version__ = "2.2.0"
__version__ = "3.0.0"
__git_version__ = "0"
__url__ = "https://github.com/tobru/pylokid"

52
pylokid/library/emailhandling.py

@ -12,12 +12,13 @@ import imaplib
_EMAIL_SUBJECTS = '(OR OR SUBJECT "Einsatzausdruck_FW" SUBJECT "Einsatzprotokoll" SUBJECT "Einsatzrapport" UNSEEN)'
class EmailHandling:
""" Email handling """
def __init__(self, server, username, password, mailbox, tmp_dir):
self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to IMAP server %s', server)
self.logger.info("Connecting to IMAP server %s", server)
self.tmp_dir = tmp_dir
socket.setdefaulttimeout(60)
@ -27,86 +28,89 @@ class EmailHandling:
self.imap.login(username, password)
self.imap.select(mailbox, readonly=False)
except Exception as err:
self.logger.error('IMAP connection failed - exiting: %s', str(err))
self.logger.error("IMAP connection failed - exiting: %s", str(err))
raise SystemExit(1)
self.logger.info('IMAP connection successfull')
self.logger.info("IMAP connection successful")
def search_emails(self):
""" searches for emails matching the configured subject """
self.logger.info('Searching for messages matching: %s', _EMAIL_SUBJECTS)
self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
try:
typ, msg_ids = self.imap.search(
None,
_EMAIL_SUBJECTS,
)
if typ != 'OK':
self.logger.error('Error searching for matching messages')
if typ != "OK":
self.logger.error("Error searching for matching messages")
return False
except imaplib.IMAP4.abort as err:
self.logger.error('IMAP search aborted - exiting: %s', str(err))
self.logger.error("IMAP search aborted - exiting: %s", str(err))
raise SystemExit(1)
num_messages = len(msg_ids[0].split())
self.logger.info('Found %s matching messages', str(num_messages))
self.logger.info("Found %s matching messages", str(num_messages))
return num_messages, msg_ids
def store_attachments(self, msg_ids):
""" stores the attachments to filesystem and marks message as read """
""" stores the attachments to filesystem """
data = {}
for msg_id in msg_ids[0].split():
# download message from imap
typ, msg_data = self.imap.fetch(msg_id, '(RFC822)')
typ, msg_data = self.imap.fetch(msg_id, "(RFC822)")
if typ != 'OK':
self.logger.error('Error fetching message')
if typ != "OK":
self.logger.error("Error fetching message")
continue
# extract attachment
for response_part in msg_data:
if isinstance(response_part, tuple):
mail = email.message_from_string(str(response_part[1], 'utf-8'))
subject = mail['subject']
mail = email.message_from_string(str(response_part[1], "utf-8"))
subject = mail["subject"]
f_type, f_id = self.parse_subject(subject)
self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
for part in mail.walk():
file_name = part.get_filename()
if not file_name:
self.logger.debug(
'Most probably not an attachment as no filename found'
"Most probably not an attachment as no filename found"
)
continue
self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name)
self.logger.info(
'[%s] Extracting attachment "%s"', f_id, file_name
)
if bool(file_name):
f_type, _ = self.parse_subject(subject)
renamed_file_name = f_type + '_' + file_name
renamed_file_name = f_type + "_" + file_name
# save attachment to filesystem
file_path = os.path.join(self.tmp_dir, renamed_file_name)
self.logger.info('[%s] Saving attachment to "%s"', f_id, file_path)
self.logger.info(
'[%s] Saving attachment to "%s"', f_id, file_path
)
if not os.path.isfile(file_path):
file = open(file_path, 'wb')
file = open(file_path, "wb")
file.write(part.get_payload(decode=True))
file.close()
data[subject] = renamed_file_name
# mark as seen
self.logger.info('[%s] Marking message "%s" as seen', f_id, subject)
self.imap.store(msg_id, '+FLAGS', '(\\Seen)')
return data
def mark_seen(self, msg_id):
self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
def parse_subject(self, subject):
""" extract f id and type from subject """
# This regex matches the subjects filtered already in IMAP search
parsed = re.search('([a-zA-Z_]*):? ?(F[0-9].*)?', subject)
parsed = re.search("([a-zA-Z_]*):? ?(F[0-9].*)?", subject)
f_type = parsed.group(1)
f_id = parsed.group(2)

366
pylokid/library/lodur.py

@ -4,16 +4,18 @@
import re
import logging
from datetime import datetime
from datetime import timedelta
import json
import mechanicalsoup
import pprint
from datetime import datetime, timedelta
class Lodur:
""" Lodur """
def __init__(self, url, username, password):
self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to Lodur')
self.logger.info("Connecting to Lodur")
self.url = url
self.username = username
@ -24,219 +26,172 @@ class Lodur:
self.login()
if self.logged_in():
self.logger.info('Login to Lodur succeeded')
self.logger.info("Login to Lodur succeeded")
else:
self.logger.fatal('Login to Lodur failed - exiting')
self.logger.fatal("Login to Lodur failed - exiting")
raise SystemExit(1)
def login(self):
""" Login to lodur """
# The login form is located in module number 9
self.browser.open(self.url + '?modul=9')
self.browser.open(self.url + "?modul=9")
# only log in when not yed logged in
if not self.logged_in():
# open login page again as the logged_in function has navigated to another page
self.browser.open(self.url + '?modul=9')
self.browser.open(self.url + "?modul=9")
self.browser.select_form()
self.browser['login_member_name'] = self.username
self.browser['login_member_pwd'] = self.password
self.browser["login_member_name"] = self.username
self.browser["login_member_pwd"] = self.password
self.browser.submit_selected()
def logged_in(self):
""" check if logged in to lodur - session is valid """
# Check if login succeeded by finding the img with
# alt text LOGOUT on dashboard
self.browser.open(self.url + '?modul=16')
self.browser.open(self.url + "?modul=16")
page = self.browser.get_current_page()
if page.find(alt='LOGOUT'):
self.logger.debug('Logged in')
if page.find(alt="LOGOUT"):
self.logger.debug("Logged in")
return True
else:
self.logger.debug('Not logged in')
self.logger.debug("Not logged in")
return False
def einsatzprotokoll(self, f_id, pdf_data, webdav_client):
""" Prepare Einsatzprotokoll to be sent to Lodur """
def get_einsatzrapport_id(self, f_id, state="open"):
""" Find ID of automatically created Einsatzrapport """
# check if data is already sent to lodur - data contains lodur_id
lodur_data = webdav_client.get_lodur_data(f_id)
if lodur_data:
# einsatz available in Lodur - updating existing entry
self.logger.info('[%s] Lodur data found - updating entry', f_id)
# when PDF parsing fails, pdf_data is false. fill with tbd when this happens
if pdf_data:
try:
zh_fw_ausg = datetime.strptime(
pdf_data['ausgerueckt'],
'%H:%M:%S',
)
zh_am_schad = datetime.strptime(
pdf_data['vorort'],
'%H:%M:%S',
)
except ValueError as err:
self.logger.error('[%s] Date parsing failed: %s', f_id, err)
zh_fw_ausg = datetime.now()
zh_am_schad = datetime.now()
else:
# Do nothing when no PDF data - we don't have anything to do then
self.logger.error('[%s] No PDF data found - filling in dummy data', f_id)
zh_fw_ausg = datetime.now()
zh_am_schad = datetime.now()
# Complement existing form data
self.logger.info('[%s] Preparing form data for Einsatzprotokoll', f_id)
lodur_data['zh_fw_ausg_h'] = zh_fw_ausg.hour # 13. FW ausgerückt
lodur_data['zh_fw_ausg_m'] = zh_fw_ausg.minute # 13. FW ausgerückt
lodur_data['zh_am_schad_h'] = zh_am_schad.hour # 14. Am Schadenplatz
lodur_data['zh_am_schad_m'] = zh_am_schad.minute # 14. Am Schadenplatz
# The following fields are currently unknown as PDF parsing is hard for these
#lodur_data['zh_fw_einge_h'] = UNKNOWN, # 15. FW eingerückt
#lodur_data['zh_fw_einge_m'] = 'UNKNOWN' # 15. FW eingerückt
#lodur_data['eins_erst_h'] = 'UNKNOWN' # 16. Einsatzbereitschaft erstellt
#lodur_data['eins_erst_m'] = 'UNKNOWN' # 16. Einsatzbereitschaft erstellt
# Submit the form
self.submit_form_einsatzrapport(lodur_data)
# save lodur data to webdav
webdav_client.store_lodur_data(f_id, lodur_data)
# Login to lodur
self.login()
else:
# einsatz not available in Lodur
self.logger.error('[%s] No lodur_id found')
return False
# Browse to Einsatzrapport page
if state == "open":
self.browser.open("{}?modul=36".format(self.url))
def einsatzrapport(self, f_id, pdf_data, webdav_client):
""" Prepare form in module 36 - Einsatzrapport """
# when PDF parsing fails, pdf_data is false. fill with placeholder when this happens
if pdf_data:
date = datetime.strptime(
pdf_data['datum'],
'%d.%m.%Y',
)
time = datetime.strptime(
pdf_data['zeit'],
'%H:%M',
)
eins_ereig = pdf_data['einsatz']
bemerkungen = pdf_data['bemerkungen'] + '\n' + pdf_data['disponierteeinheiten']
wer_ala = pdf_data['melder']
adr = pdf_data['ort']
einsatzrapport_url = self.browser.find_link(link_text=f_id)
if einsatzrapport_url:
lodur_id = re.search(
".*event=([0-9]{1,})&.*", einsatzrapport_url["href"]
).group(1)
return lodur_id
else:
date = datetime.now()
time = datetime.now()
eins_ereig = 'UNKNOWN'
bemerkungen = 'UNKNOWN'
wer_ala = 'UNKNOWN'
adr = 'UNKNOWN'
# Prepare end date and time, can cross midnight
# We blindly add 1 hours - that's the usual length of an Einsatz
time_end = time + timedelta(hours=1)
# check if date is higher after adding 1 hour, this means we crossed midnight
if datetime.date(time_end) > datetime.date(time):
date_end = date + timedelta(days=1)
return None
def retrieve_form_data(self, lodur_id):
""" Retrieve all fields from an Einsatzrapport in Lodur """
# Login to lodur
self.login()
# Browse to Einsatzrapport page
self.browser.open(
"{}?modul=36&what=144&event={}&edit=1".format(self.url, lodur_id)
)
# Lodur doesn't simply store form field values in the form value field
# LOLNOPE - it is stored in javascript in the variable fdata
# And the data format used is just crap - including mixing of different data types
# WHAT DO THEY ACTUALLY THINK ABOUT THIS!!
# Retrieve all <script></script> tags from page
json_string = None
all_scripts = self.browser.page.find_all("script", type="text/javascript")
# Iterate over all tags to find the one containing fdata
for script in all_scripts:
# Some scripts don't have content - we're not interested in them
if script.contents:
# Search for "var fdata" in all scripts - if found, that's what we're looking for
content = script.contents[0]
if "var fdata" in content:
# Cut out unnecessary "var fdata"
json_string = content.replace("var fdata = ", "")
# Now let's parse that data into a data structure which helps
# in filling out the form and make it usable in Python
if json_string:
# Remove the last character which is a ;
usable = {}
for key, value in json.loads(json_string[:-1]).items():
# WHY DO THEY MIX DIFFERENT TYPES!
if isinstance(value, list):
usable[key] = value[2]
elif isinstance(value, dict):
usable[key] = value["2"]
return usable
else:
date_end = date
return None
# Fill in form data
self.logger.info('[%s] Preparing form data for Einsatzrapport', f_id)
lodur_data = {
'e_r_num': f_id, # 01. Einsatzrapportnummer
'eins_stat_kantone': '1', # 02. Einsatzart FKS
'emergency_concept_id': '2', # 03. Verrechnungsart
'ver_sart': 'ab', # 03. Verrechnungsart internal: ab, th, uh, ak, tt
'dtv_d': str(date.day), # 04. Datum von
'dtv_m': str(date.month), # 04. Datum von
'dtv_y': str(date.year), # 04. Datum von
'dtb_d': str(date_end.day), # 04. Datum bis
'dtb_m': str(date_end.month), # 04. Datum bis
'dtb_y': str(date_end.year), # 04. Datum bis
'ztv_h': str(time.hour), # 05. Zeit von
'ztv_m': str(time.minute), # 05. Zeit von
'ztb_h': str(time_end.hour), # 05. Zeit bis - we dont know yet the end time
'ztb_m': str(time_end.minute), # 05. Zeit bis - just add 1 hour and correct later
'e_ort_1': '306', # 06. Einsatzort: Urdorf 306, Birmensdorf 298
'eins_ereig': eins_ereig, # 07. Ereignis
'adr': adr, # 08. Adresse
'wer_ala': wer_ala, # 10. Wer hat alarmiert
'zh_alarmierung_h': str(time.hour), # 12. Alarmierung
'zh_alarmierung_m': str(time.minute), # 12. Alarmierung
'ang_sit': 'TBD1', # 17. Angetroffene Situation
'mn': 'TBD2', # 19. Massnahmen
'bk': bemerkungen, # 20. Bemerkungen
'en_kr_feuwehr': '1', # 21. Einsatzkräfte
'ali_io': '1', # 24. Alarmierung
'kopie_gvz': '1', # 31. Kopie innert 10 Tagen an GVZ
'mannschaftd_einsa': '88', # 32. Einsatzleiter|in
}
def einsatzprotokoll(self, f_id, lodur_data, webdav_client):
""" Prepare Einsatzprotokoll to be sent to Lodur """
# Submit the form
lodur_id, auto_num = self.submit_form_einsatzrapport(lodur_data)
self.logger.info("[%s] Updating Lodur entry", f_id)
# save lodur id and data to webdav
lodur_data['event_id'] = lodur_id
lodur_data['auto_num'] = auto_num
webdav_client.store_lodur_data(f_id, lodur_data)
# Complement existing form data
self.logger.info("[%s] Preparing form data for Einsatzprotokoll", f_id)
return lodur_id
lodur_data["ztb_m"] = lodur_data[
"ztv_m"
] # 05. Zeit (copy minute from start to round up to 1h)
lodur_data["eins_ereig"] = "{} - {} - {}".format(
f_id, lodur_data["ala_stich"], lodur_data["adr"]
) # 07. Ereignis
lodur_data["en_kr_feuwehr"] = "1" # 21. Einsatzkräfte
lodur_data["ali_io"] = "1" # 24. Alarmierung
lodur_data["keyword_els_zutreffend"] = "1" # 25. Stichwort
lodur_data["address_zutreffend"] = "1" # 26. Adresse zutreffend
lodur_data["kopie_gvz"] = "1" # 31. Kopie innert 10 Tagen an GVZ
lodur_data["mannschaftd_einsa"] = "88" # 32. Einsatzleiter|in
def einsatzrapport_alarmdepesche(self, f_id, file_path, webdav_client):
# Submit the form
self.submit_form_einsatzrapport(lodur_data)
# save lodur data to webdav
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
def upload_alarmdepesche(self, f_id, file_path, webdav_client):
""" Upload a file to Alarmdepesche """
self.logger.info('[%s] Submitting file %s to Lodur "Alarmdepesche"', f_id, file_path)
self.logger.info(
'[%s] Submitting file %s to Lodur "Alarmdepesche"', f_id, file_path
)
# Login to lodur
self.login()
# check if data is already sent to lodur - data contains lodur_id
lodur_id = webdav_client.get_lodur_data(f_id)['event_id']
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
# Prepare the form
self.browser.open('{}?modul=36&event={}&what=828'.format(self.url,lodur_id ))
frm_alarmdepesche = self.browser.select_form('#frm_alarmdepesche')
self.browser.open("{}?modul=36&event={}&what=828".format(self.url, lodur_id))
frm_alarmdepesche = self.browser.select_form("#frm_alarmdepesche")
# Fill in form data
frm_alarmdepesche.set('alarmdepesche', file_path)
frm_alarmdepesche.set("alarmdepesche", file_path)
# Submit the form
self.browser.submit_selected()
self.logger.info('[%s] File uploaded to Lodur', f_id)
self.logger.info("[%s] File uploaded to Lodur", f_id)
def einsatzrapport_scan(self, f_id, file_path, webdav_client):
def einsatzrapport_scan(self, f_id, lodur_data, file_path, webdav_client):
""" Prepare Einsatzrapport Scan to be sent to Lodur """
# check if data is already sent to lodur - data contains lodur_id
lodur_data = webdav_client.get_lodur_data(f_id)
if lodur_data:
# einsatz available in Lodur - updating existing entry
self.logger.info('[%s] Lodur data found - updating entry', f_id)
# Complement existing form data
self.logger.info('[%s] Preparing form data for Einsatzprotokoll', f_id)
lodur_data['ang_sit'] = 'Siehe Alarmdepesche - Einsatzrapport' # 17. Angetroffene Situation
lodur_data['mn'] = 'Siehe Alarmdepesche - Einsatzrapport' # 19. Massnahmen
# Submit the form
self.submit_form_einsatzrapport(lodur_data)
# Upload scan to Alarmdepesche
self.einsatzrapport_alarmdepesche(
f_id,
file_path,
webdav_client,
)
else:
# einsatz not available in Lodur
self.logger.error('[%s] No lodur_id found')
return False
# Complement existing form data
self.logger.info("[%s] Updating Lodur entry", f_id)
lodur_data[
"ang_sit"
] = "Siehe Alarmdepesche - Einsatzrapport" # 17. Angetroffene Situation
lodur_data["mn"] = "Siehe Alarmdepesche - Einsatzrapport" # 19. Massnahmen
# Submit the form
self.submit_form_einsatzrapport(lodur_data)
# Upload scan to Alarmdepesche
self.einsatzrapport_alarmdepesche(
f_id,
file_path,
webdav_client,
)
def submit_form_einsatzrapport(self, lodur_data):
""" Form in module 36 - Einsatzrapport """
@ -244,65 +199,46 @@ class Lodur:
# Login to lodur
self.login()
# Prepare the form
if 'event_id' in lodur_data:
# existing entry to update
self.logger.info(
'[%s] Updating existing entry with ID %s',
lodur_data['e_r_num'],
lodur_data['event_id'],
)
self.browser.open(
self.url +
'?modul=36&what=144&edit=1&event=' +
lodur_data['event_id']
)
else:
self.logger.info('[%s] Creating new entry in Lodur', lodur_data['e_r_num'])
self.browser.open(
self.url +
'?modul=36'
)
f_id = lodur_data["e_r_num"]
self.logger.info(
"[%s] Updating existing entry with ID %s",
f_id,
lodur_data["event_id"],
)
self.browser.select_form('#einsatzrapport_main_form')
self.browser.open(
self.url + "?modul=36&what=144&edit=1&event=" + lodur_data["event_id"]
)
form = self.browser.select_form("#einsatzrapport_main_form")
# Prepare the form data to be submitted
for key, value in lodur_data.items():
# Not all keys in the parsed Lodur data are actually part of the form
# Encode some of the fields so they are sent in correct format
# Encoding bk causes some troubles - therefore we skip that - but it
# would be good if it would be encoded as it can / will contain f.e.abs
# Umlauts
# AttributeError: 'bytes' object has no attribute 'parent'
self.logger.info('Form data: %s = %s', key, value)
if key in ('eins_ereig', 'adr', 'wer_ala'):
self.browser[key] = value.encode('iso-8859-1')
else:
self.browser[key] = value
try:
if key in ("eins_ereig", "adr", "wer_ala"):
form.set(key, value.encode("iso-8859-1"))
else:
form.set(key, value)
self.logger.debug("[%s] Set field %s to %s", f_id, key, value)
except mechanicalsoup.LinkNotFoundError as e:
self.logger.debug(
"[%s] Could not set field %s to %s. Reason: %s",
f_id,
key,
value,
str(e),
)
# Submit the form
self.logger.info('[%s] Submitting form Einsatzrapport', lodur_data['e_r_num'])
self.logger.info("[%s] Submitting form Einsatzrapport", lodur_data["e_r_num"])
response = self.browser.submit_selected()
self.logger.info('[%s] Form Einsatzrapport submitted', lodur_data['e_r_num'])
self.logger.info("[%s] Form Einsatzrapport submitted", lodur_data["e_r_num"])
if 'event_id' in lodur_data:
return True
else:
# very ugly way to find the assigned event id by lodur
# lodur adds a script element at the bottom of the returned html
# with the location to reload the page - containing the assigned event id
# print(response.text)
lodur_id = re.search('modul=36&event=([0-9].*)&edit=1&what=144', response.text).group(1)
self.logger.info('[%s] Lodur assigned the event_id %s', lodur_data['e_r_num'], lodur_id)
# The hidden field auto_num is also needed for updating the form
# and it's written somewhere in javascript code - but not on the page
# delivered after the submission which contains the redirect URL
# It's only delivered in the next page. So we browse to this page now
content = self.browser.open(
self.url +
'?modul=36&edit=1&what=144&event=' + lodur_id
).text
auto_num = re.search(r"\"([0-9]{4}\|[0-9]{1,3})\"", content).group(1)
self.logger.info('[%s] Lodur assigned the auto_num %s', lodur_data['e_r_num'], auto_num)
return lodur_id, auto_num
return True

205
pylokid/library/pdftotext.py

@ -5,109 +5,157 @@
import subprocess
import logging
class PDFParsing:
""" PDF parsing """
def __init__(self):
self.logger = logging.getLogger(__name__)
self.logger.info('PDF parsing based on pdftotext loaded')
self.logger.info("PDF parsing based on pdftotext loaded")
def extract(self, f_id, file, datafields):
self.logger.info('[%s] parsing PDF file %s', f_id, file)
self.logger.info("[%s] parsing PDF file %s", f_id, file)
data = {}
for field, coordinate in datafields.items():
# x-coordinate of the crop area top left corner
x = coordinate['xMin']
x = coordinate["xMin"]
# y-coordinate of the crop area top left corner
y = coordinate['yMin']
y = coordinate["yMin"]
# width of crop area in pixels
w = coordinate['xMax'] - coordinate['xMin']
w = coordinate["xMax"] - coordinate["xMin"]
# height of crop area in pixels
h = coordinate['yMax'] - coordinate['yMin']
h = coordinate["yMax"] - coordinate["yMin"]
self.logger.debug('[%s] Computed command for field %s: %s', f_id, field,
'pdftotext -f 1 -l 1 -x {} -y {} -W {} -H {}'.format(x,y,w,h)
self.logger.debug(
"[%s] Computed command for field %s: %s",
f_id,
field,
"pdftotext -f 1 -l 1 -x {} -y {} -W {} -H {}".format(x, y, w, h),
)
scrapeddata = subprocess.Popen([
'/usr/bin/pdftotext',
'-f', '1',
'-l', '1',
'-x', str(x),
'-y', str(y),
'-W', str(w),
'-H', str(h),
file,
'-'
scrapeddata = subprocess.Popen(
[
"/usr/bin/pdftotext",
"-f",
"1",
"-l",
"1",
"-x",
str(x),
"-y",
str(y),
"-W",
str(w),
"-H",
str(h),
file,
"-",
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True)
text=True,
)
stdout, _ = scrapeddata.communicate()
## TODO: fixup some fields (lowercase, remove unnecessary \n)
if 'edit' in coordinate and coordinate['edit'] == 'title':
if "edit" in coordinate and coordinate["edit"] == "title":
data[field] = stdout.rstrip().title()
else:
data[field] = stdout.rstrip()
# sanity check to see if we can correlate the f_id
if f_id == data['auftrag']:
self.logger.debug('[%s] ID matches in PDF', f_id)
if f_id == data["auftrag"]:
self.logger.debug("[%s] ID matches in PDF", f_id)
return data
else:
self.logger.error('[%s] ID does not match in PDF: "%s"', f_id, data['auftrag'])
self.logger.error(
'[%s] ID does not match in PDF: "%s"', f_id, data["auftrag"]
)
return False
def extract_einsatzausdruck(self, file, f_id):
""" extracts information from Einsatzausdruck using external pdftotext """
self.logger.debug('[%s] Parsing PDF: %s', f_id, file)
self.logger.debug("[%s] Parsing PDF: %s", f_id, file)
# Get them using 'pdftotext -bbox'
# y = row
# x = column: xMax 450 / 590 means full width
coordinates = {
'auftrag': {
'xMin': 70, 'yMin': 47, 'xMax': 120,'yMax': 58,
},
'datum': {
'xMin': 190, 'yMin': 47, 'xMax': 239, 'yMax': 58,
},
'zeit': {
'xMin': 190, 'yMin': 59, 'xMax': 215, 'yMax': 70,
},
'melder': {
'xMin': 304, 'yMin': 47, 'xMax': 446, 'yMax': 70, 'edit': 'title'
},
'erfasser':{
'xMin': 448, 'yMin': 59, 'xMax': 478, 'yMax': 70,
"auftrag": {
"xMin": 70,
"yMin": 47,
"xMax": 120,
"yMax": 58,
},
"datum": {
"xMin": 190,
"yMin": 47,
"xMax": 239,
"yMax": 58,
},
"zeit": {
"xMin": 190,
"yMin": 59,
"xMax": 215,
"yMax": 70,
},
"melder": {
"xMin": 304,
"yMin": 47,
"xMax": 446,
"yMax": 70,
"edit": "title",
},
"erfasser": {
"xMin": 448,
"yMin": 59,
"xMax": 478,
"yMax": 70,
},
# big field until "Disponierte Einheiten"
'bemerkungen': {
'xMin': 28, 'yMin': 112, 'xMax': 590, 'yMax': 350,
},
'disponierteeinheiten': {
'xMin': 28, 'yMin': 366, 'xMax': 450, 'yMax': 376,
},
'einsatz': {
'xMin': 76, 'yMin': 690, 'xMax': 450, 'yMax': 703,
},
'sondersignal': {
'xMin': 76, 'yMin': 707, 'xMax': 450, 'yMax': 721,
},
'ort': {
'xMin': 76, 'yMin': 732, 'xMax': 590, 'yMax': 745,
},
'hinweis': {
'xMin': 76, 'yMin': 773, 'xMax': 450, 'yMax': 787,
"bemerkungen": {
"xMin": 28,
"yMin": 112,
"xMax": 590,
"yMax": 350,
},
"disponierteeinheiten": {
"xMin": 28,
"yMin": 366,
"xMax": 450,
"yMax": 376,
},
"einsatz": {
"xMin": 76,
"yMin": 690,
"xMax": 450,
"yMax": 703,
},
"sondersignal": {
"xMin": 76,
"yMin": 707,
"xMax": 450,
"yMax": 721,
},
"ort": {
"xMin": 76,
"yMin": 732,
"xMax": 590,
"yMax": 745,
},
"hinweis": {
"xMin": 76,
"yMin": 773,
"xMax": 450,
"yMax": 787,
},
}
@ -116,27 +164,42 @@ class PDFParsing:
def extract_einsatzprotokoll(self, file, f_id):
""" extracts information from Einsatzprotokoll using external pdftotext """
self.logger.debug('[%s] Parsing PDF: %s', f_id, file)
self.logger.debug("[%s] Parsing PDF: %s", f_id, file)
# Get them using 'pdftotext -bbox'
# y = row
# x = column: xMax 450 / 590 means full width
coordinates = {
'auftrag': {
'xMin': 192, 'yMin': 132, 'xMax': 238,'yMax': 142,
},
'angelegt': {
'xMin': 192, 'yMin': 294, 'xMax': 226, 'yMax': 304,
},
'dispo': {
'xMin': 192, 'yMin': 312, 'xMax': 226, 'yMax': 322,
},
'ausgerueckt': {
'xMin': 192, 'yMin': 331, 'xMax': 226, 'yMax': 341,
},
'vorort':{
'xMin': 192, 'yMin': 348, 'xMax': 226, 'yMax': 358,
"auftrag": {
"xMin": 192,
"yMin": 132,
"xMax": 238,
"yMax": 142,
},
"angelegt": {
"xMin": 192,
"yMin": 294,
"xMax": 226,
"yMax": 304,
},
"dispo": {
"xMin": 192,
"yMin": 312,
"xMax": 226,
"yMax": 322,
},
"ausgerueckt": {
"xMin": 192,
"yMin": 331,
"xMax": 226,
"yMax": 341,
},
"vorort": {
"xMin": 192,
"yMin": 348,
"xMax": 226,
"yMax": 358,
},
}
return self.extract(f_id, file, coordinates)
return self.extract(f_id, file, coordinates)

58
pylokid/library/webdav.py

@ -9,12 +9,13 @@ import logging
import asyncio
import aioeasywebdav
class WebDav:
""" WebDav Client """
def __init__(self, url, username, password, webdav_basedir, tmp_dir):
self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to WebDAV server %s', url)
self.logger.info("Connecting to WebDAV server %s", url)
self.loop = asyncio.get_event_loop()
self.webdav_basedir = webdav_basedir
@ -26,18 +27,20 @@ class WebDav:
password=password,
)
except:
self.logger.error('WebDAV connection failed - exiting')
self.logger.error("WebDAV connection failed - exiting")
self.logger.info('WebDAV connection successfull')
self.logger.info("WebDAV connection successfull")
def upload(self, file_name, f_id, check_exists = True):
def upload(self, file_name, f_id, check_exists=True):
""" uploads a file to webdav - checks for existence before doing so """
# upload with webdav
if f_id == None:
remote_upload_dir = self.webdav_basedir + "/Inbox"
else:
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
remote_upload_dir = (
self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
)
self.logger.info('[%s] Uploading file to WebDAV "%s"', f_id, remote_upload_dir)
@ -47,7 +50,9 @@ class WebDav:
self.loop.run_until_complete(self.webdav.mkdir(remote_upload_dir))
remote_file_path = remote_upload_dir + "/" + file_name
if check_exists and self.loop.run_until_complete(self.webdav.exists(remote_file_path)):
if check_exists and self.loop.run_until_complete(
self.webdav.exists(remote_file_path)
):
self.logger.info('[%s] File "%s" already exists on WebDAV', f_id, file_name)
else:
self.loop.run_until_complete(
@ -61,47 +66,52 @@ class WebDav:
def einsatz_exists(self, f_id):
""" check if an einsatz is already created """
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
remote_upload_dir = (
self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
)
if self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)):
self.logger.info('[%s] Einsatz exists on WebDAV', f_id)
self.logger.info("[%s] Einsatz exists on WebDAV", f_id)
return True
else:
return False
def store_lodur_data(self, f_id, lodur_data):
""" stores lodur data on webdav """
def store_data(self, f_id, file_name, data):
""" stores data on webdav """
file_name = f_id + '_lodur.json'
file_path = os.path.join(self.tmp_dir, file_name)
file = open(file_path, 'w')
file.write(json.dumps(lodur_data))
file = open(file_path, "w")
file.write(json.dumps(data))
file.close()
self.logger.info('[%s] Stored Lodur data locally in %s', f_id, file_path)
self.logger.info("[%s] Stored data locally in %s", f_id, file_path)
self.upload(file_name, f_id, False)
def get_lodur_data(self, f_id):
def get_lodur_data(self, f_id, filetype="_lodur.json"):
""" gets lodur data if it exists """
file_name = f_id + '_lodur.json'
file_name = f_id + filetype
file_path = os.path.join(self.tmp_dir, file_name)
# first check if we already have it locally - then check on webdav
if os.path.isfile(file_path):
with open(file_path, 'r') as content:
with open(file_path, "r") as content:
lodur_data = json.loads(content.read())
self.logger.info('[%s] Found Lodur data locally', f_id)
self.logger.info("[%s] Found Lodur data locally", f_id)
return lodur_data
else:
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
remote_file_path = remote_upload_dir + '/' + file_name
remote_upload_dir = (
self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
)
remote_file_path = remote_upload_dir + "/" + file_name
if self.loop.run_until_complete(self.webdav.exists(remote_file_path)):
self.loop.run_until_complete(self.webdav.download(remote_file_path, file_path))
with open(file_path, 'r') as content:
self.loop.run_until_complete(
self.webdav.download(remote_file_path, file_path)
)
with open(file_path, "r") as content:
lodur_data = json.loads(content.read())
self.logger.info('[%s] Found Lodur data on WebDAV', f_id)
self.logger.info("[%s] Found Lodur data on WebDAV", f_id)
return lodur_data
else:
self.logger.info('[%s] No existing Lodur data found', f_id)
self.logger.info("[%s] No existing Lodur data found", f_id)
return False

202
pylokid/main.py

<
@ -7,6 +7,7 @@ import os
import time
import requests
from importlib.metadata import version
from dotenv import find_dotenv, load_dotenv
from pushover import Client
@ -34,7 +35,7 @@ LODUR_BASE_URL = os.getenv("LODUR_BASE_URL")
HEARTBEAT_URL = os.getenv("HEARTBEAT_URL")
PUSHOVER_API_TOKEN = os.getenv("PUSHOVER_API_TOKEN")
PUSHOVER_USER_KEY = os.getenv("PUSHOVER_USER_KEY")
PYLOKID_VERSION = "2.2.0"
def main():
""" main """
@ -42,10 +43,10 @@ def main():
# Logging configuration
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger('pylokid')
logger.info('Starting pylokid version %s', PYLOKID_VERSION)
logger = logging.getLogger("pylokid")
logger.info("Starting pylokid version %s", version("pylokid"))
# Initialize IMAP Session
imap_client = EmailHandling(
@ -73,10 +74,7 @@ def main():
)
# Initialize Pushover
pushover = Client(
user_key=PUSHOVER_USER_KEY,
api_token=PUSHOVER_API_TOKEN
)
pushover = Client(user_key=PUSHOVER_USER_KEY, api_token=PUSHOVER_API_TOKEN)
# Initialize PDF Parser
pdf = PDFParsing()
@ -97,129 +95,161 @@ def main():
webdav_client.upload(file_name, f_id)
# Take actions - depending on the type
if f_type == 'Einsatzausdruck_FW':
logger.info('[%s] Processing type %s', f_id, f_type)
lodur_data = webdav_client.get_lodur_data(f_id)
if lodur_data:
logger.info(
'[%s] Einsatzrapport already created in Lodur', f_id
)
# Upload Alarmdepesche as it could contain more information
# than the first one
lodur_client.einsatzrapport_alarmdepesche(
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
if f_type == "Einsatzausdruck_FW":
logger.info("[%s] Processing type %s", f_id, f_type)
# Check if the PDF isn't already parsed
if webdav_client.get_lodur_data(f_id, "_pdf.json"):
logger.info("[%s] PDF already parsed", f_id)
else:
## Here we get the initial Einsatzauftrag - Time to run
# get as many information from PDF as possible
pdf_file = os.path.join(TMP_DIR, file_name)
# Extract information from PDF
pdf_data = pdf.extract_einsatzausdruck(
pdf_file,
os.path.join(TMP_DIR, file_name),
f_id,
)
# publish Einsatz on Pushover
logger.info(
'[%s] Publishing message on Pushover', f_id
)
logger.info("[%s] Publishing message on Pushover", f_id)
pushover.send_message(
"Einsatz {} eröffnet: {}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
f_id,
pdf_data['einsatz'],
pdf_data['ort'],
pdf_data['melder'].replace('\n',' '),
pdf_data['hinweis'],
pdf_data['sondersignal'],
pdf_data['disponierteeinheiten'],
pdf_data['bemerkungen'],
"<b>{}</b>\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
pdf_data["einsatz"],
pdf_data["ort"],
pdf_data["melder"].replace("\n", " "),
pdf_data["hinweis"],
pdf_data["sondersignal"],
pdf_data["bemerkungen"],
pdf_data["disponierteeinheiten"],
),
title="Feuerwehr Einsatz",
url="https://www.google.com/maps/search/?api=1&query={}".format(pdf_data['ort']),
url_title="Ort auf Karte suchen"
title="Feuerwehr Einsatz - {}".format(f_id),
url="https://www.google.com/maps/search/?api=1&query={}".format(
pdf_data["ort"]
),
url_title="Ort auf Karte suchen",
html=1,
)
# create new Einsatzrapport in Lodur
lodur_client.einsatzrapport(
f_id,
pdf_data,
webdav_client,
)
# Upload extracted data to cloud
webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data)
# upload Alarmdepesche PDF to Lodur
lodur_client.einsatzrapport_alarmdepesche(
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
if webdav_client.get_lodur_data(f_id):
logger.info("[%s] Lodur data already retrieved", f_id)
else:
# Retrieve data from Lodur
lodur_id = lodur_client.get_einsatzrapport_id(f_id)
if lodur_id:
logger.info(
"[%s] Einsatzrapport available in Lodur with ID %s",