tobru
/
pylokid
1
0
Fork 0
Code Pull Requests Releases Activity

major update to reuse already existing records
continuous-integration/drone/push Build is passing Details 

Lodur since some time automatically creates Einsatzrapporte via an API
from SRZ/GVZ. One of the main features of Pylokid was to exactly do
that. With that new change this isn't necessary anymore. Pylokid has
been amended to find the pre-existing entry and work with that -
enhancing it with any additional information missing and uploads PDFs to
the right place.

While at it a very small modernization has been made and the project
moved to use Poetry and Black formatting. But it's still the same ugly
code - to reflect Lodur.
This commit is contained in:
Tobias Brunner 2021-02-14 15:00:15 +01:00
parent a32f4041d3
commit 9c8c5396c8
9 changed files with 684 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
poetry.lock generated
View File

@ -30,6 +30,14 @@ yarl = ">=1.0,<2.0"
[package.extras] [package.extras]
speedups = ["aiodns", "brotlipy", "cchardet"] speedups = ["aiodns", "brotlipy", "cchardet"]
[[package]]
name = "appdirs"
version = "1.4.4"
description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
category = "dev"
optional = false
python-versions = "*"
[[package]] [[package]]
name = "async-timeout" name = "async-timeout"
version = "3.0.1" version = "3.0.1"
@ -67,6 +75,28 @@ soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""}
html5lib = ["html5lib"] html5lib = ["html5lib"]
lxml = ["lxml"] lxml = ["lxml"]
[[package]]
name = "black"
version = "20.8b1"
description = "The uncompromising code formatter."
category = "dev"
optional = false
python-versions = ">=3.6"
[package.dependencies]
appdirs = "*"
click = ">=7.1.2"
mypy-extensions = ">=0.4.3"
pathspec = ">=0.6,<1"
regex = ">=2020.1.8"
toml = ">=0.10.1"
typed-ast = ">=1.4.0"
typing-extensions = ">=3.7.4"
[package.extras]
colorama = ["colorama (>=0.4.3)"]
d = ["aiohttp (>=3.3.2)", "aiohttp-cors"]
[[package]] [[package]]
name = "certifi" name = "certifi"
version = "2020.12.5" version = "2020.12.5"
@ -83,6 +113,27 @@ category = "main"
optional = false optional = false
python-versions = "*" python-versions = "*"
[[package]]
name = "click"
version = "7.1.2"
description = "Composable command line interface toolkit"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[[package]]
name = "flake8"
version = "3.8.4"
description = "the modular source code checker: pep8 pyflakes and co"
category = "dev"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
[package.dependencies]
mccabe = ">=0.6.0,<0.7.0"
pycodestyle = ">=2.6.0a1,<2.7.0"
pyflakes = ">=2.2.0,<2.3.0"
[[package]] [[package]]
name = "idna" name = "idna"
version = "2.10" version = "2.10"
@ -105,6 +156,14 @@ html5 = ["html5lib"]
htmlsoup = ["beautifulsoup4"] htmlsoup = ["beautifulsoup4"]
source = ["Cython (>=0.29.7)"] source = ["Cython (>=0.29.7)"]
[[package]]
name = "mccabe"
version = "0.6.1"
description = "McCabe checker, plugin for flake8"
category = "dev"
optional = false
python-versions = "*"
[[package]] [[package]]
name = "mechanicalsoup" name = "mechanicalsoup"
version = "1.0.0" version = "1.0.0"
@ -127,6 +186,38 @@ category = "main"
optional = false optional = false
python-versions = ">=3.6" python-versions = ">=3.6"
[[package]]
name = "mypy-extensions"
version = "0.4.3"
description = "Experimental type system extensions for programs checked with the mypy typechecker."
category = "dev"
optional = false
python-versions = "*"
[[package]]
name = "pathspec"
version = "0.8.1"
description = "Utility library for gitignore style pattern matching of file paths."
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
[[package]]
name = "pycodestyle"
version = "2.6.0"
description = "Python style guide checker"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]]
name = "pyflakes"
version = "2.2.0"
description = "passive checker of Python programs"
category = "dev"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
[[package]] [[package]]
name = "python-dotenv" name = "python-dotenv"
version = "0.15.0" version = "0.15.0"
@ -149,6 +240,14 @@ python-versions = "*"
[package.dependencies] [package.dependencies]
requests = ">=1.0" requests = ">=1.0"
[[package]]
name = "regex"
version = "2020.11.13"
description = "Alternative regular expression module, to replace re."
category = "dev"
optional = false
python-versions = "*"
[[package]] [[package]]
name = "requests" name = "requests"
version = "2.25.1" version = "2.25.1"
@ -194,6 +293,22 @@ category = "main"
optional = false optional = false
python-versions = ">=3.6" python-versions = ">=3.6"
[[package]]
name = "toml"
version = "0.10.2"
description = "Python Library for Tom's Obvious, Minimal Language"
category = "dev"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
[[package]]
name = "typed-ast"
version = "1.4.2"
description = "a fork of Python 2 and 3 ast modules with type comment support"
category = "dev"
optional = false
python-versions = "*"
[[package]] [[package]]
name = "typing-extensions" name = "typing-extensions"
version = "3.7.4.3" version = "3.7.4.3"
@ -230,7 +345,7 @@ multidict = ">=4.0"
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.9" python-versions = "^3.9"
content-hash = "c5fa2a2589b88adb6a9bf94ec4a88e0e6b9528e40837cf6b2cf045a3e072d6d4" content-hash = "87029b7a633e874d04d308355076802812613a9b93a03770d10ab6b9fcbc5b44"
[metadata.files] [metadata.files]
aioeasywebdav = [ aioeasywebdav = [
@ -276,6 +391,10 @@ aiohttp = [
{file = "aiohttp-3.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7"}, {file = "aiohttp-3.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7"},
{file = "aiohttp-3.7.3.tar.gz", hash = "sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4"}, {file = "aiohttp-3.7.3.tar.gz", hash = "sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4"},
] ]
appdirs = [
{file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"},
{file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"},
]
async-timeout = [ async-timeout = [
{file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"},
{file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"},
@ -289,6 +408,9 @@ beautifulsoup4 = [
{file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"}, {file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"},
{file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"},
] ]
black = [
{file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"},
]
certifi = [ certifi = [
{file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"},
{file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"},
@ -297,6 +419,14 @@ chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
{file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
] ]
click = [
{file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"},
{file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"},
]
flake8 = [
{file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"},
{file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"},
]
idna = [ idna = [
{file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"},
{file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"},
@ -340,6 +470,10 @@ lxml = [
{file = "lxml-4.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf"}, {file = "lxml-4.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf"},
{file = "lxml-4.6.2.tar.gz", hash = "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc"}, {file = "lxml-4.6.2.tar.gz", hash = "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc"},
] ]
mccabe = [
{file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
{file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
]
mechanicalsoup = [ mechanicalsoup = [
{file = "MechanicalSoup-1.0.0-py2.py3-none-any.whl", hash = "sha256:2ed9a494c144fb2c262408dcbd5c79e1ef325a7426f1fa3a3443eaef133b5f77"}, {file = "MechanicalSoup-1.0.0-py2.py3-none-any.whl", hash = "sha256:2ed9a494c144fb2c262408dcbd5c79e1ef325a7426f1fa3a3443eaef133b5f77"},
{file = "MechanicalSoup-1.0.0.tar.gz", hash = "sha256:37d3b15c1957917d3ae171561e77f4dd4c08c35eb4500b8781f6e7e1bb6c4d07"}, {file = "MechanicalSoup-1.0.0.tar.gz", hash = "sha256:37d3b15c1957917d3ae171561e77f4dd4c08c35eb4500b8781f6e7e1bb6c4d07"},
@ -383,6 +517,22 @@ multidict = [
{file = "multidict-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359"}, {file = "multidict-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359"},
{file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"}, {file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"},
] ]
mypy-extensions = [
{file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
{file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},
]
pathspec = [
{file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"},
{file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"},
]
pycodestyle = [
{file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"},
{file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"},
]
pyflakes = [
{file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"},
{file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"},
]
python-dotenv = [ python-dotenv = [
{file = "python-dotenv-0.15.0.tar.gz", hash = "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0"}, {file = "python-dotenv-0.15.0.tar.gz", hash = "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0"},
{file = "python_dotenv-0.15.0-py2.py3-none-any.whl", hash = "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e"}, {file = "python_dotenv-0.15.0-py2.py3-none-any.whl", hash = "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e"},
@ -390,6 +540,49 @@ python-dotenv = [
python-pushover = [ python-pushover = [
{file = "python-pushover-0.4.tar.gz", hash = "sha256:dee1b1344fb8a5874365fc9f886d9cbc7775536629999be54dfa60177cf80810"}, {file = "python-pushover-0.4.tar.gz", hash = "sha256:dee1b1344fb8a5874365fc9f886d9cbc7775536629999be54dfa60177cf80810"},
] ]
regex = [
{file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa"},
{file = "regex-2020.11.13-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6"},
{file = "regex-2020.11.13-cp36-cp36m-win32.whl", hash = "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e"},
{file = "regex-2020.11.13-cp36-cp36m-win_amd64.whl", hash = "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884"},
{file = "regex-2020.11.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba"},
{file = "regex-2020.11.13-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538"},
{file = "regex-2020.11.13-cp37-cp37m-win32.whl", hash = "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4"},
{file = "regex-2020.11.13-cp37-cp37m-win_amd64.whl", hash = "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444"},
{file = "regex-2020.11.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f"},
{file = "regex-2020.11.13-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d"},
{file = "regex-2020.11.13-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5"},
{file = "regex-2020.11.13-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b"},
{file = "regex-2020.11.13-cp38-cp38-win32.whl", hash = "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c"},
{file = "regex-2020.11.13-cp38-cp38-win_amd64.whl", hash = "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683"},
{file = "regex-2020.11.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc"},
{file = "regex-2020.11.13-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364"},
{file = "regex-2020.11.13-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9"},
{file = "regex-2020.11.13-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c"},
{file = "regex-2020.11.13-cp39-cp39-win32.whl", hash = "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"},
{file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"},
{file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"},
]
requests = [ requests = [
{file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"},
{file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"},
@ -412,6 +605,42 @@ soupsieve = [
{file = "soupsieve-2.2-py3-none-any.whl", hash = "sha256:d3a5ea5b350423f47d07639f74475afedad48cf41c0ad7a82ca13a3928af34f6"}, {file = "soupsieve-2.2-py3-none-any.whl", hash = "sha256:d3a5ea5b350423f47d07639f74475afedad48cf41c0ad7a82ca13a3928af34f6"},
{file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"}, {file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"},
] ]
toml = [
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
]
typed-ast = [
{file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70"},
{file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487"},
{file = "typed_ast-1.4.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412"},
{file = "typed_ast-1.4.2-cp35-cp35m-win32.whl", hash = "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400"},
{file = "typed_ast-1.4.2-cp35-cp35m-win_amd64.whl", hash = "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606"},
{file = "typed_ast-1.4.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64"},
{file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07"},
{file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc"},
{file = "typed_ast-1.4.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a"},
{file = "typed_ast-1.4.2-cp36-cp36m-win32.whl", hash = "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151"},
{file = "typed_ast-1.4.2-cp36-cp36m-win_amd64.whl", hash = "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3"},
{file = "typed_ast-1.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41"},
{file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f"},
{file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581"},
{file = "typed_ast-1.4.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37"},
{file = "typed_ast-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd"},
{file = "typed_ast-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496"},
{file = "typed_ast-1.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc"},
{file = "typed_ast-1.4.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10"},
{file = "typed_ast-1.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea"},
{file = "typed_ast-1.4.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787"},
{file = "typed_ast-1.4.2-cp38-cp38-win32.whl", hash = "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2"},
{file = "typed_ast-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937"},
{file = "typed_ast-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1"},
{file = "typed_ast-1.4.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6"},
{file = "typed_ast-1.4.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166"},
{file = "typed_ast-1.4.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d"},
{file = "typed_ast-1.4.2-cp39-cp39-win32.whl", hash = "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b"},
{file = "typed_ast-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440"},
{file = "typed_ast-1.4.2.tar.gz", hash = "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a"},
]
typing-extensions = [ typing-extensions = [
{file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"},
{file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"},

2
pylokid/__init__.py
View File

@ -2,6 +2,6 @@
Pylokid. From Mail to Lodur - all automated. Pylokid. From Mail to Lodur - all automated.
""" """
__version__ = "2.2.0" __version__ = "3.0.0"
__git_version__ = "0" __git_version__ = "0"
__url__ = "https://github.com/tobru/pylokid" __url__ = "https://github.com/tobru/pylokid"

52
pylokid/library/emailhandling.py
View File

@ -12,12 +12,13 @@ import imaplib
_EMAIL_SUBJECTS = '(OR OR SUBJECT "Einsatzausdruck_FW" SUBJECT "Einsatzprotokoll" SUBJECT "Einsatzrapport" UNSEEN)' _EMAIL_SUBJECTS = '(OR OR SUBJECT "Einsatzausdruck_FW" SUBJECT "Einsatzprotokoll" SUBJECT "Einsatzrapport" UNSEEN)'
class EmailHandling: class EmailHandling:
""" Email handling """ """ Email handling """
def __init__(self, server, username, password, mailbox, tmp_dir): def __init__(self, server, username, password, mailbox, tmp_dir):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to IMAP server %s', server) self.logger.info("Connecting to IMAP server %s", server)
self.tmp_dir = tmp_dir self.tmp_dir = tmp_dir
socket.setdefaulttimeout(60) socket.setdefaulttimeout(60)
@ -27,86 +28,89 @@ class EmailHandling:
self.imap.login(username, password) self.imap.login(username, password)
self.imap.select(mailbox, readonly=False) self.imap.select(mailbox, readonly=False)
except Exception as err: except Exception as err:
self.logger.error('IMAP connection failed - exiting: %s', str(err)) self.logger.error("IMAP connection failed - exiting: %s", str(err))
raise SystemExit(1) raise SystemExit(1)
self.logger.info('IMAP connection successfull') self.logger.info("IMAP connection successful")
def search_emails(self): def search_emails(self):
""" searches for emails matching the configured subject """ """ searches for emails matching the configured subject """
self.logger.info('Searching for messages matching: %s', _EMAIL_SUBJECTS) self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS)
try: try:
typ, msg_ids = self.imap.search( typ, msg_ids = self.imap.search(
None, None,
_EMAIL_SUBJECTS, _EMAIL_SUBJECTS,
) )
if typ != 'OK': if typ != "OK":
self.logger.error('Error searching for matching messages') self.logger.error("Error searching for matching messages")
return False return False
except imaplib.IMAP4.abort as err: except imaplib.IMAP4.abort as err:
self.logger.error('IMAP search aborted - exiting: %s', str(err)) self.logger.error("IMAP search aborted - exiting: %s", str(err))
raise SystemExit(1) raise SystemExit(1)
num_messages = len(msg_ids[0].split()) num_messages = len(msg_ids[0].split())
self.logger.info('Found %s matching messages', str(num_messages)) self.logger.info("Found %s matching messages", str(num_messages))
return num_messages, msg_ids return num_messages, msg_ids
def store_attachments(self, msg_ids): def store_attachments(self, msg_ids):
""" stores the attachments to filesystem and marks message as read """ """ stores the attachments to filesystem """
data = {} data = {}
for msg_id in msg_ids[0].split(): for msg_id in msg_ids[0].split():
# download message from imap # download message from imap
typ, msg_data = self.imap.fetch(msg_id, '(RFC822)') typ, msg_data = self.imap.fetch(msg_id, "(RFC822)")
if typ != 'OK': if typ != "OK":
self.logger.error('Error fetching message') self.logger.error("Error fetching message")
continue continue
# extract attachment # extract attachment
for response_part in msg_data: for response_part in msg_data:
if isinstance(response_part, tuple): if isinstance(response_part, tuple):
mail = email.message_from_string(str(response_part[1], 'utf-8')) mail = email.message_from_string(str(response_part[1], "utf-8"))
subject = mail['subject'] subject = mail["subject"]
f_type, f_id = self.parse_subject(subject) f_type, f_id = self.parse_subject(subject)
self.logger.info('[%s] Getting attachment from "%s"', f_id, subject) self.logger.info('[%s] Getting attachment from "%s"', f_id, subject)
for part in mail.walk(): for part in mail.walk():
file_name = part.get_filename() file_name = part.get_filename()
if not file_name: if not file_name:
self.logger.debug( self.logger.debug(
'Most probably not an attachment as no filename found' "Most probably not an attachment as no filename found"
) )
continue continue
self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name) self.logger.info(
'[%s] Extracting attachment "%s"', f_id, file_name
)
if bool(file_name): if bool(file_name):
f_type, _ = self.parse_subject(subject) f_type, _ = self.parse_subject(subject)
renamed_file_name = f_type + '_' + file_name renamed_file_name = f_type + "_" + file_name
# save attachment to filesystem # save attachment to filesystem
file_path = os.path.join(self.tmp_dir, renamed_file_name) file_path = os.path.join(self.tmp_dir, renamed_file_name)
self.logger.info('[%s] Saving attachment to "%s"', f_id, file_path) self.logger.info(
'[%s] Saving attachment to "%s"', f_id, file_path
)
if not os.path.isfile(file_path): if not os.path.isfile(file_path):
file = open(file_path, 'wb') file = open(file_path, "wb")
file.write(part.get_payload(decode=True)) file.write(part.get_payload(decode=True))
file.close() file.close()
data[subject] = renamed_file_name data[subject] = renamed_file_name
# mark as seen
self.logger.info('[%s] Marking message "%s" as seen', f_id, subject)
self.imap.store(msg_id, '+FLAGS', '(\\Seen)')
return data return data
def mark_seen(self, msg_id):
self.imap.store(msg_id, "+FLAGS", "(\\Seen)")
def parse_subject(self, subject): def parse_subject(self, subject):
""" extract f id and type from subject """ """ extract f id and type from subject """
# This regex matches the subjects filtered already in IMAP search # This regex matches the subjects filtered already in IMAP search
parsed = re.search('([a-zA-Z_]*):? ?(F[0-9].*)?', subject) parsed = re.search("([a-zA-Z_]*):? ?(F[0-9].*)?", subject)
f_type = parsed.group(1) f_type = parsed.group(1)
f_id = parsed.group(2) f_id = parsed.group(2)

366
pylokid/library/lodur.py
View File

@ -4,16 +4,18 @@
import re import re
import logging import logging
from datetime import datetime import json
from datetime import timedelta
import mechanicalsoup import mechanicalsoup
import pprint
from datetime import datetime, timedelta
class Lodur: class Lodur:
""" Lodur """ """ Lodur """
def __init__(self, url, username, password): def __init__(self, url, username, password):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to Lodur') self.logger.info("Connecting to Lodur")
self.url = url self.url = url
self.username = username self.username = username
@ -24,219 +26,172 @@ class Lodur:
self.login() self.login()
if self.logged_in(): if self.logged_in():
self.logger.info('Login to Lodur succeeded') self.logger.info("Login to Lodur succeeded")
else: else:
self.logger.fatal('Login to Lodur failed - exiting') self.logger.fatal("Login to Lodur failed - exiting")
raise SystemExit(1) raise SystemExit(1)
def login(self): def login(self):
""" Login to lodur """ """ Login to lodur """
# The login form is located in module number 9 # The login form is located in module number 9
self.browser.open(self.url + '?modul=9') self.browser.open(self.url + "?modul=9")
# only log in when not yed logged in # only log in when not yed logged in
if not self.logged_in(): if not self.logged_in():
# open login page again as the logged_in function has navigated to another page # open login page again as the logged_in function has navigated to another page
self.browser.open(self.url + '?modul=9') self.browser.open(self.url + "?modul=9")
self.browser.select_form() self.browser.select_form()
self.browser['login_member_name'] = self.username self.browser["login_member_name"] = self.username
self.browser['login_member_pwd'] = self.password self.browser["login_member_pwd"] = self.password
self.browser.submit_selected() self.browser.submit_selected()
def logged_in(self): def logged_in(self):
""" check if logged in to lodur - session is valid """ """ check if logged in to lodur - session is valid """
# Check if login succeeded by finding the img with # Check if login succeeded by finding the img with
# alt text LOGOUT on dashboard # alt text LOGOUT on dashboard
self.browser.open(self.url + '?modul=16') self.browser.open(self.url + "?modul=16")
page = self.browser.get_current_page() page = self.browser.get_current_page()
if page.find(alt='LOGOUT'): if page.find(alt="LOGOUT"):
self.logger.debug('Logged in') self.logger.debug("Logged in")
return True return True
else: else:
self.logger.debug('Not logged in') self.logger.debug("Not logged in")
return False return False
def einsatzprotokoll(self, f_id, pdf_data, webdav_client): def get_einsatzrapport_id(self, f_id, state="open"):
""" Find ID of automatically created Einsatzrapport """
# Login to lodur
self.login()
# Browse to Einsatzrapport page
if state == "open":
self.browser.open("{}?modul=36".format(self.url))
einsatzrapport_url = self.browser.find_link(link_text=f_id)
if einsatzrapport_url:
lodur_id = re.search(
".*event=([0-9]{1,})&.*", einsatzrapport_url["href"]
).group(1)
return lodur_id
else:
return None
def retrieve_form_data(self, lodur_id):
""" Retrieve all fields from an Einsatzrapport in Lodur """
# Login to lodur
self.login()
# Browse to Einsatzrapport page
self.browser.open(
"{}?modul=36&what=144&event={}&edit=1".format(self.url, lodur_id)
)
# Lodur doesn't simply store form field values in the form value field
# LOLNOPE - it is stored in javascript in the variable fdata
# And the data format used is just crap - including mixing of different data types
# WHAT DO THEY ACTUALLY THINK ABOUT THIS!!
# Retrieve all <script></script> tags from page
json_string = None
all_scripts = self.browser.page.find_all("script", type="text/javascript")
# Iterate over all tags to find the one containing fdata
for script in all_scripts:
# Some scripts don't have content - we're not interested in them
if script.contents:
# Search for "var fdata" in all scripts - if found, that's what we're looking for
content = script.contents[0]
if "var fdata" in content:
# Cut out unnecessary "var fdata"
json_string = content.replace("var fdata = ", "")
# Now let's parse that data into a data structure which helps
# in filling out the form and make it usable in Python
if json_string:
# Remove the last character which is a ;
usable = {}
for key, value in json.loads(json_string[:-1]).items():
# WHY DO THEY MIX DIFFERENT TYPES!
if isinstance(value, list):
usable[key] = value[2]
elif isinstance(value, dict):
usable[key] = value["2"]
return usable
else:
return None
def einsatzprotokoll(self, f_id, lodur_data, webdav_client):
""" Prepare Einsatzprotokoll to be sent to Lodur """ """ Prepare Einsatzprotokoll to be sent to Lodur """
# check if data is already sent to lodur - data contains lodur_id self.logger.info("[%s] Updating Lodur entry", f_id)
lodur_data = webdav_client.get_lodur_data(f_id)
if lodur_data: # Complement existing form data
# einsatz available in Lodur - updating existing entry self.logger.info("[%s] Preparing form data for Einsatzprotokoll", f_id)
self.logger.info('[%s] Lodur data found - updating entry', f_id)
# when PDF parsing fails, pdf_data is false. fill with tbd when this happens lodur_data["ztb_m"] = lodur_data[
if pdf_data: "ztv_m"
try: ] # 05. Zeit (copy minute from start to round up to 1h)
zh_fw_ausg = datetime.strptime( lodur_data["eins_ereig"] = "{} - {} - {}".format(
pdf_data['ausgerueckt'], f_id, lodur_data["ala_stich"], lodur_data["adr"]
'%H:%M:%S', ) # 07. Ereignis
) lodur_data["en_kr_feuwehr"] = "1" # 21. Einsatzkräfte
zh_am_schad = datetime.strptime( lodur_data["ali_io"] = "1" # 24. Alarmierung
pdf_data['vorort'], lodur_data["keyword_els_zutreffend"] = "1" # 25. Stichwort
'%H:%M:%S', lodur_data["address_zutreffend"] = "1" # 26. Adresse zutreffend
) lodur_data["kopie_gvz"] = "1" # 31. Kopie innert 10 Tagen an GVZ
except ValueError as err: lodur_data["mannschaftd_einsa"] = "88" # 32. Einsatzleiter|in
self.logger.error('[%s] Date parsing failed: %s', f_id, err)
zh_fw_ausg = datetime.now()
zh_am_schad = datetime.now()
else:
# Do nothing when no PDF data - we don't have anything to do then
self.logger.error('[%s] No PDF data found - filling in dummy data', f_id)
zh_fw_ausg = datetime.now()
zh_am_schad = datetime.now()
# Complement existing form data
self.logger.info('[%s] Preparing form data for Einsatzprotokoll', f_id)
lodur_data['zh_fw_ausg_h'] = zh_fw_ausg.hour # 13. FW ausgerückt
lodur_data['zh_fw_ausg_m'] = zh_fw_ausg.minute # 13. FW ausgerückt
lodur_data['zh_am_schad_h'] = zh_am_schad.hour # 14. Am Schadenplatz
lodur_data['zh_am_schad_m'] = zh_am_schad.minute # 14. Am Schadenplatz
# The following fields are currently unknown as PDF parsing is hard for these
#lodur_data['zh_fw_einge_h'] = UNKNOWN, # 15. FW eingerückt
#lodur_data['zh_fw_einge_m'] = 'UNKNOWN' # 15. FW eingerückt
#lodur_data['eins_erst_h'] = 'UNKNOWN' # 16. Einsatzbereitschaft erstellt
#lodur_data['eins_erst_m'] = 'UNKNOWN' # 16. Einsatzbereitschaft erstellt
# Submit the form
self.submit_form_einsatzrapport(lodur_data)
# save lodur data to webdav
webdav_client.store_lodur_data(f_id, lodur_data)
else:
# einsatz not available in Lodur
self.logger.error('[%s] No lodur_id found')
return False
def einsatzrapport(self, f_id, pdf_data, webdav_client):
""" Prepare form in module 36 - Einsatzrapport """
# when PDF parsing fails, pdf_data is false. fill with placeholder when this happens
if pdf_data:
date = datetime.strptime(
pdf_data['datum'],
'%d.%m.%Y',
)
time = datetime.strptime(
pdf_data['zeit'],
'%H:%M',
)
eins_ereig = pdf_data['einsatz']
bemerkungen = pdf_data['bemerkungen'] + '\n' + pdf_data['disponierteeinheiten']
wer_ala = pdf_data['melder']
adr = pdf_data['ort']
else:
date = datetime.now()
time = datetime.now()
eins_ereig = 'UNKNOWN'
bemerkungen = 'UNKNOWN'
wer_ala = 'UNKNOWN'
adr = 'UNKNOWN'
# Prepare end date and time, can cross midnight
# We blindly add 1 hours - that's the usual length of an Einsatz
time_end = time + timedelta(hours=1)
# check if date is higher after adding 1 hour, this means we crossed midnight
if datetime.date(time_end) > datetime.date(time):
date_end = date + timedelta(days=1)
else:
date_end = date
# Fill in form data
self.logger.info('[%s] Preparing form data for Einsatzrapport', f_id)
lodur_data = {
'e_r_num': f_id, # 01. Einsatzrapportnummer
'eins_stat_kantone': '1', # 02. Einsatzart FKS
'emergency_concept_id': '2', # 03. Verrechnungsart
'ver_sart': 'ab', # 03. Verrechnungsart internal: ab, th, uh, ak, tt
'dtv_d': str(date.day), # 04. Datum von
'dtv_m': str(date.month), # 04. Datum von
'dtv_y': str(date.year), # 04. Datum von
'dtb_d': str(date_end.day), # 04. Datum bis
'dtb_m': str(date_end.month), # 04. Datum bis
'dtb_y': str(date_end.year), # 04. Datum bis
'ztv_h': str(time.hour), # 05. Zeit von
'ztv_m': str(time.minute), # 05. Zeit von
'ztb_h': str(time_end.hour), # 05. Zeit bis - we dont know yet the end time
'ztb_m': str(time_end.minute), # 05. Zeit bis - just add 1 hour and correct later
'e_ort_1': '306', # 06. Einsatzort: Urdorf 306, Birmensdorf 298
'eins_ereig': eins_ereig, # 07. Ereignis
'adr': adr, # 08. Adresse
'wer_ala': wer_ala, # 10. Wer hat alarmiert
'zh_alarmierung_h': str(time.hour), # 12. Alarmierung
'zh_alarmierung_m': str(time.minute), # 12. Alarmierung
'ang_sit': 'TBD1', # 17. Angetroffene Situation
'mn': 'TBD2', # 19. Massnahmen
'bk': bemerkungen, # 20. Bemerkungen
'en_kr_feuwehr': '1', # 21. Einsatzkräfte
'ali_io': '1', # 24. Alarmierung
'kopie_gvz': '1', # 31. Kopie innert 10 Tagen an GVZ
'mannschaftd_einsa': '88', # 32. Einsatzleiter|in
}
# Submit the form # Submit the form
lodur_id, auto_num = self.submit_form_einsatzrapport(lodur_data) self.submit_form_einsatzrapport(lodur_data)
# save lodur id and data to webdav # save lodur data to webdav
lodur_data['event_id'] = lodur_id webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
lodur_data['auto_num'] = auto_num
webdav_client.store_lodur_data(f_id, lodur_data)
return lodur_id def upload_alarmdepesche(self, f_id, file_path, webdav_client):
def einsatzrapport_alarmdepesche(self, f_id, file_path, webdav_client):
""" Upload a file to Alarmdepesche """ """ Upload a file to Alarmdepesche """
self.logger.info('[%s] Submitting file %s to Lodur "Alarmdepesche"', f_id, file_path) self.logger.info(
'[%s] Submitting file %s to Lodur "Alarmdepesche"', f_id, file_path
)
# Login to lodur # Login to lodur
self.login() self.login()
# check if data is already sent to lodur - data contains lodur_id # check if data is already sent to lodur - data contains lodur_id
lodur_id = webdav_client.get_lodur_data(f_id)['event_id'] lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
# Prepare the form # Prepare the form
self.browser.open('{}?modul=36&event={}&what=828'.format(self.url,lodur_id )) self.browser.open("{}?modul=36&event={}&what=828".format(self.url, lodur_id))
frm_alarmdepesche = self.browser.select_form('#frm_alarmdepesche') frm_alarmdepesche = self.browser.select_form("#frm_alarmdepesche")
# Fill in form data # Fill in form data
frm_alarmdepesche.set('alarmdepesche', file_path) frm_alarmdepesche.set("alarmdepesche", file_path)
# Submit the form # Submit the form
self.browser.submit_selected() self.browser.submit_selected()
self.logger.info('[%s] File uploaded to Lodur', f_id) self.logger.info("[%s] File uploaded to Lodur", f_id)
def einsatzrapport_scan(self, f_id, file_path, webdav_client): def einsatzrapport_scan(self, f_id, lodur_data, file_path, webdav_client):
""" Prepare Einsatzrapport Scan to be sent to Lodur """ """ Prepare Einsatzrapport Scan to be sent to Lodur """
# check if data is already sent to lodur - data contains lodur_id # Complement existing form data
lodur_data = webdav_client.get_lodur_data(f_id) self.logger.info("[%s] Updating Lodur entry", f_id)
lodur_data[
"ang_sit"
] = "Siehe Alarmdepesche - Einsatzrapport" # 17. Angetroffene Situation
lodur_data["mn"] = "Siehe Alarmdepesche - Einsatzrapport" # 19. Massnahmen
if lodur_data: # Submit the form
# einsatz available in Lodur - updating existing entry self.submit_form_einsatzrapport(lodur_data)
self.logger.info('[%s] Lodur data found - updating entry', f_id)
# Complement existing form data # Upload scan to Alarmdepesche
self.logger.info('[%s] Preparing form data for Einsatzprotokoll', f_id) self.einsatzrapport_alarmdepesche(
lodur_data['ang_sit'] = 'Siehe Alarmdepesche - Einsatzrapport' # 17. Angetroffene Situation f_id,
lodur_data['mn'] = 'Siehe Alarmdepesche - Einsatzrapport' # 19. Massnahmen file_path,
webdav_client,
# Submit the form )
self.submit_form_einsatzrapport(lodur_data)
# Upload scan to Alarmdepesche
self.einsatzrapport_alarmdepesche(
f_id,
file_path,
webdav_client,
)
else:
# einsatz not available in Lodur
self.logger.error('[%s] No lodur_id found')
return False
def submit_form_einsatzrapport(self, lodur_data): def submit_form_einsatzrapport(self, lodur_data):
""" Form in module 36 - Einsatzrapport """ """ Form in module 36 - Einsatzrapport """
@ -244,65 +199,46 @@ class Lodur:
# Login to lodur # Login to lodur
self.login() self.login()
# Prepare the form f_id = lodur_data["e_r_num"]
if 'event_id' in lodur_data:
# existing entry to update
self.logger.info(
'[%s] Updating existing entry with ID %s',
lodur_data['e_r_num'],
lodur_data['event_id'],
)
self.browser.open(
self.url +
'?modul=36&what=144&edit=1&event=' +
lodur_data['event_id']
)
else:
self.logger.info('[%s] Creating new entry in Lodur', lodur_data['e_r_num'])
self.browser.open(
self.url +
'?modul=36'
)
self.browser.select_form('#einsatzrapport_main_form') self.logger.info(
"[%s] Updating existing entry with ID %s",
f_id,
lodur_data["event_id"],
)
self.browser.open(
self.url + "?modul=36&what=144&edit=1&event=" + lodur_data["event_id"]
)
form = self.browser.select_form("#einsatzrapport_main_form")
# Prepare the form data to be submitted # Prepare the form data to be submitted
for key, value in lodur_data.items(): for key, value in lodur_data.items():
# Not all keys in the parsed Lodur data are actually part of the form
# Encode some of the fields so they are sent in correct format # Encode some of the fields so they are sent in correct format
# Encoding bk causes some troubles - therefore we skip that - but it # Encoding bk causes some troubles - therefore we skip that - but it
# would be good if it would be encoded as it can / will contain f.e.abs # would be good if it would be encoded as it can / will contain f.e.abs
# Umlauts # Umlauts
# AttributeError: 'bytes' object has no attribute 'parent' # AttributeError: 'bytes' object has no attribute 'parent'
self.logger.info('Form data: %s = %s', key, value) try:
if key in ('eins_ereig', 'adr', 'wer_ala'): if key in ("eins_ereig", "adr", "wer_ala"):
self.browser[key] = value.encode('iso-8859-1') form.set(key, value.encode("iso-8859-1"))
else: else:
self.browser[key] = value form.set(key, value)
self.logger.debug("[%s] Set field %s to %s", f_id, key, value)
except mechanicalsoup.LinkNotFoundError as e:
self.logger.debug(
"[%s] Could not set field %s to %s. Reason: %s",
f_id,
key,
value,
str(e),
)
# Submit the form # Submit the form
self.logger.info('[%s] Submitting form Einsatzrapport', lodur_data['e_r_num']) self.logger.info("[%s] Submitting form Einsatzrapport", lodur_data["e_r_num"])
response = self.browser.submit_selected() response = self.browser.submit_selected()
self.logger.info('[%s] Form Einsatzrapport submitted', lodur_data['e_r_num']) self.logger.info("[%s] Form Einsatzrapport submitted", lodur_data["e_r_num"])
if 'event_id' in lodur_data: return True
return True
else:
# very ugly way to find the assigned event id by lodur
# lodur adds a script element at the bottom of the returned html
# with the location to reload the page - containing the assigned event id
# print(response.text)
lodur_id = re.search('modul=36&event=([0-9].*)&edit=1&what=144', response.text).group(1)
self.logger.info('[%s] Lodur assigned the event_id %s', lodur_data['e_r_num'], lodur_id)
# The hidden field auto_num is also needed for updating the form
# and it's written somewhere in javascript code - but not on the page
# delivered after the submission which contains the redirect URL
# It's only delivered in the next page. So we browse to this page now
content = self.browser.open(
self.url +
'?modul=36&edit=1&what=144&event=' + lodur_id
).text
auto_num = re.search(r"\"([0-9]{4}\|[0-9]{1,3})\"", content).group(1)
self.logger.info('[%s] Lodur assigned the auto_num %s', lodur_data['e_r_num'], auto_num)
return lodur_id, auto_num

179
pylokid/library/pdftotext.py
View File

@ -5,109 +5,157 @@
import subprocess import subprocess
import logging import logging
class PDFParsing: class PDFParsing:
""" PDF parsing """ """ PDF parsing """
def __init__(self): def __init__(self):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.logger.info('PDF parsing based on pdftotext loaded') self.logger.info("PDF parsing based on pdftotext loaded")
def extract(self, f_id, file, datafields): def extract(self, f_id, file, datafields):
self.logger.info('[%s] parsing PDF file %s', f_id, file) self.logger.info("[%s] parsing PDF file %s", f_id, file)
data = {} data = {}
for field, coordinate in datafields.items(): for field, coordinate in datafields.items():
# x-coordinate of the crop area top left corner # x-coordinate of the crop area top left corner
x = coordinate['xMin'] x = coordinate["xMin"]
# y-coordinate of the crop area top left corner # y-coordinate of the crop area top left corner
y = coordinate['yMin'] y = coordinate["yMin"]
# width of crop area in pixels # width of crop area in pixels
w = coordinate['xMax'] - coordinate['xMin'] w = coordinate["xMax"] - coordinate["xMin"]
# height of crop area in pixels # height of crop area in pixels
h = coordinate['yMax'] - coordinate['yMin'] h = coordinate["yMax"] - coordinate["yMin"]
self.logger.debug('[%s] Computed command for field %s: %s', f_id, field, self.logger.debug(
'pdftotext -f 1 -l 1 -x {} -y {} -W {} -H {}'.format(x,y,w,h) "[%s] Computed command for field %s: %s",
f_id,
field,
"pdftotext -f 1 -l 1 -x {} -y {} -W {} -H {}".format(x, y, w, h),
) )
scrapeddata = subprocess.Popen([ scrapeddata = subprocess.Popen(
'/usr/bin/pdftotext', [
'-f', '1', "/usr/bin/pdftotext",
'-l', '1', "-f",
'-x', str(x), "1",
'-y', str(y), "-l",
'-W', str(w), "1",
'-H', str(h), "-x",
file, str(x),
'-' "-y",
str(y),
"-W",
str(w),
"-H",
str(h),
file,
"-",
], ],
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stderr=subprocess.STDOUT,
text=True) text=True,
)
stdout, _ = scrapeddata.communicate() stdout, _ = scrapeddata.communicate()
## TODO: fixup some fields (lowercase, remove unnecessary \n) ## TODO: fixup some fields (lowercase, remove unnecessary \n)
if 'edit' in coordinate and coordinate['edit'] == 'title': if "edit" in coordinate and coordinate["edit"] == "title":
data[field] = stdout.rstrip().title() data[field] = stdout.rstrip().title()
else: else:
data[field] = stdout.rstrip() data[field] = stdout.rstrip()
# sanity check to see if we can correlate the f_id # sanity check to see if we can correlate the f_id
if f_id == data['auftrag']: if f_id == data["auftrag"]:
self.logger.debug('[%s] ID matches in PDF', f_id) self.logger.debug("[%s] ID matches in PDF", f_id)
return data return data
else: else:
self.logger.error('[%s] ID does not match in PDF: "%s"', f_id, data['auftrag']) self.logger.error(
'[%s] ID does not match in PDF: "%s"', f_id, data["auftrag"]
)
return False return False
def extract_einsatzausdruck(self, file, f_id): def extract_einsatzausdruck(self, file, f_id):
""" extracts information from Einsatzausdruck using external pdftotext """ """ extracts information from Einsatzausdruck using external pdftotext """
self.logger.debug('[%s] Parsing PDF: %s', f_id, file) self.logger.debug("[%s] Parsing PDF: %s", f_id, file)
# Get them using 'pdftotext -bbox' # Get them using 'pdftotext -bbox'
# y = row # y = row
# x = column: xMax 450 / 590 means full width # x = column: xMax 450 / 590 means full width
coordinates = { coordinates = {
'auftrag': { "auftrag": {
'xMin': 70, 'yMin': 47, 'xMax': 120,'yMax': 58, "xMin": 70,
"yMin": 47,
"xMax": 120,
"yMax": 58,
}, },
'datum': { "datum": {
'xMin': 190, 'yMin': 47, 'xMax': 239, 'yMax': 58, "xMin": 190,
"yMin": 47,
"xMax": 239,
"yMax": 58,
}, },
'zeit': { "zeit": {
'xMin': 190, 'yMin': 59, 'xMax': 215, 'yMax': 70, "xMin": 190,
"yMin": 59,
"xMax": 215,
"yMax": 70,
}, },
'melder': { "melder": {
'xMin': 304, 'yMin': 47, 'xMax': 446, 'yMax': 70, 'edit': 'title' "xMin": 304,
"yMin": 47,
"xMax": 446,
"yMax": 70,
"edit": "title",
}, },
'erfasser':{ "erfasser": {
'xMin': 448, 'yMin': 59, 'xMax': 478, 'yMax': 70, "xMin": 448,
"yMin": 59,
"xMax": 478,
"yMax": 70,
}, },
# big field until "Disponierte Einheiten" # big field until "Disponierte Einheiten"
'bemerkungen': { "bemerkungen": {
'xMin': 28, 'yMin': 112, 'xMax': 590, 'yMax': 350, "xMin": 28,
"yMin": 112,
"xMax": 590,
"yMax": 350,
}, },
'disponierteeinheiten': { "disponierteeinheiten": {
'xMin': 28, 'yMin': 366, 'xMax': 450, 'yMax': 376, "xMin": 28,
"yMin": 366,
"xMax": 450,
"yMax": 376,
}, },
'einsatz': { "einsatz": {
'xMin': 76, 'yMin': 690, 'xMax': 450, 'yMax': 703, "xMin": 76,
"yMin": 690,
"xMax": 450,
"yMax": 703,
}, },
'sondersignal': { "sondersignal": {
'xMin': 76, 'yMin': 707, 'xMax': 450, 'yMax': 721, "xMin": 76,
"yMin": 707,
"xMax": 450,
"yMax": 721,
}, },
'ort': { "ort": {
'xMin': 76, 'yMin': 732, 'xMax': 590, 'yMax': 745, "xMin": 76,
"yMin": 732,
"xMax": 590,
"yMax": 745,
}, },
'hinweis': { "hinweis": {
'xMin': 76, 'yMin': 773, 'xMax': 450, 'yMax': 787, "xMin": 76,
"yMin": 773,
"xMax": 450,
"yMax": 787,
}, },
} }
@ -116,27 +164,42 @@ class PDFParsing:
def extract_einsatzprotokoll(self, file, f_id): def extract_einsatzprotokoll(self, file, f_id):
""" extracts information from Einsatzprotokoll using external pdftotext """ """ extracts information from Einsatzprotokoll using external pdftotext """
self.logger.debug('[%s] Parsing PDF: %s', f_id, file) self.logger.debug("[%s] Parsing PDF: %s", f_id, file)
# Get them using 'pdftotext -bbox' # Get them using 'pdftotext -bbox'
# y = row # y = row
# x = column: xMax 450 / 590 means full width # x = column: xMax 450 / 590 means full width
coordinates = { coordinates = {
'auftrag': { "auftrag": {
'xMin': 192, 'yMin': 132, 'xMax': 238,'yMax': 142, "xMin": 192,
"yMin": 132,
"xMax": 238,
"yMax": 142,
}, },
'angelegt': { "angelegt": {
'xMin': 192, 'yMin': 294, 'xMax': 226, 'yMax': 304, "xMin": 192,
"yMin": 294,
"xMax": 226,
"yMax": 304,
}, },
'dispo': { "dispo": {
'xMin': 192, 'yMin': 312, 'xMax': 226, 'yMax': 322, "xMin": 192,
"yMin": 312,
"xMax": 226,
"yMax": 322,
}, },
'ausgerueckt': { "ausgerueckt": {
'xMin': 192, 'yMin': 331, 'xMax': 226, 'yMax': 341, "xMin": 192,
"yMin": 331,
"xMax": 226,
"yMax": 341,
}, },
'vorort':{ "vorort": {
'xMin': 192, 'yMin': 348, 'xMax': 226, 'yMax': 358, "xMin": 192,
"yMin": 348,
"xMax": 226,
"yMax": 358,
}, },
} }
return self.extract(f_id, file, coordinates) return self.extract(f_id, file, coordinates)

58
pylokid/library/webdav.py
View File

@ -9,12 +9,13 @@ import logging
import asyncio import asyncio
import aioeasywebdav import aioeasywebdav
class WebDav: class WebDav:
""" WebDav Client """ """ WebDav Client """
def __init__(self, url, username, password, webdav_basedir, tmp_dir): def __init__(self, url, username, password, webdav_basedir, tmp_dir):
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.logger.info('Connecting to WebDAV server %s', url) self.logger.info("Connecting to WebDAV server %s", url)
self.loop = asyncio.get_event_loop() self.loop = asyncio.get_event_loop()
self.webdav_basedir = webdav_basedir self.webdav_basedir = webdav_basedir
@ -26,18 +27,20 @@ class WebDav:
password=password, password=password,
) )
except: except:
self.logger.error('WebDAV connection failed - exiting') self.logger.error("WebDAV connection failed - exiting")
self.logger.info('WebDAV connection successfull') self.logger.info("WebDAV connection successfull")
def upload(self, file_name, f_id, check_exists = True): def upload(self, file_name, f_id, check_exists=True):
""" uploads a file to webdav - checks for existence before doing so """ """ uploads a file to webdav - checks for existence before doing so """
# upload with webdav # upload with webdav
if f_id == None: if f_id == None:
remote_upload_dir = self.webdav_basedir + "/Inbox" remote_upload_dir = self.webdav_basedir + "/Inbox"
else: else:
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id remote_upload_dir = (
self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
)
self.logger.info('[%s] Uploading file to WebDAV "%s"', f_id, remote_upload_dir) self.logger.info('[%s] Uploading file to WebDAV "%s"', f_id, remote_upload_dir)
@ -47,7 +50,9 @@ class WebDav:
self.loop.run_until_complete(self.webdav.mkdir(remote_upload_dir)) self.loop.run_until_complete(self.webdav.mkdir(remote_upload_dir))
remote_file_path = remote_upload_dir + "/" + file_name remote_file_path = remote_upload_dir + "/" + file_name
if check_exists and self.loop.run_until_complete(self.webdav.exists(remote_file_path)): if check_exists and self.loop.run_until_complete(
self.webdav.exists(remote_file_path)
):
self.logger.info('[%s] File "%s" already exists on WebDAV', f_id, file_name) self.logger.info('[%s] File "%s" already exists on WebDAV', f_id, file_name)
else: else:
self.loop.run_until_complete( self.loop.run_until_complete(
@ -61,47 +66,52 @@ class WebDav:
def einsatz_exists(self, f_id): def einsatz_exists(self, f_id):
""" check if an einsatz is already created """ """ check if an einsatz is already created """
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id remote_upload_dir = (
self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
)
if self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)): if self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)):
self.logger.info('[%s] Einsatz exists on WebDAV', f_id) self.logger.info("[%s] Einsatz exists on WebDAV", f_id)
return True return True
else: else:
return False return False
def store_lodur_data(self, f_id, lodur_data): def store_data(self, f_id, file_name, data):
""" stores lodur data on webdav """ """ stores data on webdav """
file_name = f_id + '_lodur.json'
file_path = os.path.join(self.tmp_dir, file_name) file_path = os.path.join(self.tmp_dir, file_name)
file = open(file_path, 'w') file = open(file_path, "w")
file.write(json.dumps(lodur_data)) file.write(json.dumps(data))
file.close() file.close()
self.logger.info('[%s] Stored Lodur data locally in %s', f_id, file_path) self.logger.info("[%s] Stored data locally in %s", f_id, file_path)
self.upload(file_name, f_id, False) self.upload(file_name, f_id, False)
def get_lodur_data(self, f_id): def get_lodur_data(self, f_id, filetype="_lodur.json"):
""" gets lodur data if it exists """ """ gets lodur data if it exists """
file_name = f_id + '_lodur.json' file_name = f_id + filetype
file_path = os.path.join(self.tmp_dir, file_name) file_path = os.path.join(self.tmp_dir, file_name)
# first check if we already have it locally - then check on webdav # first check if we already have it locally - then check on webdav
if os.path.isfile(file_path): if os.path.isfile(file_path):
with open(file_path, 'r') as content: with open(file_path, "r") as content:
lodur_data = json.loads(content.read()) lodur_data = json.loads(content.read())
self.logger.info('[%s] Found Lodur data locally', f_id) self.logger.info("[%s] Found Lodur data locally", f_id)
return lodur_data return lodur_data
else: else:
remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id remote_upload_dir = (
remote_file_path = remote_upload_dir + '/' + file_name self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id
)
remote_file_path = remote_upload_dir + "/" + file_name
if self.loop.run_until_complete(self.webdav.exists(remote_file_path)): if self.loop.run_until_complete(self.webdav.exists(remote_file_path)):
self.loop.run_until_complete(self.webdav.download(remote_file_path, file_path)) self.loop.run_until_complete(
with open(file_path, 'r') as content: self.webdav.download(remote_file_path, file_path)
)
with open(file_path, "r") as content:
lodur_data = json.loads(content.read()) lodur_data = json.loads(content.read())
self.logger.info('[%s] Found Lodur data on WebDAV', f_id) self.logger.info("[%s] Found Lodur data on WebDAV", f_id)
return lodur_data return lodur_data
else: else:
self.logger.info('[%s] No existing Lodur data found', f_id) self.logger.info("[%s] No existing Lodur data found", f_id)
return False return False

202
pylokid/main.py
View File

@ -7,6 +7,7 @@ import os
import time import time
import requests import requests
from importlib.metadata import version
from dotenv import find_dotenv, load_dotenv from dotenv import find_dotenv, load_dotenv
from pushover import Client from pushover import Client
@ -34,7 +35,7 @@ LODUR_BASE_URL = os.getenv("LODUR_BASE_URL")
HEARTBEAT_URL = os.getenv("HEARTBEAT_URL") HEARTBEAT_URL = os.getenv("HEARTBEAT_URL")
PUSHOVER_API_TOKEN = os.getenv("PUSHOVER_API_TOKEN") PUSHOVER_API_TOKEN = os.getenv("PUSHOVER_API_TOKEN")
PUSHOVER_USER_KEY = os.getenv("PUSHOVER_USER_KEY") PUSHOVER_USER_KEY = os.getenv("PUSHOVER_USER_KEY")
PYLOKID_VERSION = "2.2.0"
def main(): def main():
""" main """ """ main """
@ -42,10 +43,10 @@ def main():
# Logging configuration # Logging configuration
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
) )
logger = logging.getLogger('pylokid') logger = logging.getLogger("pylokid")
logger.info('Starting pylokid version %s', PYLOKID_VERSION) logger.info("Starting pylokid version %s", version("pylokid"))
# Initialize IMAP Session # Initialize IMAP Session
imap_client = EmailHandling( imap_client = EmailHandling(
@ -73,10 +74,7 @@ def main():
) )
# Initialize Pushover # Initialize Pushover
pushover = Client( pushover = Client(user_key=PUSHOVER_USER_KEY, api_token=PUSHOVER_API_TOKEN)
user_key=PUSHOVER_USER_KEY,
api_token=PUSHOVER_API_TOKEN
)
# Initialize PDF Parser # Initialize PDF Parser
pdf = PDFParsing() pdf = PDFParsing()
@ -97,129 +95,161 @@ def main():
webdav_client.upload(file_name, f_id) webdav_client.upload(file_name, f_id)
# Take actions - depending on the type # Take actions - depending on the type
if f_type == 'Einsatzausdruck_FW': if f_type == "Einsatzausdruck_FW":
logger.info('[%s] Processing type %s', f_id, f_type) logger.info("[%s] Processing type %s", f_id, f_type)
lodur_data = webdav_client.get_lodur_data(f_id)
if lodur_data:
logger.info(
'[%s] Einsatzrapport already created in Lodur', f_id
)
# Upload Alarmdepesche as it could contain more information
# than the first one
lodur_client.einsatzrapport_alarmdepesche(
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
# Check if the PDF isn't already parsed
if webdav_client.get_lodur_data(f_id, "_pdf.json"):
logger.info("[%s] PDF already parsed", f_id)
else: else:
## Here we get the initial Einsatzauftrag - Time to run # Extract information from PDF
# get as many information from PDF as possible
pdf_file = os.path.join(TMP_DIR, file_name)
pdf_data = pdf.extract_einsatzausdruck( pdf_data = pdf.extract_einsatzausdruck(
pdf_file, os.path.join(TMP_DIR, file_name),
f_id, f_id,
) )
# publish Einsatz on Pushover # publish Einsatz on Pushover
logger.info( logger.info("[%s] Publishing message on Pushover", f_id)
'[%s] Publishing message on Pushover', f_id
)
pushover.send_message( pushover.send_message(
"Einsatz {} eröffnet: {}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format( "<b>{}</b>\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format(
f_id, pdf_data["einsatz"],
pdf_data['einsatz'], pdf_data["ort"],
pdf_data['ort'], pdf_data["melder"].replace("\n", " "),
pdf_data['melder'].replace('\n',' '), pdf_data["hinweis"],
pdf_data['hinweis'], pdf_data["sondersignal"],
pdf_data['sondersignal'], pdf_data["bemerkungen"],
pdf_data['disponierteeinheiten'], pdf_data["disponierteeinheiten"],
pdf_data['bemerkungen'],
), ),
title="Feuerwehr Einsatz", title="Feuerwehr Einsatz - {}".format(f_id),
url="https://www.google.com/maps/search/?api=1&query={}".format(pdf_data['ort']), url="https://www.google.com/maps/search/?api=1&query={}".format(
url_title="Ort auf Karte suchen" pdf_data["ort"]
),
url_title="Ort auf Karte suchen",
html=1,
) )
# create new Einsatzrapport in Lodur # Upload extracted data to cloud
lodur_client.einsatzrapport( webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data)
f_id,
pdf_data,
webdav_client,
)
# upload Alarmdepesche PDF to Lodur if webdav_client.get_lodur_data(f_id):
lodur_client.einsatzrapport_alarmdepesche( logger.info("[%s] Lodur data already retrieved", f_id)
f_id, else:
os.path.join(TMP_DIR, file_name), # Retrieve data from Lodur
webdav_client, lodur_id = lodur_client.get_einsatzrapport_id(f_id)
) if lodur_id:
logger.info(
"[%s] Einsatzrapport available in Lodur with ID %s",
f_id,
lodur_id,
)
logger.info(
"%s?modul=36&what=144&event=%s&edit=1",
LODUR_BASE_URL,
lodur_id,
)
elif f_type == 'Einsatzprotokoll': lodur_data = lodur_client.retrieve_form_data(lodur_id)
logger.info('[%s] Processing type %s', f_id, f_type) webdav_client.store_data(
f_id, f_id + "_lodur.json", lodur_data
)
# upload Alarmdepesche PDF to Lodur
lodur_client.upload_alarmdepesche(
f_id,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
# Marking message as seen, no need to reprocess again
for msg_id in msg_ids:
logger.info("[%s] Marking E-Mail message as seen", f_id)
imap_client.mark_seen(msg_id)
else:
logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id)
elif f_type == "Einsatzprotokoll":
lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
logger.info(
"[%s] Processing type %s with Lodur ID %s",
f_id,
f_type,
lodur_id,
)
# Retrieve Lodur data again and store it in Webdav
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
if (
"aut_created_report" in lodur_data
and lodur_data["aut_created_report"] == "finished"
):
logger.info("[%s] Record in Lodur ready to be updated", f_id)
lodur_data = webdav_client.get_lodur_data(f_id)
if lodur_data:
# Upload Einsatzprotokoll to Lodur # Upload Einsatzprotokoll to Lodur
lodur_client.einsatzrapport_alarmdepesche( lodur_client.upload_alarmdepesche(
f_id, f_id,
os.path.join(TMP_DIR, file_name), os.path.join(TMP_DIR, file_name),
webdav_client, webdav_client,
) )
# Parse the Einsatzprotokoll PDF # Update entry in Lodur
pdf_file = os.path.join(TMP_DIR, file_name) lodur_client.einsatzprotokoll(f_id, lodur_data, webdav_client)
pdf_data = pdf.extract_einsatzprotokoll(
pdf_file,
f_id,
)
# Update entry in Lodur with parsed PDF data
lodur_client.einsatzprotokoll(f_id, pdf_data, webdav_client)
# Einsatz finished - publish on pushover # Einsatz finished - publish on pushover
logger.info( logger.info("[%s] Publishing message on Pushover", f_id)
'[%s] Publishing message on Pushover', f_id
)
pushover.send_message( pushover.send_message(
"Einsatz {} beendet".format(f_id), "Einsatz {} beendet".format(f_id),
title="Feuerwehr Einsatz beendet", title="Feuerwehr Einsatz beendet - {}".format(f_id),
) )
# Marking message as seen, no need to reprocess again
for msg_id in msg_ids:
logger.info("[%s] Marking E-Mail message as seen", f_id)
imap_client.mark_seen(msg_id)
else: else:
logger.error( logger.warn(
'[%s] Cannot process Einsatzprotokoll as there is no Lodur ID', "[%s] Record in Lodur NOT ready yet to be updated", f_id
f_id
) )
# This is usually a scan from the Depot printer # This is usually a scan from the Depot printer
elif f_type == 'Einsatzrapport': elif f_type == "Einsatzrapport":
logger.info('[%s] Processing type %s', f_id, f_type)
logger.info("[%s] Processing type %s", f_id, f_type)
# Attach scan in Lodur if f_id is available # Attach scan in Lodur if f_id is available
# f_id can be empty when scan was misconfigured
if f_id != None: if f_id != None:
pdf_file = os.path.join(TMP_DIR, file_name) lodur_id = webdav_client.get_lodur_data(f_id)["event_id"]
lodur_client.einsatzrapport_scan(f_id, pdf_file, webdav_client) # Retrieve Lodur data again and store it in Webdav
lodur_data = lodur_client.retrieve_form_data(lodur_id)
webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data)
lodur_client.einsatzrapport_scan(
f_id,
lodur_data,
os.path.join(TMP_DIR, file_name),
webdav_client,
)
logger.info( logger.info("[%s] Publishing message on Pushover", f_id)
'[%s] Publishing message on Pushover', f_id
)
pushover.send_message( pushover.send_message(
"Scan {} wurde bearbeitet und in Cloud geladen".format(f_id), "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id),
title="Feuerwehr Scan bearbeitet", title="Feuerwehr Scan bearbeitet - {}".format(f_id),
) )
else: else:
logger.error('[%s] Unknown type: %s', f_id, f_type) logger.error("[%s] Unknown type: %s", f_id, f_type)
# send heartbeat # send heartbeat
requests.get(HEARTBEAT_URL) requests.get(HEARTBEAT_URL)
# repeat every # repeat every
logger.info('Waiting %s seconds until next check', IMAP_CHECK_INTERVAL) logger.info("Waiting %s seconds until next check", IMAP_CHECK_INTERVAL)
time.sleep(int(IMAP_CHECK_INTERVAL)) time.sleep(int(IMAP_CHECK_INTERVAL))
if __name__ == '__main__':
if __name__ == "__main__":
try: try:
main() main()
except KeyboardInterrupt: except KeyboardInterrupt:

4
pyproject.toml
View File

@ -1,6 +1,6 @@
[tool.poetry] [tool.poetry]
name = "pylokid" name = "pylokid"
version = "2.2.0" version = "3.0.0"
description = "" description = ""
authors = ["Tobias Brunner <tobias@tobru.ch>"] authors = ["Tobias Brunner <tobias@tobru.ch>"]
license = "MIT" license = "MIT"
@ -14,6 +14,8 @@ python-pushover = "^0.4"
MechanicalSoup = "^1.0.0" MechanicalSoup = "^1.0.0"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
flake8 = "^3.8.4"
black = "^20.8b1"
[build-system] [build-system]
requires = ["poetry-core>=1.0.0"] requires = ["poetry-core>=1.0.0"]

30
test_pdftotext.py
View File

@ -1,30 +0,0 @@
import re
import logging
from pprint import pprint
from pathlib import Path
from library.pdftotext import PDFParsing
PATH = '/home/tobru/Documents/Feuerwehr/Stab/Fourier/Einsatzdepeschen/2019'
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
PDF = PDFParsing()
for path in Path(PATH).glob('**/Einsatzausdruck*.pdf'):
file = str(path)
print(file)
f_id = re.search('.*(F[0-9]{8})_.*', file).group(1)
print(f_id)
pprint(PDF.extract_einsatzausdruck(file, f_id))
"""
for path in Path(PATH).glob('**/Einsatzprotokoll*.pdf'):
file = str(path)
print(file)
f_id = re.search('.*(F[0-9]{8})_.*', file).group(1)
print(f_id)
pprint(PDF.extract_einsatzprotokoll(file, f_id))
"""