From 9c8c5396c8d7bd0ce65dfb21b2b94d936f978fa9 Mon Sep 17 00:00:00 2001 From: Tobias Brunner Date: Sun, 14 Feb 2021 15:00:15 +0100 Subject: [PATCH 1/2] major update to reuse already existing records Lodur since some time automatically creates Einsatzrapporte via an API from SRZ/GVZ. One of the main features of Pylokid was to exactly do that. With that new change this isn't necessary anymore. Pylokid has been amended to find the pre-existing entry and work with that - enhancing it with any additional information missing and uploads PDFs to the right place. While at it a very small modernization has been made and the project moved to use Poetry and Black formatting. But it's still the same ugly code - to reflect Lodur. --- poetry.lock | 231 ++++++++++++++++++- pylokid/__init__.py | 2 +- pylokid/library/emailhandling.py | 52 +++-- pylokid/library/lodur.py | 366 +++++++++++++------------------ pylokid/library/pdftotext.py | 179 ++++++++++----- pylokid/library/webdav.py | 58 +++-- pylokid/main.py | 202 +++++++++-------- pyproject.toml | 4 +- test_pdftotext.py | 30 --- 9 files changed, 684 insertions(+), 440 deletions(-) delete mode 100644 test_pdftotext.py diff --git a/poetry.lock b/poetry.lock index 1a8214f..ce15630 100644 --- a/poetry.lock +++ b/poetry.lock @@ -30,6 +30,14 @@ yarl = ">=1.0,<2.0" [package.extras] speedups = ["aiodns", "brotlipy", "cchardet"] +[[package]] +name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "async-timeout" version = "3.0.1" @@ -67,6 +75,28 @@ soupsieve = {version = ">1.2", markers = "python_version >= \"3.0\""} html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +name = "black" +version = "20.8b1" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +appdirs = "*" +click = ">=7.1.2" +mypy-extensions = ">=0.4.3" +pathspec = ">=0.6,<1" +regex = ">=2020.1.8" +toml = ">=0.10.1" +typed-ast = ">=1.4.0" +typing-extensions = ">=3.7.4" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] + [[package]] name = "certifi" version = "2020.12.5" @@ -83,6 +113,27 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "click" +version = "7.1.2" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "flake8" +version = "3.8.4" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +mccabe = ">=0.6.0,<0.7.0" +pycodestyle = ">=2.6.0a1,<2.7.0" +pyflakes = ">=2.2.0,<2.3.0" + [[package]] name = "idna" version = "2.10" @@ -105,6 +156,14 @@ html5 = ["html5lib"] htmlsoup = ["beautifulsoup4"] source = ["Cython (>=0.29.7)"] +[[package]] +name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "mechanicalsoup" version = "1.0.0" @@ -127,6 +186,38 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "pathspec" +version = "0.8.1" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "pycodestyle" +version = "2.6.0" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pyflakes" +version = "2.2.0" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + [[package]] name = "python-dotenv" version = "0.15.0" @@ -149,6 +240,14 @@ python-versions = "*" [package.dependencies] requests = ">=1.0" +[[package]] +name = "regex" +version = "2020.11.13" +description = "Alternative regular expression module, to replace re." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "requests" version = "2.25.1" @@ -194,6 +293,22 @@ category = "main" optional = false python-versions = ">=3.6" +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "typed-ast" +version = "1.4.2" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "typing-extensions" version = "3.7.4.3" @@ -230,7 +345,7 @@ multidict = ">=4.0" [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "c5fa2a2589b88adb6a9bf94ec4a88e0e6b9528e40837cf6b2cf045a3e072d6d4" +content-hash = "87029b7a633e874d04d308355076802812613a9b93a03770d10ab6b9fcbc5b44" [metadata.files] aioeasywebdav = [ @@ -276,6 +391,10 @@ aiohttp = [ {file = "aiohttp-3.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:e1b95972a0ae3f248a899cdbac92ba2e01d731225f566569311043ce2226f5e7"}, {file = "aiohttp-3.7.3.tar.gz", hash = "sha256:9c1a81af067e72261c9cbe33ea792893e83bc6aa987bfbd6fdc1e5e7b22777c4"}, ] +appdirs = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] async-timeout = [ {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, @@ -289,6 +408,9 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.3-py3-none-any.whl", hash = "sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"}, {file = "beautifulsoup4-4.9.3.tar.gz", hash = "sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25"}, ] +black = [ + {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, +] certifi = [ {file = "certifi-2020.12.5-py2.py3-none-any.whl", hash = "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"}, {file = "certifi-2020.12.5.tar.gz", hash = "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c"}, @@ -297,6 +419,14 @@ chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, ] +click = [ + {file = "click-7.1.2-py2.py3-none-any.whl", hash = "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"}, + {file = "click-7.1.2.tar.gz", hash = "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a"}, +] +flake8 = [ + {file = "flake8-3.8.4-py2.py3-none-any.whl", hash = "sha256:749dbbd6bfd0cf1318af27bf97a14e28e5ff548ef8e5b1566ccfb25a11e7c839"}, + {file = "flake8-3.8.4.tar.gz", hash = "sha256:aadae8761ec651813c24be05c6f7b4680857ef6afaae4651a4eccaef97ce6c3b"}, +] idna = [ {file = "idna-2.10-py2.py3-none-any.whl", hash = "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"}, {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, @@ -340,6 +470,10 @@ lxml = [ {file = "lxml-4.6.2-cp39-cp39-win_amd64.whl", hash = "sha256:535332fe9d00c3cd455bd3dd7d4bacab86e2d564bdf7606079160fa6251caacf"}, {file = "lxml-4.6.2.tar.gz", hash = "sha256:cd11c7e8d21af997ee8079037fff88f16fda188a9776eb4b81c7e4c9c0a7d7fc"}, ] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] mechanicalsoup = [ {file = "MechanicalSoup-1.0.0-py2.py3-none-any.whl", hash = "sha256:2ed9a494c144fb2c262408dcbd5c79e1ef325a7426f1fa3a3443eaef133b5f77"}, {file = "MechanicalSoup-1.0.0.tar.gz", hash = "sha256:37d3b15c1957917d3ae171561e77f4dd4c08c35eb4500b8781f6e7e1bb6c4d07"}, @@ -383,6 +517,22 @@ multidict = [ {file = "multidict-5.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:7df80d07818b385f3129180369079bd6934cf70469f99daaebfac89dca288359"}, {file = "multidict-5.1.0.tar.gz", hash = "sha256:25b4e5f22d3a37ddf3effc0710ba692cfc792c2b9edfb9c05aefe823256e84d5"}, ] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +pathspec = [ + {file = "pathspec-0.8.1-py2.py3-none-any.whl", hash = "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"}, + {file = "pathspec-0.8.1.tar.gz", hash = "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd"}, +] +pycodestyle = [ + {file = "pycodestyle-2.6.0-py2.py3-none-any.whl", hash = "sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367"}, + {file = "pycodestyle-2.6.0.tar.gz", hash = "sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e"}, +] +pyflakes = [ + {file = "pyflakes-2.2.0-py2.py3-none-any.whl", hash = "sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92"}, + {file = "pyflakes-2.2.0.tar.gz", hash = "sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8"}, +] python-dotenv = [ {file = "python-dotenv-0.15.0.tar.gz", hash = "sha256:587825ed60b1711daea4832cf37524dfd404325b7db5e25ebe88c495c9f807a0"}, {file = "python_dotenv-0.15.0-py2.py3-none-any.whl", hash = "sha256:0c8d1b80d1a1e91717ea7d526178e3882732420b03f08afea0406db6402e220e"}, @@ -390,6 +540,49 @@ python-dotenv = [ python-pushover = [ {file = "python-pushover-0.4.tar.gz", hash = "sha256:dee1b1344fb8a5874365fc9f886d9cbc7775536629999be54dfa60177cf80810"}, ] +regex = [ + {file = "regex-2020.11.13-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:8b882a78c320478b12ff024e81dc7d43c1462aa4a3341c754ee65d857a521f85"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:a63f1a07932c9686d2d416fb295ec2c01ab246e89b4d58e5fa468089cab44b70"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:6e4b08c6f8daca7d8f07c8d24e4331ae7953333dbd09c648ed6ebd24db5a10ee"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:bba349276b126947b014e50ab3316c027cac1495992f10e5682dc677b3dfa0c5"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:56e01daca75eae420bce184edd8bb341c8eebb19dd3bce7266332258f9fb9dd7"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:6a8ce43923c518c24a2579fda49f093f1397dad5d18346211e46f134fc624e31"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_i686.whl", hash = "sha256:1ab79fcb02b930de09c76d024d279686ec5d532eb814fd0ed1e0051eb8bd2daa"}, + {file = "regex-2020.11.13-cp36-cp36m-manylinux2014_x86_64.whl", hash = "sha256:9801c4c1d9ae6a70aeb2128e5b4b68c45d4f0af0d1535500884d644fa9b768c6"}, + {file = "regex-2020.11.13-cp36-cp36m-win32.whl", hash = "sha256:49cae022fa13f09be91b2c880e58e14b6da5d10639ed45ca69b85faf039f7a4e"}, + {file = "regex-2020.11.13-cp36-cp36m-win_amd64.whl", hash = "sha256:749078d1eb89484db5f34b4012092ad14b327944ee7f1c4f74d6279a6e4d1884"}, + {file = "regex-2020.11.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b2f4007bff007c96a173e24dcda236e5e83bde4358a557f9ccf5e014439eae4b"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:38c8fd190db64f513fe4e1baa59fed086ae71fa45083b6936b52d34df8f86a88"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5862975b45d451b6db51c2e654990c1820523a5b07100fc6903e9c86575202a0"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:262c6825b309e6485ec2493ffc7e62a13cf13fb2a8b6d212f72bd53ad34118f1"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:bafb01b4688833e099d79e7efd23f99172f501a15c44f21ea2118681473fdba0"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:e32f5f3d1b1c663af7f9c4c1e72e6ffe9a78c03a31e149259f531e0fed826512"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:3bddc701bdd1efa0d5264d2649588cbfda549b2899dc8d50417e47a82e1387ba"}, + {file = "regex-2020.11.13-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:02951b7dacb123d8ea6da44fe45ddd084aa6777d4b2454fa0da61d569c6fa538"}, + {file = "regex-2020.11.13-cp37-cp37m-win32.whl", hash = "sha256:0d08e71e70c0237883d0bef12cad5145b84c3705e9c6a588b2a9c7080e5af2a4"}, + {file = "regex-2020.11.13-cp37-cp37m-win_amd64.whl", hash = "sha256:1fa7ee9c2a0e30405e21031d07d7ba8617bc590d391adfc2b7f1e8b99f46f444"}, + {file = "regex-2020.11.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:baf378ba6151f6e272824b86a774326f692bc2ef4cc5ce8d5bc76e38c813a55f"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e3faaf10a0d1e8e23a9b51d1900b72e1635c2d5b0e1bea1c18022486a8e2e52d"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2a11a3e90bd9901d70a5b31d7dd85114755a581a5da3fc996abfefa48aee78af"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d1ebb090a426db66dd80df8ca85adc4abfcbad8a7c2e9a5ec7513ede522e0a8f"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:b2b1a5ddae3677d89b686e5c625fc5547c6e492bd755b520de5332773a8af06b"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2c99e97d388cd0a8d30f7c514d67887d8021541b875baf09791a3baad48bb4f8"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:c084582d4215593f2f1d28b65d2a2f3aceff8342aa85afd7be23a9cad74a0de5"}, + {file = "regex-2020.11.13-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:a3d748383762e56337c39ab35c6ed4deb88df5326f97a38946ddd19028ecce6b"}, + {file = "regex-2020.11.13-cp38-cp38-win32.whl", hash = "sha256:7913bd25f4ab274ba37bc97ad0e21c31004224ccb02765ad984eef43e04acc6c"}, + {file = "regex-2020.11.13-cp38-cp38-win_amd64.whl", hash = "sha256:6c54ce4b5d61a7129bad5c5dc279e222afd00e721bf92f9ef09e4fae28755683"}, + {file = "regex-2020.11.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1862a9d9194fae76a7aaf0150d5f2a8ec1da89e8b55890b1786b8f88a0f619dc"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux1_i686.whl", hash = "sha256:4902e6aa086cbb224241adbc2f06235927d5cdacffb2425c73e6570e8d862364"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7a25fcbeae08f96a754b45bdc050e1fb94b95cab046bf56b016c25e9ab127b3e"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:d2d8ce12b7c12c87e41123997ebaf1a5767a5be3ec545f64675388970f415e2e"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:f7d29a6fc4760300f86ae329e3b6ca28ea9c20823df123a2ea8693e967b29917"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:717881211f46de3ab130b58ec0908267961fadc06e44f974466d1887f865bd5b"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:3128e30d83f2e70b0bed9b2a34e92707d0877e460b402faca908c6667092ada9"}, + {file = "regex-2020.11.13-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:8f6a2229e8ad946e36815f2a03386bb8353d4bde368fdf8ca5f0cb97264d3b5c"}, + {file = "regex-2020.11.13-cp39-cp39-win32.whl", hash = "sha256:f8f295db00ef5f8bae530fc39af0b40486ca6068733fb860b42115052206466f"}, + {file = "regex-2020.11.13-cp39-cp39-win_amd64.whl", hash = "sha256:a15f64ae3a027b64496a71ab1f722355e570c3fac5ba2801cafce846bf5af01d"}, + {file = "regex-2020.11.13.tar.gz", hash = "sha256:83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562"}, +] requests = [ {file = "requests-2.25.1-py2.py3-none-any.whl", hash = "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"}, {file = "requests-2.25.1.tar.gz", hash = "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804"}, @@ -412,6 +605,42 @@ soupsieve = [ {file = "soupsieve-2.2-py3-none-any.whl", hash = "sha256:d3a5ea5b350423f47d07639f74475afedad48cf41c0ad7a82ca13a3928af34f6"}, {file = "soupsieve-2.2.tar.gz", hash = "sha256:407fa1e8eb3458d1b5614df51d9651a1180ea5fedf07feb46e45d7e25e6d6cdd"}, ] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +typed-ast = [ + {file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70"}, + {file = "typed_ast-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487"}, + {file = "typed_ast-1.4.2-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412"}, + {file = "typed_ast-1.4.2-cp35-cp35m-win32.whl", hash = "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400"}, + {file = "typed_ast-1.4.2-cp35-cp35m-win_amd64.whl", hash = "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606"}, + {file = "typed_ast-1.4.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64"}, + {file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07"}, + {file = "typed_ast-1.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc"}, + {file = "typed_ast-1.4.2-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a"}, + {file = "typed_ast-1.4.2-cp36-cp36m-win32.whl", hash = "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151"}, + {file = "typed_ast-1.4.2-cp36-cp36m-win_amd64.whl", hash = "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3"}, + {file = "typed_ast-1.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41"}, + {file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f"}, + {file = "typed_ast-1.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581"}, + {file = "typed_ast-1.4.2-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37"}, + {file = "typed_ast-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd"}, + {file = "typed_ast-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496"}, + {file = "typed_ast-1.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc"}, + {file = "typed_ast-1.4.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10"}, + {file = "typed_ast-1.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea"}, + {file = "typed_ast-1.4.2-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787"}, + {file = "typed_ast-1.4.2-cp38-cp38-win32.whl", hash = "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2"}, + {file = "typed_ast-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937"}, + {file = "typed_ast-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1"}, + {file = "typed_ast-1.4.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6"}, + {file = "typed_ast-1.4.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166"}, + {file = "typed_ast-1.4.2-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d"}, + {file = "typed_ast-1.4.2-cp39-cp39-win32.whl", hash = "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b"}, + {file = "typed_ast-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440"}, + {file = "typed_ast-1.4.2.tar.gz", hash = "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a"}, +] typing-extensions = [ {file = "typing_extensions-3.7.4.3-py2-none-any.whl", hash = "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"}, {file = "typing_extensions-3.7.4.3-py3-none-any.whl", hash = "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918"}, diff --git a/pylokid/__init__.py b/pylokid/__init__.py index 53bc1a7..5146f93 100644 --- a/pylokid/__init__.py +++ b/pylokid/__init__.py @@ -2,6 +2,6 @@ Pylokid. From Mail to Lodur - all automated. """ -__version__ = "2.2.0" +__version__ = "3.0.0" __git_version__ = "0" __url__ = "https://github.com/tobru/pylokid" diff --git a/pylokid/library/emailhandling.py b/pylokid/library/emailhandling.py index 8317b7b..f2f704c 100644 --- a/pylokid/library/emailhandling.py +++ b/pylokid/library/emailhandling.py @@ -12,12 +12,13 @@ import imaplib _EMAIL_SUBJECTS = '(OR OR SUBJECT "Einsatzausdruck_FW" SUBJECT "Einsatzprotokoll" SUBJECT "Einsatzrapport" UNSEEN)' + class EmailHandling: """ Email handling """ def __init__(self, server, username, password, mailbox, tmp_dir): self.logger = logging.getLogger(__name__) - self.logger.info('Connecting to IMAP server %s', server) + self.logger.info("Connecting to IMAP server %s", server) self.tmp_dir = tmp_dir socket.setdefaulttimeout(60) @@ -27,86 +28,89 @@ class EmailHandling: self.imap.login(username, password) self.imap.select(mailbox, readonly=False) except Exception as err: - self.logger.error('IMAP connection failed - exiting: %s', str(err)) + self.logger.error("IMAP connection failed - exiting: %s", str(err)) raise SystemExit(1) - self.logger.info('IMAP connection successfull') + self.logger.info("IMAP connection successful") def search_emails(self): """ searches for emails matching the configured subject """ - self.logger.info('Searching for messages matching: %s', _EMAIL_SUBJECTS) + self.logger.info("Searching for messages matching: %s", _EMAIL_SUBJECTS) try: typ, msg_ids = self.imap.search( None, _EMAIL_SUBJECTS, ) - if typ != 'OK': - self.logger.error('Error searching for matching messages') + if typ != "OK": + self.logger.error("Error searching for matching messages") return False except imaplib.IMAP4.abort as err: - self.logger.error('IMAP search aborted - exiting: %s', str(err)) + self.logger.error("IMAP search aborted - exiting: %s", str(err)) raise SystemExit(1) num_messages = len(msg_ids[0].split()) - self.logger.info('Found %s matching messages', str(num_messages)) + self.logger.info("Found %s matching messages", str(num_messages)) return num_messages, msg_ids def store_attachments(self, msg_ids): - """ stores the attachments to filesystem and marks message as read """ + """ stores the attachments to filesystem """ data = {} for msg_id in msg_ids[0].split(): # download message from imap - typ, msg_data = self.imap.fetch(msg_id, '(RFC822)') + typ, msg_data = self.imap.fetch(msg_id, "(RFC822)") - if typ != 'OK': - self.logger.error('Error fetching message') + if typ != "OK": + self.logger.error("Error fetching message") continue # extract attachment for response_part in msg_data: if isinstance(response_part, tuple): - mail = email.message_from_string(str(response_part[1], 'utf-8')) - subject = mail['subject'] + mail = email.message_from_string(str(response_part[1], "utf-8")) + subject = mail["subject"] f_type, f_id = self.parse_subject(subject) self.logger.info('[%s] Getting attachment from "%s"', f_id, subject) for part in mail.walk(): file_name = part.get_filename() if not file_name: self.logger.debug( - 'Most probably not an attachment as no filename found' + "Most probably not an attachment as no filename found" ) continue - self.logger.info('[%s] Extracting attachment "%s"', f_id, file_name) + self.logger.info( + '[%s] Extracting attachment "%s"', f_id, file_name + ) if bool(file_name): f_type, _ = self.parse_subject(subject) - renamed_file_name = f_type + '_' + file_name + renamed_file_name = f_type + "_" + file_name # save attachment to filesystem file_path = os.path.join(self.tmp_dir, renamed_file_name) - self.logger.info('[%s] Saving attachment to "%s"', f_id, file_path) + self.logger.info( + '[%s] Saving attachment to "%s"', f_id, file_path + ) if not os.path.isfile(file_path): - file = open(file_path, 'wb') + file = open(file_path, "wb") file.write(part.get_payload(decode=True)) file.close() data[subject] = renamed_file_name - # mark as seen - self.logger.info('[%s] Marking message "%s" as seen', f_id, subject) - self.imap.store(msg_id, '+FLAGS', '(\\Seen)') - return data + def mark_seen(self, msg_id): + self.imap.store(msg_id, "+FLAGS", "(\\Seen)") + def parse_subject(self, subject): """ extract f id and type from subject """ # This regex matches the subjects filtered already in IMAP search - parsed = re.search('([a-zA-Z_]*):? ?(F[0-9].*)?', subject) + parsed = re.search("([a-zA-Z_]*):? ?(F[0-9].*)?", subject) f_type = parsed.group(1) f_id = parsed.group(2) diff --git a/pylokid/library/lodur.py b/pylokid/library/lodur.py index 1096d13..02040d4 100644 --- a/pylokid/library/lodur.py +++ b/pylokid/library/lodur.py @@ -4,16 +4,18 @@ import re import logging -from datetime import datetime -from datetime import timedelta +import json import mechanicalsoup +import pprint +from datetime import datetime, timedelta + class Lodur: """ Lodur """ def __init__(self, url, username, password): self.logger = logging.getLogger(__name__) - self.logger.info('Connecting to Lodur') + self.logger.info("Connecting to Lodur") self.url = url self.username = username @@ -24,219 +26,172 @@ class Lodur: self.login() if self.logged_in(): - self.logger.info('Login to Lodur succeeded') + self.logger.info("Login to Lodur succeeded") else: - self.logger.fatal('Login to Lodur failed - exiting') + self.logger.fatal("Login to Lodur failed - exiting") raise SystemExit(1) def login(self): """ Login to lodur """ # The login form is located in module number 9 - self.browser.open(self.url + '?modul=9') + self.browser.open(self.url + "?modul=9") # only log in when not yed logged in if not self.logged_in(): # open login page again as the logged_in function has navigated to another page - self.browser.open(self.url + '?modul=9') + self.browser.open(self.url + "?modul=9") self.browser.select_form() - self.browser['login_member_name'] = self.username - self.browser['login_member_pwd'] = self.password + self.browser["login_member_name"] = self.username + self.browser["login_member_pwd"] = self.password self.browser.submit_selected() def logged_in(self): """ check if logged in to lodur - session is valid """ # Check if login succeeded by finding the img with # alt text LOGOUT on dashboard - self.browser.open(self.url + '?modul=16') + self.browser.open(self.url + "?modul=16") page = self.browser.get_current_page() - if page.find(alt='LOGOUT'): - self.logger.debug('Logged in') + if page.find(alt="LOGOUT"): + self.logger.debug("Logged in") return True else: - self.logger.debug('Not logged in') + self.logger.debug("Not logged in") return False - def einsatzprotokoll(self, f_id, pdf_data, webdav_client): + def get_einsatzrapport_id(self, f_id, state="open"): + """ Find ID of automatically created Einsatzrapport """ + + # Login to lodur + self.login() + + # Browse to Einsatzrapport page + if state == "open": + self.browser.open("{}?modul=36".format(self.url)) + + einsatzrapport_url = self.browser.find_link(link_text=f_id) + if einsatzrapport_url: + lodur_id = re.search( + ".*event=([0-9]{1,})&.*", einsatzrapport_url["href"] + ).group(1) + return lodur_id + else: + return None + + def retrieve_form_data(self, lodur_id): + """ Retrieve all fields from an Einsatzrapport in Lodur """ + + # Login to lodur + self.login() + + # Browse to Einsatzrapport page + self.browser.open( + "{}?modul=36&what=144&event={}&edit=1".format(self.url, lodur_id) + ) + + # Lodur doesn't simply store form field values in the form value field + # LOLNOPE - it is stored in javascript in the variable fdata + # And the data format used is just crap - including mixing of different data types + # WHAT DO THEY ACTUALLY THINK ABOUT THIS!! + + # Retrieve all tags from page + json_string = None + all_scripts = self.browser.page.find_all("script", type="text/javascript") + # Iterate over all tags to find the one containing fdata + for script in all_scripts: + # Some scripts don't have content - we're not interested in them + if script.contents: + # Search for "var fdata" in all scripts - if found, that's what we're looking for + content = script.contents[0] + if "var fdata" in content: + # Cut out unnecessary "var fdata" + json_string = content.replace("var fdata = ", "") + + # Now let's parse that data into a data structure which helps + # in filling out the form and make it usable in Python + if json_string: + # Remove the last character which is a ; + usable = {} + for key, value in json.loads(json_string[:-1]).items(): + # WHY DO THEY MIX DIFFERENT TYPES! + if isinstance(value, list): + usable[key] = value[2] + elif isinstance(value, dict): + usable[key] = value["2"] + return usable + else: + return None + + def einsatzprotokoll(self, f_id, lodur_data, webdav_client): """ Prepare Einsatzprotokoll to be sent to Lodur """ - # check if data is already sent to lodur - data contains lodur_id - lodur_data = webdav_client.get_lodur_data(f_id) + self.logger.info("[%s] Updating Lodur entry", f_id) - if lodur_data: - # einsatz available in Lodur - updating existing entry - self.logger.info('[%s] Lodur data found - updating entry', f_id) + # Complement existing form data + self.logger.info("[%s] Preparing form data for Einsatzprotokoll", f_id) - # when PDF parsing fails, pdf_data is false. fill with tbd when this happens - if pdf_data: - try: - zh_fw_ausg = datetime.strptime( - pdf_data['ausgerueckt'], - '%H:%M:%S', - ) - zh_am_schad = datetime.strptime( - pdf_data['vorort'], - '%H:%M:%S', - ) - except ValueError as err: - self.logger.error('[%s] Date parsing failed: %s', f_id, err) - zh_fw_ausg = datetime.now() - zh_am_schad = datetime.now() - else: - # Do nothing when no PDF data - we don't have anything to do then - self.logger.error('[%s] No PDF data found - filling in dummy data', f_id) - zh_fw_ausg = datetime.now() - zh_am_schad = datetime.now() - - # Complement existing form data - self.logger.info('[%s] Preparing form data for Einsatzprotokoll', f_id) - lodur_data['zh_fw_ausg_h'] = zh_fw_ausg.hour # 13. FW ausgerückt - lodur_data['zh_fw_ausg_m'] = zh_fw_ausg.minute # 13. FW ausgerückt - lodur_data['zh_am_schad_h'] = zh_am_schad.hour # 14. Am Schadenplatz - lodur_data['zh_am_schad_m'] = zh_am_schad.minute # 14. Am Schadenplatz - # The following fields are currently unknown as PDF parsing is hard for these - #lodur_data['zh_fw_einge_h'] = UNKNOWN, # 15. FW eingerückt - #lodur_data['zh_fw_einge_m'] = 'UNKNOWN' # 15. FW eingerückt - #lodur_data['eins_erst_h'] = 'UNKNOWN' # 16. Einsatzbereitschaft erstellt - #lodur_data['eins_erst_m'] = 'UNKNOWN' # 16. Einsatzbereitschaft erstellt - - # Submit the form - self.submit_form_einsatzrapport(lodur_data) - - # save lodur data to webdav - webdav_client.store_lodur_data(f_id, lodur_data) - - else: - # einsatz not available in Lodur - self.logger.error('[%s] No lodur_id found') - return False - - def einsatzrapport(self, f_id, pdf_data, webdav_client): - """ Prepare form in module 36 - Einsatzrapport """ - - # when PDF parsing fails, pdf_data is false. fill with placeholder when this happens - if pdf_data: - date = datetime.strptime( - pdf_data['datum'], - '%d.%m.%Y', - ) - time = datetime.strptime( - pdf_data['zeit'], - '%H:%M', - ) - eins_ereig = pdf_data['einsatz'] - bemerkungen = pdf_data['bemerkungen'] + '\n' + pdf_data['disponierteeinheiten'] - wer_ala = pdf_data['melder'] - adr = pdf_data['ort'] - else: - date = datetime.now() - time = datetime.now() - eins_ereig = 'UNKNOWN' - bemerkungen = 'UNKNOWN' - wer_ala = 'UNKNOWN' - adr = 'UNKNOWN' - - # Prepare end date and time, can cross midnight - # We blindly add 1 hours - that's the usual length of an Einsatz - time_end = time + timedelta(hours=1) - # check if date is higher after adding 1 hour, this means we crossed midnight - if datetime.date(time_end) > datetime.date(time): - date_end = date + timedelta(days=1) - else: - date_end = date - - # Fill in form data - self.logger.info('[%s] Preparing form data for Einsatzrapport', f_id) - lodur_data = { - 'e_r_num': f_id, # 01. Einsatzrapportnummer - 'eins_stat_kantone': '1', # 02. Einsatzart FKS - 'emergency_concept_id': '2', # 03. Verrechnungsart - 'ver_sart': 'ab', # 03. Verrechnungsart internal: ab, th, uh, ak, tt - 'dtv_d': str(date.day), # 04. Datum von - 'dtv_m': str(date.month), # 04. Datum von - 'dtv_y': str(date.year), # 04. Datum von - 'dtb_d': str(date_end.day), # 04. Datum bis - 'dtb_m': str(date_end.month), # 04. Datum bis - 'dtb_y': str(date_end.year), # 04. Datum bis - 'ztv_h': str(time.hour), # 05. Zeit von - 'ztv_m': str(time.minute), # 05. Zeit von - 'ztb_h': str(time_end.hour), # 05. Zeit bis - we dont know yet the end time - 'ztb_m': str(time_end.minute), # 05. Zeit bis - just add 1 hour and correct later - 'e_ort_1': '306', # 06. Einsatzort: Urdorf 306, Birmensdorf 298 - 'eins_ereig': eins_ereig, # 07. Ereignis - 'adr': adr, # 08. Adresse - 'wer_ala': wer_ala, # 10. Wer hat alarmiert - 'zh_alarmierung_h': str(time.hour), # 12. Alarmierung - 'zh_alarmierung_m': str(time.minute), # 12. Alarmierung - 'ang_sit': 'TBD1', # 17. Angetroffene Situation - 'mn': 'TBD2', # 19. Massnahmen - 'bk': bemerkungen, # 20. Bemerkungen - 'en_kr_feuwehr': '1', # 21. Einsatzkräfte - 'ali_io': '1', # 24. Alarmierung - 'kopie_gvz': '1', # 31. Kopie innert 10 Tagen an GVZ - 'mannschaftd_einsa': '88', # 32. Einsatzleiter|in - } + lodur_data["ztb_m"] = lodur_data[ + "ztv_m" + ] # 05. Zeit (copy minute from start to round up to 1h) + lodur_data["eins_ereig"] = "{} - {} - {}".format( + f_id, lodur_data["ala_stich"], lodur_data["adr"] + ) # 07. Ereignis + lodur_data["en_kr_feuwehr"] = "1" # 21. Einsatzkräfte + lodur_data["ali_io"] = "1" # 24. Alarmierung + lodur_data["keyword_els_zutreffend"] = "1" # 25. Stichwort + lodur_data["address_zutreffend"] = "1" # 26. Adresse zutreffend + lodur_data["kopie_gvz"] = "1" # 31. Kopie innert 10 Tagen an GVZ + lodur_data["mannschaftd_einsa"] = "88" # 32. Einsatzleiter|in # Submit the form - lodur_id, auto_num = self.submit_form_einsatzrapport(lodur_data) + self.submit_form_einsatzrapport(lodur_data) - # save lodur id and data to webdav - lodur_data['event_id'] = lodur_id - lodur_data['auto_num'] = auto_num - webdav_client.store_lodur_data(f_id, lodur_data) + # save lodur data to webdav + webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) - return lodur_id - - def einsatzrapport_alarmdepesche(self, f_id, file_path, webdav_client): + def upload_alarmdepesche(self, f_id, file_path, webdav_client): """ Upload a file to Alarmdepesche """ - self.logger.info('[%s] Submitting file %s to Lodur "Alarmdepesche"', f_id, file_path) + self.logger.info( + '[%s] Submitting file %s to Lodur "Alarmdepesche"', f_id, file_path + ) # Login to lodur self.login() # check if data is already sent to lodur - data contains lodur_id - lodur_id = webdav_client.get_lodur_data(f_id)['event_id'] + lodur_id = webdav_client.get_lodur_data(f_id)["event_id"] # Prepare the form - self.browser.open('{}?modul=36&event={}&what=828'.format(self.url,lodur_id )) - frm_alarmdepesche = self.browser.select_form('#frm_alarmdepesche') + self.browser.open("{}?modul=36&event={}&what=828".format(self.url, lodur_id)) + frm_alarmdepesche = self.browser.select_form("#frm_alarmdepesche") # Fill in form data - frm_alarmdepesche.set('alarmdepesche', file_path) + frm_alarmdepesche.set("alarmdepesche", file_path) # Submit the form self.browser.submit_selected() - self.logger.info('[%s] File uploaded to Lodur', f_id) + self.logger.info("[%s] File uploaded to Lodur", f_id) - def einsatzrapport_scan(self, f_id, file_path, webdav_client): + def einsatzrapport_scan(self, f_id, lodur_data, file_path, webdav_client): """ Prepare Einsatzrapport Scan to be sent to Lodur """ - # check if data is already sent to lodur - data contains lodur_id - lodur_data = webdav_client.get_lodur_data(f_id) + # Complement existing form data + self.logger.info("[%s] Updating Lodur entry", f_id) + lodur_data[ + "ang_sit" + ] = "Siehe Alarmdepesche - Einsatzrapport" # 17. Angetroffene Situation + lodur_data["mn"] = "Siehe Alarmdepesche - Einsatzrapport" # 19. Massnahmen - if lodur_data: - # einsatz available in Lodur - updating existing entry - self.logger.info('[%s] Lodur data found - updating entry', f_id) + # Submit the form + self.submit_form_einsatzrapport(lodur_data) - # Complement existing form data - self.logger.info('[%s] Preparing form data for Einsatzprotokoll', f_id) - lodur_data['ang_sit'] = 'Siehe Alarmdepesche - Einsatzrapport' # 17. Angetroffene Situation - lodur_data['mn'] = 'Siehe Alarmdepesche - Einsatzrapport' # 19. Massnahmen - - # Submit the form - self.submit_form_einsatzrapport(lodur_data) - - # Upload scan to Alarmdepesche - self.einsatzrapport_alarmdepesche( - f_id, - file_path, - webdav_client, - ) - else: - # einsatz not available in Lodur - self.logger.error('[%s] No lodur_id found') - return False + # Upload scan to Alarmdepesche + self.einsatzrapport_alarmdepesche( + f_id, + file_path, + webdav_client, + ) def submit_form_einsatzrapport(self, lodur_data): """ Form in module 36 - Einsatzrapport """ @@ -244,65 +199,46 @@ class Lodur: # Login to lodur self.login() - # Prepare the form - if 'event_id' in lodur_data: - # existing entry to update - self.logger.info( - '[%s] Updating existing entry with ID %s', - lodur_data['e_r_num'], - lodur_data['event_id'], - ) - self.browser.open( - self.url + - '?modul=36&what=144&edit=1&event=' + - lodur_data['event_id'] - ) - else: - self.logger.info('[%s] Creating new entry in Lodur', lodur_data['e_r_num']) - self.browser.open( - self.url + - '?modul=36' - ) + f_id = lodur_data["e_r_num"] - self.browser.select_form('#einsatzrapport_main_form') + self.logger.info( + "[%s] Updating existing entry with ID %s", + f_id, + lodur_data["event_id"], + ) + + self.browser.open( + self.url + "?modul=36&what=144&edit=1&event=" + lodur_data["event_id"] + ) + + form = self.browser.select_form("#einsatzrapport_main_form") # Prepare the form data to be submitted for key, value in lodur_data.items(): + # Not all keys in the parsed Lodur data are actually part of the form # Encode some of the fields so they are sent in correct format # Encoding bk causes some troubles - therefore we skip that - but it # would be good if it would be encoded as it can / will contain f.e.abs # Umlauts # AttributeError: 'bytes' object has no attribute 'parent' - self.logger.info('Form data: %s = %s', key, value) - if key in ('eins_ereig', 'adr', 'wer_ala'): - self.browser[key] = value.encode('iso-8859-1') - else: - self.browser[key] = value + try: + if key in ("eins_ereig", "adr", "wer_ala"): + form.set(key, value.encode("iso-8859-1")) + else: + form.set(key, value) + self.logger.debug("[%s] Set field %s to %s", f_id, key, value) + except mechanicalsoup.LinkNotFoundError as e: + self.logger.debug( + "[%s] Could not set field %s to %s. Reason: %s", + f_id, + key, + value, + str(e), + ) # Submit the form - self.logger.info('[%s] Submitting form Einsatzrapport', lodur_data['e_r_num']) + self.logger.info("[%s] Submitting form Einsatzrapport", lodur_data["e_r_num"]) response = self.browser.submit_selected() - self.logger.info('[%s] Form Einsatzrapport submitted', lodur_data['e_r_num']) + self.logger.info("[%s] Form Einsatzrapport submitted", lodur_data["e_r_num"]) - if 'event_id' in lodur_data: - return True - else: - # very ugly way to find the assigned event id by lodur - # lodur adds a script element at the bottom of the returned html - # with the location to reload the page - containing the assigned event id - # print(response.text) - lodur_id = re.search('modul=36&event=([0-9].*)&edit=1&what=144', response.text).group(1) - self.logger.info('[%s] Lodur assigned the event_id %s', lodur_data['e_r_num'], lodur_id) - - # The hidden field auto_num is also needed for updating the form - # and it's written somewhere in javascript code - but not on the page - # delivered after the submission which contains the redirect URL - # It's only delivered in the next page. So we browse to this page now - content = self.browser.open( - self.url + - '?modul=36&edit=1&what=144&event=' + lodur_id - ).text - auto_num = re.search(r"\"([0-9]{4}\|[0-9]{1,3})\"", content).group(1) - self.logger.info('[%s] Lodur assigned the auto_num %s', lodur_data['e_r_num'], auto_num) - - return lodur_id, auto_num + return True diff --git a/pylokid/library/pdftotext.py b/pylokid/library/pdftotext.py index de4e186..f86d736 100644 --- a/pylokid/library/pdftotext.py +++ b/pylokid/library/pdftotext.py @@ -5,109 +5,157 @@ import subprocess import logging + class PDFParsing: """ PDF parsing """ def __init__(self): self.logger = logging.getLogger(__name__) - self.logger.info('PDF parsing based on pdftotext loaded') + self.logger.info("PDF parsing based on pdftotext loaded") def extract(self, f_id, file, datafields): - self.logger.info('[%s] parsing PDF file %s', f_id, file) + self.logger.info("[%s] parsing PDF file %s", f_id, file) data = {} for field, coordinate in datafields.items(): # x-coordinate of the crop area top left corner - x = coordinate['xMin'] + x = coordinate["xMin"] # y-coordinate of the crop area top left corner - y = coordinate['yMin'] + y = coordinate["yMin"] # width of crop area in pixels - w = coordinate['xMax'] - coordinate['xMin'] + w = coordinate["xMax"] - coordinate["xMin"] # height of crop area in pixels - h = coordinate['yMax'] - coordinate['yMin'] + h = coordinate["yMax"] - coordinate["yMin"] - self.logger.debug('[%s] Computed command for field %s: %s', f_id, field, - 'pdftotext -f 1 -l 1 -x {} -y {} -W {} -H {}'.format(x,y,w,h) + self.logger.debug( + "[%s] Computed command for field %s: %s", + f_id, + field, + "pdftotext -f 1 -l 1 -x {} -y {} -W {} -H {}".format(x, y, w, h), ) - scrapeddata = subprocess.Popen([ - '/usr/bin/pdftotext', - '-f', '1', - '-l', '1', - '-x', str(x), - '-y', str(y), - '-W', str(w), - '-H', str(h), - file, - '-' + scrapeddata = subprocess.Popen( + [ + "/usr/bin/pdftotext", + "-f", + "1", + "-l", + "1", + "-x", + str(x), + "-y", + str(y), + "-W", + str(w), + "-H", + str(h), + file, + "-", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True) + text=True, + ) stdout, _ = scrapeddata.communicate() ## TODO: fixup some fields (lowercase, remove unnecessary \n) - if 'edit' in coordinate and coordinate['edit'] == 'title': + if "edit" in coordinate and coordinate["edit"] == "title": data[field] = stdout.rstrip().title() else: data[field] = stdout.rstrip() # sanity check to see if we can correlate the f_id - if f_id == data['auftrag']: - self.logger.debug('[%s] ID matches in PDF', f_id) + if f_id == data["auftrag"]: + self.logger.debug("[%s] ID matches in PDF", f_id) return data else: - self.logger.error('[%s] ID does not match in PDF: "%s"', f_id, data['auftrag']) + self.logger.error( + '[%s] ID does not match in PDF: "%s"', f_id, data["auftrag"] + ) return False def extract_einsatzausdruck(self, file, f_id): """ extracts information from Einsatzausdruck using external pdftotext """ - self.logger.debug('[%s] Parsing PDF: %s', f_id, file) + self.logger.debug("[%s] Parsing PDF: %s", f_id, file) # Get them using 'pdftotext -bbox' # y = row # x = column: xMax 450 / 590 means full width coordinates = { - 'auftrag': { - 'xMin': 70, 'yMin': 47, 'xMax': 120,'yMax': 58, + "auftrag": { + "xMin": 70, + "yMin": 47, + "xMax": 120, + "yMax": 58, }, - 'datum': { - 'xMin': 190, 'yMin': 47, 'xMax': 239, 'yMax': 58, + "datum": { + "xMin": 190, + "yMin": 47, + "xMax": 239, + "yMax": 58, }, - 'zeit': { - 'xMin': 190, 'yMin': 59, 'xMax': 215, 'yMax': 70, + "zeit": { + "xMin": 190, + "yMin": 59, + "xMax": 215, + "yMax": 70, }, - 'melder': { - 'xMin': 304, 'yMin': 47, 'xMax': 446, 'yMax': 70, 'edit': 'title' + "melder": { + "xMin": 304, + "yMin": 47, + "xMax": 446, + "yMax": 70, + "edit": "title", }, - 'erfasser':{ - 'xMin': 448, 'yMin': 59, 'xMax': 478, 'yMax': 70, + "erfasser": { + "xMin": 448, + "yMin": 59, + "xMax": 478, + "yMax": 70, }, # big field until "Disponierte Einheiten" - 'bemerkungen': { - 'xMin': 28, 'yMin': 112, 'xMax': 590, 'yMax': 350, + "bemerkungen": { + "xMin": 28, + "yMin": 112, + "xMax": 590, + "yMax": 350, }, - 'disponierteeinheiten': { - 'xMin': 28, 'yMin': 366, 'xMax': 450, 'yMax': 376, + "disponierteeinheiten": { + "xMin": 28, + "yMin": 366, + "xMax": 450, + "yMax": 376, }, - 'einsatz': { - 'xMin': 76, 'yMin': 690, 'xMax': 450, 'yMax': 703, + "einsatz": { + "xMin": 76, + "yMin": 690, + "xMax": 450, + "yMax": 703, }, - 'sondersignal': { - 'xMin': 76, 'yMin': 707, 'xMax': 450, 'yMax': 721, + "sondersignal": { + "xMin": 76, + "yMin": 707, + "xMax": 450, + "yMax": 721, }, - 'ort': { - 'xMin': 76, 'yMin': 732, 'xMax': 590, 'yMax': 745, + "ort": { + "xMin": 76, + "yMin": 732, + "xMax": 590, + "yMax": 745, }, - 'hinweis': { - 'xMin': 76, 'yMin': 773, 'xMax': 450, 'yMax': 787, + "hinweis": { + "xMin": 76, + "yMin": 773, + "xMax": 450, + "yMax": 787, }, } @@ -116,27 +164,42 @@ class PDFParsing: def extract_einsatzprotokoll(self, file, f_id): """ extracts information from Einsatzprotokoll using external pdftotext """ - self.logger.debug('[%s] Parsing PDF: %s', f_id, file) + self.logger.debug("[%s] Parsing PDF: %s", f_id, file) # Get them using 'pdftotext -bbox' # y = row # x = column: xMax 450 / 590 means full width coordinates = { - 'auftrag': { - 'xMin': 192, 'yMin': 132, 'xMax': 238,'yMax': 142, + "auftrag": { + "xMin": 192, + "yMin": 132, + "xMax": 238, + "yMax": 142, }, - 'angelegt': { - 'xMin': 192, 'yMin': 294, 'xMax': 226, 'yMax': 304, + "angelegt": { + "xMin": 192, + "yMin": 294, + "xMax": 226, + "yMax": 304, }, - 'dispo': { - 'xMin': 192, 'yMin': 312, 'xMax': 226, 'yMax': 322, + "dispo": { + "xMin": 192, + "yMin": 312, + "xMax": 226, + "yMax": 322, }, - 'ausgerueckt': { - 'xMin': 192, 'yMin': 331, 'xMax': 226, 'yMax': 341, + "ausgerueckt": { + "xMin": 192, + "yMin": 331, + "xMax": 226, + "yMax": 341, }, - 'vorort':{ - 'xMin': 192, 'yMin': 348, 'xMax': 226, 'yMax': 358, + "vorort": { + "xMin": 192, + "yMin": 348, + "xMax": 226, + "yMax": 358, }, } - return self.extract(f_id, file, coordinates) \ No newline at end of file + return self.extract(f_id, file, coordinates) diff --git a/pylokid/library/webdav.py b/pylokid/library/webdav.py index 5c1b5bb..624f0fe 100644 --- a/pylokid/library/webdav.py +++ b/pylokid/library/webdav.py @@ -9,12 +9,13 @@ import logging import asyncio import aioeasywebdav + class WebDav: """ WebDav Client """ def __init__(self, url, username, password, webdav_basedir, tmp_dir): self.logger = logging.getLogger(__name__) - self.logger.info('Connecting to WebDAV server %s', url) + self.logger.info("Connecting to WebDAV server %s", url) self.loop = asyncio.get_event_loop() self.webdav_basedir = webdav_basedir @@ -26,18 +27,20 @@ class WebDav: password=password, ) except: - self.logger.error('WebDAV connection failed - exiting') + self.logger.error("WebDAV connection failed - exiting") - self.logger.info('WebDAV connection successfull') + self.logger.info("WebDAV connection successfull") - def upload(self, file_name, f_id, check_exists = True): + def upload(self, file_name, f_id, check_exists=True): """ uploads a file to webdav - checks for existence before doing so """ # upload with webdav if f_id == None: remote_upload_dir = self.webdav_basedir + "/Inbox" else: - remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id + remote_upload_dir = ( + self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id + ) self.logger.info('[%s] Uploading file to WebDAV "%s"', f_id, remote_upload_dir) @@ -47,7 +50,9 @@ class WebDav: self.loop.run_until_complete(self.webdav.mkdir(remote_upload_dir)) remote_file_path = remote_upload_dir + "/" + file_name - if check_exists and self.loop.run_until_complete(self.webdav.exists(remote_file_path)): + if check_exists and self.loop.run_until_complete( + self.webdav.exists(remote_file_path) + ): self.logger.info('[%s] File "%s" already exists on WebDAV', f_id, file_name) else: self.loop.run_until_complete( @@ -61,47 +66,52 @@ class WebDav: def einsatz_exists(self, f_id): """ check if an einsatz is already created """ - remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id + remote_upload_dir = ( + self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id + ) if self.loop.run_until_complete(self.webdav.exists(remote_upload_dir)): - self.logger.info('[%s] Einsatz exists on WebDAV', f_id) + self.logger.info("[%s] Einsatz exists on WebDAV", f_id) return True else: return False - def store_lodur_data(self, f_id, lodur_data): - """ stores lodur data on webdav """ + def store_data(self, f_id, file_name, data): + """ stores data on webdav """ - file_name = f_id + '_lodur.json' file_path = os.path.join(self.tmp_dir, file_name) - file = open(file_path, 'w') - file.write(json.dumps(lodur_data)) + file = open(file_path, "w") + file.write(json.dumps(data)) file.close() - self.logger.info('[%s] Stored Lodur data locally in %s', f_id, file_path) + self.logger.info("[%s] Stored data locally in %s", f_id, file_path) self.upload(file_name, f_id, False) - def get_lodur_data(self, f_id): + def get_lodur_data(self, f_id, filetype="_lodur.json"): """ gets lodur data if it exists """ - file_name = f_id + '_lodur.json' + file_name = f_id + filetype file_path = os.path.join(self.tmp_dir, file_name) # first check if we already have it locally - then check on webdav if os.path.isfile(file_path): - with open(file_path, 'r') as content: + with open(file_path, "r") as content: lodur_data = json.loads(content.read()) - self.logger.info('[%s] Found Lodur data locally', f_id) + self.logger.info("[%s] Found Lodur data locally", f_id) return lodur_data else: - remote_upload_dir = self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id - remote_file_path = remote_upload_dir + '/' + file_name + remote_upload_dir = ( + self.webdav_basedir + "/" + str(datetime.now().year) + "/" + f_id + ) + remote_file_path = remote_upload_dir + "/" + file_name if self.loop.run_until_complete(self.webdav.exists(remote_file_path)): - self.loop.run_until_complete(self.webdav.download(remote_file_path, file_path)) - with open(file_path, 'r') as content: + self.loop.run_until_complete( + self.webdav.download(remote_file_path, file_path) + ) + with open(file_path, "r") as content: lodur_data = json.loads(content.read()) - self.logger.info('[%s] Found Lodur data on WebDAV', f_id) + self.logger.info("[%s] Found Lodur data on WebDAV", f_id) return lodur_data else: - self.logger.info('[%s] No existing Lodur data found', f_id) + self.logger.info("[%s] No existing Lodur data found", f_id) return False diff --git a/pylokid/main.py b/pylokid/main.py index 44eee87..2983e83 100644 --- a/pylokid/main.py +++ b/pylokid/main.py @@ -7,6 +7,7 @@ import os import time import requests +from importlib.metadata import version from dotenv import find_dotenv, load_dotenv from pushover import Client @@ -34,7 +35,7 @@ LODUR_BASE_URL = os.getenv("LODUR_BASE_URL") HEARTBEAT_URL = os.getenv("HEARTBEAT_URL") PUSHOVER_API_TOKEN = os.getenv("PUSHOVER_API_TOKEN") PUSHOVER_USER_KEY = os.getenv("PUSHOVER_USER_KEY") -PYLOKID_VERSION = "2.2.0" + def main(): """ main """ @@ -42,10 +43,10 @@ def main(): # Logging configuration logging.basicConfig( level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", ) - logger = logging.getLogger('pylokid') - logger.info('Starting pylokid version %s', PYLOKID_VERSION) + logger = logging.getLogger("pylokid") + logger.info("Starting pylokid version %s", version("pylokid")) # Initialize IMAP Session imap_client = EmailHandling( @@ -73,10 +74,7 @@ def main(): ) # Initialize Pushover - pushover = Client( - user_key=PUSHOVER_USER_KEY, - api_token=PUSHOVER_API_TOKEN - ) + pushover = Client(user_key=PUSHOVER_USER_KEY, api_token=PUSHOVER_API_TOKEN) # Initialize PDF Parser pdf = PDFParsing() @@ -97,129 +95,161 @@ def main(): webdav_client.upload(file_name, f_id) # Take actions - depending on the type - if f_type == 'Einsatzausdruck_FW': - logger.info('[%s] Processing type %s', f_id, f_type) - lodur_data = webdav_client.get_lodur_data(f_id) - - if lodur_data: - logger.info( - '[%s] Einsatzrapport already created in Lodur', f_id - ) - # Upload Alarmdepesche as it could contain more information - # than the first one - lodur_client.einsatzrapport_alarmdepesche( - f_id, - os.path.join(TMP_DIR, file_name), - webdav_client, - ) + if f_type == "Einsatzausdruck_FW": + logger.info("[%s] Processing type %s", f_id, f_type) + # Check if the PDF isn't already parsed + if webdav_client.get_lodur_data(f_id, "_pdf.json"): + logger.info("[%s] PDF already parsed", f_id) else: - ## Here we get the initial Einsatzauftrag - Time to run - # get as many information from PDF as possible - pdf_file = os.path.join(TMP_DIR, file_name) + # Extract information from PDF pdf_data = pdf.extract_einsatzausdruck( - pdf_file, + os.path.join(TMP_DIR, file_name), f_id, ) # publish Einsatz on Pushover - logger.info( - '[%s] Publishing message on Pushover', f_id - ) + logger.info("[%s] Publishing message on Pushover", f_id) pushover.send_message( - "Einsatz {} eröffnet: {}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format( - f_id, - pdf_data['einsatz'], - pdf_data['ort'], - pdf_data['melder'].replace('\n',' '), - pdf_data['hinweis'], - pdf_data['sondersignal'], - pdf_data['disponierteeinheiten'], - pdf_data['bemerkungen'], + "{}\n\n* Ort: {}\n* Melder: {}\n* Hinweis: {}\n* {}\n\n{}\n\n{}".format( + pdf_data["einsatz"], + pdf_data["ort"], + pdf_data["melder"].replace("\n", " "), + pdf_data["hinweis"], + pdf_data["sondersignal"], + pdf_data["bemerkungen"], + pdf_data["disponierteeinheiten"], ), - title="Feuerwehr Einsatz", - url="https://www.google.com/maps/search/?api=1&query={}".format(pdf_data['ort']), - url_title="Ort auf Karte suchen" + title="Feuerwehr Einsatz - {}".format(f_id), + url="https://www.google.com/maps/search/?api=1&query={}".format( + pdf_data["ort"] + ), + url_title="Ort auf Karte suchen", + html=1, ) - # create new Einsatzrapport in Lodur - lodur_client.einsatzrapport( - f_id, - pdf_data, - webdav_client, - ) + # Upload extracted data to cloud + webdav_client.store_data(f_id, f_id + "_pdf.json", pdf_data) - # upload Alarmdepesche PDF to Lodur - lodur_client.einsatzrapport_alarmdepesche( - f_id, - os.path.join(TMP_DIR, file_name), - webdav_client, - ) + if webdav_client.get_lodur_data(f_id): + logger.info("[%s] Lodur data already retrieved", f_id) + else: + # Retrieve data from Lodur + lodur_id = lodur_client.get_einsatzrapport_id(f_id) + if lodur_id: + logger.info( + "[%s] Einsatzrapport available in Lodur with ID %s", + f_id, + lodur_id, + ) + logger.info( + "%s?modul=36&what=144&event=%s&edit=1", + LODUR_BASE_URL, + lodur_id, + ) - elif f_type == 'Einsatzprotokoll': - logger.info('[%s] Processing type %s', f_id, f_type) + lodur_data = lodur_client.retrieve_form_data(lodur_id) + webdav_client.store_data( + f_id, f_id + "_lodur.json", lodur_data + ) + + # upload Alarmdepesche PDF to Lodur + lodur_client.upload_alarmdepesche( + f_id, + os.path.join(TMP_DIR, file_name), + webdav_client, + ) + + # Marking message as seen, no need to reprocess again + for msg_id in msg_ids: + logger.info("[%s] Marking E-Mail message as seen", f_id) + imap_client.mark_seen(msg_id) + else: + logger.warn("[%s] Einsatzrapport NOT found in Lodur", f_id) + + elif f_type == "Einsatzprotokoll": + + lodur_id = webdav_client.get_lodur_data(f_id)["event_id"] + logger.info( + "[%s] Processing type %s with Lodur ID %s", + f_id, + f_type, + lodur_id, + ) + + # Retrieve Lodur data again and store it in Webdav + lodur_data = lodur_client.retrieve_form_data(lodur_id) + webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) + + if ( + "aut_created_report" in lodur_data + and lodur_data["aut_created_report"] == "finished" + ): + logger.info("[%s] Record in Lodur ready to be updated", f_id) - lodur_data = webdav_client.get_lodur_data(f_id) - if lodur_data: # Upload Einsatzprotokoll to Lodur - lodur_client.einsatzrapport_alarmdepesche( + lodur_client.upload_alarmdepesche( f_id, os.path.join(TMP_DIR, file_name), webdav_client, ) - # Parse the Einsatzprotokoll PDF - pdf_file = os.path.join(TMP_DIR, file_name) - pdf_data = pdf.extract_einsatzprotokoll( - pdf_file, - f_id, - ) - - # Update entry in Lodur with parsed PDF data - lodur_client.einsatzprotokoll(f_id, pdf_data, webdav_client) + # Update entry in Lodur + lodur_client.einsatzprotokoll(f_id, lodur_data, webdav_client) # Einsatz finished - publish on pushover - logger.info( - '[%s] Publishing message on Pushover', f_id - ) + logger.info("[%s] Publishing message on Pushover", f_id) pushover.send_message( "Einsatz {} beendet".format(f_id), - title="Feuerwehr Einsatz beendet", + title="Feuerwehr Einsatz beendet - {}".format(f_id), ) + # Marking message as seen, no need to reprocess again + for msg_id in msg_ids: + logger.info("[%s] Marking E-Mail message as seen", f_id) + imap_client.mark_seen(msg_id) + else: - logger.error( - '[%s] Cannot process Einsatzprotokoll as there is no Lodur ID', - f_id + logger.warn( + "[%s] Record in Lodur NOT ready yet to be updated", f_id ) # This is usually a scan from the Depot printer - elif f_type == 'Einsatzrapport': - logger.info('[%s] Processing type %s', f_id, f_type) + elif f_type == "Einsatzrapport": + + logger.info("[%s] Processing type %s", f_id, f_type) # Attach scan in Lodur if f_id is available + # f_id can be empty when scan was misconfigured if f_id != None: - pdf_file = os.path.join(TMP_DIR, file_name) - lodur_client.einsatzrapport_scan(f_id, pdf_file, webdav_client) + lodur_id = webdav_client.get_lodur_data(f_id)["event_id"] + # Retrieve Lodur data again and store it in Webdav + lodur_data = lodur_client.retrieve_form_data(lodur_id) + webdav_client.store_data(f_id, f_id + "_lodur.json", lodur_data) + lodur_client.einsatzrapport_scan( + f_id, + lodur_data, + os.path.join(TMP_DIR, file_name), + webdav_client, + ) - logger.info( - '[%s] Publishing message on Pushover', f_id - ) + logger.info("[%s] Publishing message on Pushover", f_id) pushover.send_message( "Scan {} wurde bearbeitet und in Cloud geladen".format(f_id), - title="Feuerwehr Scan bearbeitet", + title="Feuerwehr Scan bearbeitet - {}".format(f_id), ) else: - logger.error('[%s] Unknown type: %s', f_id, f_type) + logger.error("[%s] Unknown type: %s", f_id, f_type) # send heartbeat requests.get(HEARTBEAT_URL) # repeat every - logger.info('Waiting %s seconds until next check', IMAP_CHECK_INTERVAL) + logger.info("Waiting %s seconds until next check", IMAP_CHECK_INTERVAL) time.sleep(int(IMAP_CHECK_INTERVAL)) -if __name__ == '__main__': + +if __name__ == "__main__": try: main() except KeyboardInterrupt: diff --git a/pyproject.toml b/pyproject.toml index deb5932..a7c0070 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pylokid" -version = "2.2.0" +version = "3.0.0" description = "" authors = ["Tobias Brunner "] license = "MIT" @@ -14,6 +14,8 @@ python-pushover = "^0.4" MechanicalSoup = "^1.0.0" [tool.poetry.dev-dependencies] +flake8 = "^3.8.4" +black = "^20.8b1" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/test_pdftotext.py b/test_pdftotext.py deleted file mode 100644 index 86e1f0b..0000000 --- a/test_pdftotext.py +++ /dev/null @@ -1,30 +0,0 @@ -import re -import logging -from pprint import pprint -from pathlib import Path -from library.pdftotext import PDFParsing - -PATH = '/home/tobru/Documents/Feuerwehr/Stab/Fourier/Einsatzdepeschen/2019' - -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) - -PDF = PDFParsing() - -for path in Path(PATH).glob('**/Einsatzausdruck*.pdf'): - file = str(path) - print(file) - f_id = re.search('.*(F[0-9]{8})_.*', file).group(1) - print(f_id) - pprint(PDF.extract_einsatzausdruck(file, f_id)) - -""" -for path in Path(PATH).glob('**/Einsatzprotokoll*.pdf'): - file = str(path) - print(file) - f_id = re.search('.*(F[0-9]{8})_.*', file).group(1) - print(f_id) - pprint(PDF.extract_einsatzprotokoll(file, f_id)) -""" From c0ec11d8048078c600b3e295710b79a30b99e906 Mon Sep 17 00:00:00 2001 From: Tobias Brunner Date: Tue, 23 Feb 2021 20:15:45 +0100 Subject: [PATCH 2/2] update README --- README.md | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6fd6d4c..64e6da5 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ pylokid funktioniert so: "Einsatzausdruck_FW" oder "Einsatzprotokoll". * Wird ein passendes E-Mail gefunden, wird der Anhang (das PDF) heruntergeladen, in die Cloud gespeichert (WebDAV) und im Lodur - ein entsprechender Einsatzrapport eröffnet und vorausgefüllt. + der entsprechende Einsatzrapport gesucht. Das PDF wird sinnvoll umbenannt und als Alarmdepesche ins Lodur geladen. * Kommen weitere E-Mails mit dem Betreff "Einsatzausdruck_FW" werden @@ -42,6 +42,7 @@ pylokid funktioniert so: * Wird der von Hand ausgefüllte Einsatzrapport via Scanner per E-Mail an das E-Mail Postfach gesendet (Betreff "Attached Image FXXXXXXXX") wird das PDF in der Cloud und im Lodur gespeichert. +* Nach jedem Durchgang wird ein Heartbeat an den konfigurierten Healthcheck Service gesendet, z.B. https://healthchecks.io/ Desweiteren wird über Pushover eine Nachricht mit möglichst vielen Informationen publiziert. @@ -72,6 +73,38 @@ an Lodur gesendet werden, in einem JSON File im WebDAV neben den PDFs abgelegt. So lässt sich im Nachhinein ein Datensatz bearbeiten und eine Zuordnung des Einsatzes im WebDAV und in Lodur herstellen. +## Detailierter Ablauf + +### Einsatzausdruck_FW + +1. PDF extrahieren und in Cloud hochladen +2. Falls PDF noch nicht geparst wurde wird davon ausgegangen, dass dies die initiale Meldung ist: + 1. PDF parsen + 2. Push Nachricht mit Infos aus PDF senden + 3. Geparste Daten als JSON in Cloud speichern +3. Falls Einsatz im Lodur noch nicht ausgelesen: + 1. Einsatz Datensatz ID im Lodur suchen + 2. Ganzer Datensatz auslesen + 3. Datensatz als JSON in Cloud speichern + 4. PDF in Lodur speichern + 5. E-Mail als gelesen markieren - wird somit nicht nochmals bearbeitet +### Einsatzprotokoll + +1. Lodur Datensatz ID aus Cloud laden (JSON Datei) +2. Ganzer Datensatz aus Lodur auslesen und als JSON in Cloud speichern +3. Falls Datensatz zur Bearbeitung freigegeben ist (`aut_created_report == finished`) + 1. PDF in Lodur speichern + 2. Einsatzprotokoll Daten ergänzen und in Lodur speichern + 3. Push Nachricht senden (Einsatz beendet) + 4. E-Mail als gelesen markieren - wird somit nicht nochmals bearbeitet + +### Einsatzrapport + +1. Prüfen, ob F-Nummer aus Scan E-Mail Betreff gefunden +2. Lodur Datensatz ID aus Cloud laden (JSON Datei) +3. Ganzer Datensatz aus Lodur auslesen und als JSON in Cloud speichern +4. PDF in Lodur speichern und Datensatz ergänzen +5. Push Nachricht senden (Rapport bearbeitet) ## Installation and Configuration The application is written in Python and runs perfectly on Kubernetes.