From 1e6fc92e4168685b81d9c332c70c2b49ea3ae9a3 Mon Sep 17 00:00:00 2001 From: gurkenhabicht Date: Mon, 24 Jul 2023 22:57:31 +0200 Subject: [PATCH] make the database work again --- .gitignore | 5 + Cargo.lock | 691 +++++++++++++++++++++++---------------------- Cargo.toml | 8 +- src/flamegraph.svg | 4 +- src/main.rs | 50 +++- src/main_bkp | 211 -------------- src/parser.json | 2 +- 7 files changed, 405 insertions(+), 566 deletions(-) delete mode 100644 src/main_bkp diff --git a/.gitignore b/.gitignore index ea8c4bf..51ba4ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ /target +*.pcap +*.pcapng +*.log +*.svg + diff --git a/Cargo.lock b/Cargo.lock index d25064e..a9330da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,22 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "0.7.10" @@ -23,20 +40,35 @@ checksum = "da71fef07bc806586090247e971229289f64c210a278ee5ae419314eb386b31d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.20", ] [[package]] name = "autocfg" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] [[package]] name = "base64" -version = "0.12.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d1ccbaf7d9ec9537465a97bf19edc1a4e158ecb49fc16178202238c569cc42" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bitfield" @@ -46,37 +78,19 @@ checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719" [[package]] name = "bitflags" -version = "1.2.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "block-buffer" -version = "0.7.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "block-padding", - "byte-tools", - "byteorder", "generic-array", ] -[[package]] -name = "block-padding" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -dependencies = [ - "byte-tools", -] - -[[package]] -name = "byte-tools" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" - [[package]] name = "byteorder" version = "1.3.4" @@ -85,9 +99,15 @@ checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" [[package]] name = "bytes" -version = "0.5.4" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -96,12 +116,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] -name = "cloudabi" -version = "0.0.3" +name = "cfg-if" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cpufeatures" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ - "bitflags", + "libc", ] [[package]] @@ -122,7 +148,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", "crossbeam-utils", "lazy_static", "maybe-uninit", @@ -136,7 +162,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "crossbeam-utils", ] @@ -147,27 +173,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", "lazy_static", ] [[package]] -name = "crypto-mac" -version = "0.7.0" +name = "crypto-common" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "subtle", + "typenum", ] [[package]] name = "digest" -version = "0.8.1" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "generic-array", + "block-buffer", + "crypto-common", + "subtle", ] [[package]] @@ -183,14 +211,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c4cf866e4d3e5e773691f5f61615a224a7b0b72b7daf994fc56d1b82dab0b6b" dependencies = [ "rustc-serialize", + "serde", ] -[[package]] -name = "fake-simd" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -198,32 +221,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] -name = "fnv" -version = "1.0.7" +name = "fs_extra" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +checksum = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" [[package]] name = "futures" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -236,9 +243,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -246,15 +253,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -263,42 +270,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-macro" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ - "proc-macro-hack", "proc-macro2", "quote", - "syn", + "syn 2.0.27", ] [[package]] name = "futures-sink" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" -dependencies = [ - "once_cell", -] +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.5" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", @@ -307,33 +310,38 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project", + "pin-project-lite", "pin-utils", - "proc-macro-hack", - "proc-macro-nested", "slab", ] [[package]] name = "generic-array" -version = "0.12.3" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", + "version_check", ] [[package]] name = "getrandom" -version = "0.1.14" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "hermit-abi" version = "0.1.12" @@ -345,23 +353,13 @@ dependencies = [ [[package]] name = "hmac" -version = "0.7.1" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "crypto-mac", "digest", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "itoa" version = "0.4.5" @@ -369,13 +367,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" [[package]] -name = "kernel32-sys" -version = "0.2.2" +name = "jemalloc-sys" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +checksum = "0d3b9f3f5c9b31aa0f5ed3260385ac205db665baa41d49bb8338008ae94ede45" dependencies = [ - "winapi 0.2.8", - "winapi-build", + "cc", + "fs_extra", + "libc", +] + +[[package]] +name = "jemallocator" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43ae63fcfc45e99ab3d1b29a46782ad679e98436c3169d15a167a1108a724b69" +dependencies = [ + "jemalloc-sys", + "libc", ] [[package]] @@ -386,16 +395,17 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.69" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "lock_api" -version = "0.3.4" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ + "autocfg", "scopeguard", ] @@ -405,7 +415,7 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", ] [[package]] @@ -421,16 +431,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] -name = "md5" -version = "0.7.0" +name = "md-5" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest", +] [[package]] name = "memchr" -version = "2.3.3" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" @@ -441,79 +454,24 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "mio" -version = "0.6.22" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ - "cfg-if", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", "libc", - "log", - "miow 0.2.1", - "net2", - "slab", - "winapi 0.2.8", -] - -[[package]] -name = "mio-named-pipes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" -dependencies = [ - "log", - "mio", - "miow 0.3.4", - "winapi 0.3.8", -] - -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio", -] - -[[package]] -name = "miow" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - -[[package]] -name = "miow" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22dfdd1d51b2639a5abd17ed07005c3af05fb7a2a3b1a1d0d7af1000a520c1c7" -dependencies = [ - "socket2", - "winapi 0.3.8", -] - -[[package]] -name = "net2" -version = "0.2.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" -dependencies = [ - "cfg-if", - "libc", - "winapi 0.3.8", + "wasi", + "windows-sys", ] [[package]] @@ -527,22 +485,25 @@ dependencies = [ ] [[package]] -name = "once_cell" -version = "1.3.1" +name = "object" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] [[package]] -name = "opaque-debug" -version = "0.2.3" +name = "once_cell" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "parking_lot" -version = "0.10.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", "parking_lot_core", @@ -550,16 +511,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.7.2" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ - "cfg-if", - "cloudabi", + "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", - "winapi 0.3.8", + "windows-targets", ] [[package]] @@ -579,47 +539,27 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" [[package]] name = "phf" -version = "0.8.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_shared", ] [[package]] name = "phf_shared" -version = "0.8.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d480cb4e89522ccda96d0eed9af94180b7a5f93fb28f66e1fd7d68431663d1" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82996f11efccb19b685b14b5df818de31c1edcee3daa256ab5775dd98e72feb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" -version = "0.1.5" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7505eeebd78492e0f6108f7171c4948dbb120ee8119d9d77d0afa5469bef67f" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" [[package]] name = "pin-utils" @@ -629,16 +569,16 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "postgres-protocol" -version = "0.5.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f611afe4d1407ebe7f3ced1ffc66f730fac1b1c13085e230a8cdcb921e97710" +checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" dependencies = [ "base64", "byteorder", "bytes", "fallible-iterator", "hmac", - "md5", + "md-5", "memchr", "rand", "sha2", @@ -647,9 +587,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.1.1" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e634590e8812c500088d88db721195979223dabb05149f43cb50931d0ff5865d" +checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6" dependencies = [ "bytes", "eui48", @@ -661,58 +601,44 @@ dependencies = [ [[package]] name = "ppv-lite86" -version = "0.2.6" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" - -[[package]] -name = "proc-macro-hack" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" - -[[package]] -name = "proc-macro-nested" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.12" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8872cf6f48eee44265156c111456a700ab3483686b3f96df4cf5481c89157319" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] name = "quote" -version = "1.0.4" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c1f4b0efa5fc5e8ceb705136bfee52cfdb6a4e3509f770b478cd6ed434232a7" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] [[package]] name = "rand" -version = "0.7.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "getrandom", "libc", "rand_chacha", "rand_core", - "rand_hc", ] [[package]] name = "rand_chacha" -version = "0.2.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", "rand_core", @@ -720,22 +646,13 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.5.1" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ "getrandom", ] -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core", -] - [[package]] name = "rayon" version = "1.3.0" @@ -762,9 +679,12 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.56" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] [[package]] name = "regex" @@ -784,6 +704,12 @@ version = "0.6.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc-serialize" version = "0.3.24" @@ -819,7 +745,7 @@ checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.20", ] [[package]] @@ -835,14 +761,13 @@ dependencies = [ [[package]] name = "sha2" -version = "0.8.1" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ - "block-buffer", + "cfg-if 1.0.0", + "cpufeatures", "digest", - "fake-simd", - "opaque-debug", ] [[package]] @@ -869,20 +794,28 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.4.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" -version = "0.3.12" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ - "cfg-if", "libc", - "redox_syscall", - "winapi 0.3.8", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", ] [[package]] @@ -897,9 +830,9 @@ dependencies = [ [[package]] name = "subtle" -version = "1.0.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -912,6 +845,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "syn" +version = "2.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "thread_local" version = "1.0.1" @@ -923,50 +867,47 @@ dependencies = [ [[package]] name = "tokio" -version = "0.2.20" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c1d570eb1a36f0345a5ce9c6c6e665b70b73d11236912c0b477616aeec47b1" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ + "autocfg", + "backtrace", "bytes", - "fnv", - "futures-core", - "iovec", - "lazy_static", "libc", - "memchr", "mio", - "mio-named-pipes", - "mio-uds", "num_cpus", + "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", + "socket2 0.4.9", "tokio-macros", - "winapi 0.3.8", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "0.2.5" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.27", ] [[package]] name = "tokio-postgres" -version = "0.5.4" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56010a704311361b7c9e870aaa4ddffaf9f2db89cbcf3e14773ac8a14469c9c" +checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" dependencies = [ "async-trait", "byteorder", "bytes", "fallible-iterator", - "futures", + "futures-channel", + "futures-util", "log", "parking_lot", "percent-encoding", @@ -974,22 +915,23 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", + "socket2 0.5.3", "tokio", "tokio-util", ] [[package]] name = "tokio-util" -version = "0.3.1" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", "futures-sink", - "log", "pin-project-lite", "tokio", + "tracing", ] [[package]] @@ -999,7 +941,8 @@ dependencies = [ "bitfield", "byteorder", "eui48", - "libc", + "futures", + "jemallocator", "pcap", "rayon", "regex", @@ -1010,10 +953,30 @@ dependencies = [ ] [[package]] -name = "typenum" -version = "1.12.0" +name = "tracing" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "typenum" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" @@ -1024,6 +987,12 @@ dependencies = [ "matches", ] +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + [[package]] name = "unicode-normalization" version = "0.1.12" @@ -1039,34 +1008,28 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" +version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "winapi" -version = "0.2.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - -[[package]] -name = "winapi" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" dependencies = [ "winapi-i686-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -1080,11 +1043,67 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "ws2_32-sys" -version = "0.2.1" +name = "windows-sys" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "winapi 0.2.8", - "winapi-build", + "windows-targets", ] + +[[package]] +name = "windows-targets" +version = "0.48.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/Cargo.toml b/Cargo.toml index 367d532..d79ad34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,10 @@ edition = "2018" [dependencies] -tokio-postgres = { version="0.5.4", features = ["runtime","with-eui48-0_4","with-serde_json-1"] } -tokio = { version = "0.2", features = ["full"] } +# tokio-postgres = { version="0.5.4", features = ["runtime","with-eui48-0_4","with-serde_json-1"] } +tokio-postgres = { version="0.7", features = ["runtime","with-eui48-0_4","with-serde_json-1"] } +#tokio = { version = "0.2", features = ["full"] } +tokio = { version = "1.0", features = ["full"] } pcap = "~0.7.0" byteorder = "*" bitfield = "*" @@ -24,5 +26,5 @@ serde_json = { version = "1.0", features = ["raw_value"] } serde = { version = "1.0.3", features = ["derive"] } rayon = "1.3" regex = "1.3.7" -futures = "~0.3.5" +futures = "~0.3.28" jemallocator = "~0.3.2" diff --git a/src/flamegraph.svg b/src/flamegraph.svg index 1a2ade7..92d33d4 100644 --- a/src/flamegraph.svg +++ b/src/flamegraph.svg @@ -1,4 +1,4 @@ - \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index df63d28..dca1436 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,11 +1,12 @@ mod configure; mod parser; mod serializer; +use std::error::Error; use futures::pin_mut; use tokio::task; use tokio_postgres::binary_copy::BinaryCopyInWriter; use tokio_postgres::types::{ToSql, Type}; -use tokio_postgres::{Error, NoTls}; +use tokio_postgres::{NoTls, Client}; extern crate jemallocator; #[global_allocator] @@ -30,8 +31,25 @@ fn query_string(insert_max: &usize, table_name: &str) -> String { insert_template } -#[tokio::main(core_threads = 4)] -async fn main() -> Result<(), Error> { + +async fn connect_to_postgres(config: &std::string::String ) -> Result> { + // let config = "postgres://postgres:password@172.17.0.2:5432/postgres"; + let (client, _connection) = tokio_postgres::connect(&config, NoTls).await?; + tokio::spawn(async move { + if let Err(e) = _connection.await { + eprintln!("connection error: {}", e); + } + }); + + // Do any other necessary processing with the client or connection... + + Ok(client) +} + +// #[tokio::main(core_threads = 4)] +// #[tokio::main(worker_threads = 4)] +#[tokio::main] +async fn main() -> Result<(), Box> { /* Init values from file */ let config: configure::Config = configure::from_json_file().unwrap(); let pcap_map = configure::map_pcap_dir(&config.pcap_dir).unwrap(); @@ -41,20 +59,24 @@ async fn main() -> Result<(), Error> { // This db table should include UUIDs as primary keys, so it can be joined effectively with past and future runs. // TODO: Use inotify crate to update pcap_map according to files created while parser is running + // println!("{:?}", pcap_map); + /* db connection */ - let (client, connection) = tokio_postgres::connect(&config.connection, NoTls).await?; - tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("connection error: {}", e); - } - }); + + if let Err(err) = connect_to_postgres(&config.connection).await { + println!("Error chain: {:?}", err); + println!("Error: {}", err); + eprintln!("Error chain: {:?}", err); + } + + let client = connect_to_postgres(&config.connection).await.unwrap(); client .execute(&*format!("DROP TABLE IF EXISTS {}", &config.tablename), &[]) .await?; client .execute( &*format!( - "CREATE TABLE {} ( ID serial NOT NULL PRIMARY KEY, packet json NOT NULL)", + "CREATE TABLE {} ( ID serial NOT NULL PRIMARY KEY, packet json NOT NULL, packet_time double precision NOT NULL)", &config.tablename ), &[], @@ -78,15 +100,17 @@ async fn main() -> Result<(), Error> { /* Copy data to db */ let sink = client - .copy_in("COPY json_dump(packet) from STDIN BINARY") + .copy_in("COPY json_dump(packet, packet_time) from STDIN BINARY") .await .unwrap(); - let writer = BinaryCopyInWriter::new(sink, &[Type::JSON]); + let writer = BinaryCopyInWriter::new(sink, &[Type::JSON, Type::FLOAT8]); let join = task::spawn(async move { pin_mut!(writer); for pack in data_serialized { - writer.as_mut().write(&[&pack]).await.unwrap(); + //println!("{}", pack); + //let time : Result= serde_json::from_value(pack["time"]); + writer.as_mut().write(&[&pack, &pack["time"].as_f64().unwrap()]).await.unwrap(); } writer.finish().await.unwrap(); }); diff --git a/src/main_bkp b/src/main_bkp deleted file mode 100644 index ae1065f..0000000 --- a/src/main_bkp +++ /dev/null @@ -1,211 +0,0 @@ -extern crate serde_json; -extern crate tokio; -extern crate tokio_postgres; - -mod configure; -mod parser; -mod serializer; -//use postgres::{Client, NoTls}; -//use postgres::types::ToSql; -//use postgres::binary_copy::{BinaryCopyInWriter}; -use tokio_postgres::types::{Type, ToSql}; -use tokio_postgres::{Error, NoTls}; -use tokio_postgres::binary_copy::{BinaryCopyInWriter}; -use futures::{pin_mut}; -use tokio::task; -use tokio::sync::mpsc; -//use std::thread::{spawn, JoinHandle}; -//use std::sync::mpsc::{channel, Receiver}; -//use std::sync::mpsc; -//use std::alloc::System; -// -//#[global_allocator] -//static A: System = System; - -/* conditionals */ -const FROM_FILE: bool = false; -const FROM_DEVICE: bool = true; -//const NON_CHUNKED: bool = true; -//const CHUNKED: bool = false; - -fn query_string(insert_max: &usize, table_name: &str) -> String { - let mut insert_template = String::with_capacity(insert_max * 8 + 96); - insert_template.push_str(&*format!("INSERT INTO {} (packet) Values ", table_name)); - - for insert in 0..insert_max - 1 { - insert_template.push_str(&*format!("(${}), ", insert + 1)); - } - insert_template.push_str(&*format!("(${})", insert_max)); - - insert_template -} - -#[tokio::main(core_threads = 4)] -async fn main() -> Result<(), Error> { - /* Init values from file */ - let config: configure::Config = configure::from_json_file().unwrap(); - let pcap_map = configure::map_pcap_dir(&config.pcap_dir).unwrap(); - - // TODO: Create db table with pcap file hashes - // TODO: hash file metadata, so its state is comparable with future file updates and can be written to a db table (and read e.g. after system crash) - // This db table should include UUIDs as primary keys, so it can be joined effectively with past and future runs. - // TODO: Use inotify crate to update pcap_map according to files created while parser is running - - /* db connection */ - let (client, connection) = tokio_postgres::connect(&config.connection, NoTls).await?; - tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("connection error: {}", e); - } - }); - client - .execute(&*format!("DROP TABLE IF EXISTS {}", &config.tablename), &[]) - .await?; - client - .execute( - &*format!( - "CREATE TABLE {} ( ID serial NOT NULL PRIMARY KEY, packet json NOT NULL)", - &config.tablename - ), - &[], - ) - .await?; - - /* device or file input */ - match config.is_device { - FROM_FILE => { - for (_pcap_file, _pcap_info) in pcap_map.iter() { - //println!("{:?}: {:?}", &_pcap_file, &_pcap_info); - - - - /* MPSC channeled serialization */ - // let (qry_data, h1) = parser::mpsc_parser(_pcap_file.to_owned(), config.filter.to_owned(), config.regex_filter.to_owned()); - // let (data_serialized, h2) = serializer::mpsc_serialize(qry_data); - // let packets_serialized = serializer::mpsc_collect_serialized(data_serialized); - // let _r1 = h1.join().unwrap(); - // let _r2 = h2.join().unwrap(); - - /* This is serializing data without mpsc, which results in higher memory consumption, it's faster but 12GB main memory is needed */ -// let v: Vec = -// parser::parse(&_pcap_file, &config.filter, &config.regex_filter); -// let len = v.len(); - - /* tokio mpsc channel */ - let (tx, mut rx) = mpsc::channel(1000); - let pcap_file = _pcap_file.clone(); - let filter = config.filter.clone(); - let regex_filter = config.regex_filter.clone(); - let join_handle: task::JoinHandle> = task::spawn( async move { - parser::tokio_parse(pcap_file, &filter, ®ex_filter).await - }); - let v = join_handle.await.unwrap(); - for packet in v.into_iter(){ - let mut tx = tx.clone(); - - tokio::spawn( async move { - //println!("serializing!number {:?}", i); - - let packet_serialized = serializer::tokio_serialize(packet).await; - tx.send(packet_serialized).await.unwrap(); - }); - - } - drop(tx); - // let mut packets_serialized: Vec = Vec::new(); - // while let Some(res) = rx.recv().await { - // //println!("collecting"); - // packets_serialized.push(res); - // //let packets_serialized = serializer::serialize_packets(v); - // } - - let sink = client.copy_in("COPY json_dump(packet) from STDIN BINARY").await.unwrap(); - let writer = BinaryCopyInWriter::new(sink, &[Type::JSON]); - - let join = task::spawn( async move { - pin_mut!(writer); - //for pack in packets_serialized { - while let Some(res) = rx.recv().await { - writer.as_mut().write(&[&res]).await.unwrap(); - drop(res); - // Reminder: write_raw() behavior is very strange, so it's write() for now. - // writer.as_mut().write_raw(chunk.into_iter().map(|p| p as &dyn ToSql).collect()).await.unwrap(); - } - - //thread::sleep(time::Duration::from_millis(3000)); - writer.finish().await.unwrap(); - }); - assert!(join.await.is_ok()); - - // TODO: MPSC channel - // let mut v = Vec::::with_capacity(100000); - // v.extend(parser::parse(&_pcap_file, &config.filter, &config.regex_filter)); - // let mut packets_serialized = Vec::::with_capacity(100000); - // packets_serialized.extend(serializer::serialize_packets(v)); -// Reminder: If COPY doesn't cut it and INSERT is the way to go, uncomment and use following logic inside FROM_FILE -// /* Do chunks and query data */ -// let chunker = (&packets_serialized.len() < &config.insert_max) && (0 < packets_serialized.len()) ; -// match chunker { -// NON_CHUNKED => { -// let insert_str = query_string(&packets_serialized.len(), &config.tablename); -// let statement = client.prepare(&insert_str).await?; -// client -// .query_raw( -// &statement, -// packets_serialized.iter().map(|p| p as &dyn ToSql), -// ) -// .await?; -// } -// CHUNKED => { -// let insert_str = query_string(&config.insert_max, &config.tablename); -// let statement = client.prepare(&insert_str).await?; -// -// for chunk in packets_serialized.chunks_exact(config.insert_max) { -// client -// .query_raw(&statement, chunk.iter().map(|p| p as &dyn ToSql)) -// .await?; -// } -// let remainder_len = packets_serialized -// .chunks_exact(config.insert_max) -// .remainder() -// .len(); -// if 0 < remainder_len { -// let rem_str = query_string(&remainder_len, &config.tablename); -// let statement = client.prepare(&rem_str).await?; -// client -// .query_raw( -// &statement, -// packets_serialized -// .chunks_exact(config.insert_max) -// .remainder() -// .iter() -// .map(|p| p as &dyn ToSql), -// ) -// .await?; -// } -// } -// } - } - } - FROM_DEVICE => { - let insert_str = query_string(&config.insert_max, &config.tablename); - let statement = client.prepare(&insert_str).await?; - loop { - let v: Vec = parser::parse_device( - &config.device, - &config.filter, - &config.insert_max, - &config.regex_filter, - ); - let packets_serialized = serializer::serialize_packets(v); - client - .query_raw( - &statement, - packets_serialized.iter().map(|p| p as &dyn ToSql), - ) - .await?; - } - } - } - Ok(()) -} diff --git a/src/parser.json b/src/parser.json index bfe340b..ef407ab 100644 --- a/src/parser.json +++ b/src/parser.json @@ -8,6 +8,6 @@ "pcap_dir": "../target/files", "database_tablename": "json_dump", "database_user": "postgres", - "database_host": "localhost", + "database_host": "172.17.0.2", "database_password": "password" }