diff --git a/client/package.json b/client/package.json index 6411345..98a1104 100644 --- a/client/package.json +++ b/client/package.json @@ -9,7 +9,7 @@ "lint": "vue-cli-service lint" }, "dependencies": { - "@chartwerk/line-pod": "0.4.4", + "@chartwerk/line-pod": "^0.4.4", "@chartwerk/scatter-pod": "^0.2.4", "@kyvg/vue3-notification": "^2.3.4", "@types/lodash": "^4.14.176", diff --git a/client/yarn.lock b/client/yarn.lock index 9d138fd..c601824 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -232,7 +232,7 @@ version "0.1.0" resolved "https://codeload.github.com/chartwerk/core/tar.gz/532eddbc8ad938091b1d9ec1693cec5eddfdbfc2" -"@chartwerk/line-pod@0.4.4": +"@chartwerk/line-pod@^0.4.4": version "0.4.4" resolved "https://registry.yarnpkg.com/@chartwerk/line-pod/-/line-pod-0.4.4.tgz#724152e886c529df97019ba3dfed335ef42b2f57" integrity sha512-L7RmTzy+TVMueV91E3upVCK/V7t9ZYLwC0tIYhFJxSJjrLOhRMAOLxZdkp7CqxXFj+u+FuFDhY0Dch2nYDvLpg== diff --git a/server/Cargo.lock b/server/Cargo.lock index 42cef73..2fcdfa4 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -37,15 +37,6 @@ version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1" -[[package]] -name = "approx" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2a05fd1bd10b2527e20a2cd32d8873d115b8b39fe219ee25f42a8aca6ba278" -dependencies = [ - "num-traits 0.2.14", -] - [[package]] name = "arrayvec" version = "0.5.2" @@ -74,6 +65,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + [[package]] name = "autocfg" version = "1.0.1" @@ -150,6 +147,12 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -184,6 +187,15 @@ dependencies = [ "vec_map", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + [[package]] name = "config" version = "0.11.0" @@ -256,12 +268,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "either" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -314,6 +320,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.17" @@ -368,7 +380,7 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" dependencies = [ - "autocfg", + "autocfg 1.0.1", "proc-macro-hack", "proc-macro2", "quote", @@ -393,7 +405,7 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" dependencies = [ - "autocfg", + "autocfg 1.0.1", "futures-channel", "futures-core", "futures-io", @@ -408,6 +420,20 @@ dependencies = [ "slab", ] +[[package]] +name = "gbdt" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74248386ea349f903cee13fae53f41bdae987f74eb798c6cbcb08370c99ccd34" +dependencies = [ + "cfg-if 0.1.10", + "rand 0.6.5", + "regex", + "serde 1.0.130", + "serde_derive", + "serde_json", +] + [[package]] name = "generic-array" version = "0.14.4" @@ -424,7 +450,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -435,7 +461,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi 0.10.0+wasi-snapshot-preview1", ] @@ -488,13 +514,13 @@ dependencies = [ "config", "fastrand", "futures", + "gbdt", "libsqlite3-sys", - "linfa", - "linfa-svm", "ndarray", "openssl", "parking_lot", "rusqlite", + "rustfft", "serde 1.0.130", "serde_json", "subbeat", @@ -624,7 +650,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ - "autocfg", + "autocfg 1.0.1", "hashbrown", ] @@ -643,16 +669,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", -] - -[[package]] -name = "itertools" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf" -dependencies = [ - "either", + "cfg-if 1.0.0", ] [[package]] @@ -661,15 +678,6 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" -[[package]] -name = "kdtree" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ee359328fc9087e9e3fc0a4567c4dd27ec69a127d6a70e8d9dd22845b8b1a2" -dependencies = [ - "num-traits 0.2.14", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -684,7 +692,7 @@ checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec", "bitflags", - "cfg-if", + "cfg-if 1.0.0", "ryu", "static_assertions", ] @@ -695,12 +703,6 @@ version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "869d572136620d55835903746bcb5cdc54cb2851fd0aeec53220b4bb65ef3013" -[[package]] -name = "libm" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a" - [[package]] name = "libsqlite3-sys" version = "0.23.1" @@ -712,64 +714,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linfa" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7496612871c85de124d12006109360ed7ffc4740ce3fe3fe35a749e2d7f2f88f" -dependencies = [ - "approx", - "ndarray", - "num-traits 0.2.14", - "rand 0.8.4", - "thiserror", -] - -[[package]] -name = "linfa-kernel" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6da781111566c8b9867c8e9c0ac69fd72808dda1e111164aa23ece9f15d6a95a" -dependencies = [ - "linfa", - "linfa-nn", - "ndarray", - "num-traits 0.2.14", - "serde 1.0.130", - "sprs", -] - -[[package]] -name = "linfa-nn" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83748e3f71c8fa7fc490a54d2e33d40b8246a335618159fb313989ff6eb209ef" -dependencies = [ - "kdtree", - "linfa", - "ndarray", - "ndarray-stats", - "noisy_float", - "num-traits 0.2.14", - "order-stat", - "thiserror", -] - -[[package]] -name = "linfa-svm" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecfffa7a24c3019dc4c4fa9c58bf8f187ff2fdbbd08b6fee454591c6ec285f3f" -dependencies = [ - "linfa", - "linfa-kernel", - "ndarray", - "ndarray-rand", - "num-traits 0.2.14", - "serde 1.0.130", - "thiserror", -] - [[package]] name = "linked-hash-map" version = "0.5.4" @@ -791,7 +735,7 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -895,48 +839,11 @@ version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08e854964160a323e65baa19a0b1a027f76d590faba01f05c0cbc3187221a8c9" dependencies = [ - "approx", "matrixmultiply", "num-complex", "num-integer", "num-traits 0.2.14", "rawpointer", - "serde 1.0.130", -] - -[[package]] -name = "ndarray-rand" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65608f937acc725f5b164dcf40f4f0bc5d67dc268ab8a649d3002606718c4588" -dependencies = [ - "ndarray", - "rand 0.8.4", - "rand_distr", -] - -[[package]] -name = "ndarray-stats" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22877ad014bafa2f7dcfa5d556b0c7a52b0546cc98061a1ebef6d1834957b069" -dependencies = [ - "indexmap", - "itertools", - "ndarray", - "noisy_float", - "num-integer", - "num-traits 0.2.14", - "rand 0.8.4", -] - -[[package]] -name = "noisy_float" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978fe6e6ebc0bf53de533cd456ca2d9de13de13856eda1518a285d7705a213af" -dependencies = [ - "num-traits 0.2.14", ] [[package]] @@ -974,7 +881,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg", + "autocfg 1.0.1", "num-traits 0.2.14", ] @@ -993,8 +900,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg", - "libm", + "autocfg 1.0.1", ] [[package]] @@ -1026,7 +932,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" dependencies = [ "bitflags", - "cfg-if", + "cfg-if 1.0.0", "foreign-types", "libc", "once_cell", @@ -1054,7 +960,7 @@ version = "0.9.67" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058" dependencies = [ - "autocfg", + "autocfg 1.0.1", "cc", "libc", "openssl-src", @@ -1062,12 +968,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "order-stat" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa535d5117d3661134dbf1719b6f0ffe06f2375843b13935db186cd094105eb" - [[package]] name = "parking_lot" version = "0.11.2" @@ -1085,7 +985,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", @@ -1143,6 +1043,15 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741" +[[package]] +name = "primal-check" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01419cee72c1a1ca944554e23d83e483e1bccf378753344e881de28b5487511d" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -1179,6 +1088,25 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + [[package]] name = "rand" version = "0.7.3" @@ -1204,6 +1132,16 @@ dependencies = [ "rand_hc 0.3.1", ] +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + [[package]] name = "rand_chacha" version = "0.2.2" @@ -1224,6 +1162,21 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.5.1" @@ -1243,13 +1196,12 @@ dependencies = [ ] [[package]] -name = "rand_distr" -version = "0.4.2" +name = "rand_hc" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "964d548f8e7d12e102ef183a0de7e98180c9f8729f555897a857b96e48122d2f" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" dependencies = [ - "num-traits 0.2.14", - "rand 0.8.4", + "rand_core 0.3.1", ] [[package]] @@ -1270,12 +1222,74 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "rawpointer" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -1338,6 +1352,20 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" +[[package]] +name = "rustfft" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d089e5c57521629a59f5f39bca7434849ff89bd6873b521afe389c1c602543" +dependencies = [ + "num-complex", + "num-integer", + "num-traits 0.2.14", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "ryu" version = "1.0.5" @@ -1474,7 +1502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" dependencies = [ "block-buffer", - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", "opaque-debug", @@ -1511,25 +1539,18 @@ dependencies = [ "winapi", ] -[[package]] -name = "sprs" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea71e48b3eab4c4b153e8e35dcaeac132720809ef68359097b8cb54a18edd70" -dependencies = [ - "ndarray", - "num-complex", - "num-traits 0.2.14", - "serde 1.0.130", - "smallvec", -] - [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strength_reduce" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ff2f71c82567c565ba4b3009a9350a96a7269eaa4001ebedae926230bc2254" + [[package]] name = "strsim" version = "0.8.0" @@ -1574,7 +1595,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "rand 0.8.4", "redox_syscall", @@ -1643,7 +1664,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc" dependencies = [ - "autocfg", + "autocfg 1.0.1", "bytes", "libc", "memchr", @@ -1737,7 +1758,7 @@ version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-core", @@ -1752,6 +1773,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "transpose" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95f9c900aa98b6ea43aee227fd680550cdec726526aab8ac801549eadb25e39f" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "try-lock" version = "0.2.3" diff --git a/server/Cargo.toml b/server/Cargo.toml index 525ae5b..26c037f 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -23,11 +23,11 @@ libsqlite3-sys = { version = "*", features = ["bundled"] } futures = "0.3.17" chrono = "0.4.19" -linfa = "0.5.0" -linfa-svm = { version="0.5.0", features=["serde"] } +gbdt = "0.1.1" ndarray = "0.15.3" bincode = "1.3.3" async-trait = "0.1.51" +rustfft = "6.0.1" # TODO: remove this from prod # plotlib = "0.5.1" diff --git a/server/src/services/analytic_service/analytic_unit/pattern_analytic_unit.rs b/server/src/services/analytic_service/analytic_unit/pattern_analytic_unit.rs index 4e9aa17..1a910e8 100644 --- a/server/src/services/analytic_service/analytic_unit/pattern_analytic_unit.rs +++ b/server/src/services/analytic_service/analytic_unit/pattern_analytic_unit.rs @@ -3,10 +3,12 @@ use std::{collections::VecDeque, fmt, sync::Arc}; use futures::future; use parking_lot::Mutex; -use linfa::prelude::*; -use linfa; -use linfa_svm::Svm; +use gbdt::config::Config; +use gbdt::decision_tree::{Data, DataVec, PredVec}; +use gbdt::gradient_boost::GBDT; + + use ndarray::Array; @@ -16,13 +18,20 @@ use super::types::{AnalyticUnit, AnalyticUnitConfig, LearningResult, PatternConf use async_trait::async_trait; +use rustfft::{self, FftPlanner, num_complex::Complex}; + // TODO: move to config const DETECTION_STEP: u64 = 10; +const FFT_LEN: usize = 64; + +// TODO: convert to vector +pub const FEATURES_SIZE: usize = 4 + 16 * 2; +pub type Features = Vec; #[derive(Clone)] pub struct LearningResults { // TODO: replace with RWLock - model: Arc>>, + model: Arc>, pub learning_train: LearningTrain, @@ -55,9 +64,7 @@ impl fmt::Debug for LearningResults { } } -pub const FEATURES_SIZE: usize = 4; -pub type Features = [f64; FEATURES_SIZE]; fn nan_to_zero(n: f64) -> f64 { if n.is_nan() { @@ -111,12 +118,44 @@ fn get_features(xs: &Vec) -> Features { let sd = sum.sqrt(); - // TODO: add autocorrelation - // TODO: add FFT // TODO: add DWT - return [ + let mut planner = FftPlanner::::new(); + + // TODO: move 128 to config + let fft = planner.plan_fft_forward(FFT_LEN); + let mut c_buffer = vec![Complex{ re: 0.0f64, im: 0.0f64 }; FFT_LEN]; + + for i in 0..FFT_LEN.min(xs.len()) { + c_buffer[i].re = xs[i]; + } + + fft.process(&mut c_buffer); + + // https://docs.rs/rustfft/6.0.1/rustfft/index.html#normalization + let norm_factor = (FFT_LEN.min(xs.len()) as f64).sqrt(); + for i in 0..FFT_LEN.min(xs.len()) { + c_buffer[i] /= norm_factor; + } + + return vec![ min, max, mean, sd, + c_buffer[0].re, c_buffer[0].im, + c_buffer[1].re, c_buffer[1].im, + c_buffer[2].re, c_buffer[2].im, + c_buffer[3].re, c_buffer[3].im, + c_buffer[4].re, c_buffer[4].im, + c_buffer[5].re, c_buffer[5].im, + c_buffer[6].re, c_buffer[6].im, + c_buffer[7].re, c_buffer[7].im, + c_buffer[8].re, c_buffer[8].im, + c_buffer[9].re, c_buffer[9].im, + c_buffer[10].re, c_buffer[10].im, + c_buffer[11].re, c_buffer[11].im, + c_buffer[12].re, c_buffer[12].im, + c_buffer[13].re, c_buffer[13].im, + c_buffer[14].re, c_buffer[14].im, + c_buffer[15].re, c_buffer[15].im, // 0f64,0f64, // 0f64,0f64,0f64, 0f64 ]; @@ -202,6 +241,19 @@ impl AnalyticUnit for PatternAnalyticUnit { } async fn learn(&mut self, ms: MetricService, ss: SegmentsService) -> LearningResult { + + // TODO: move to config + let mut cfg = Config::new(); + cfg.set_feature_size(FEATURES_SIZE); + cfg.set_max_depth(3); + cfg.set_iterations(50); + cfg.set_shrinkage(0.1); + cfg.set_loss("LogLikelyhood"); + cfg.set_debug(false); + cfg.set_data_sample_ratio(1.0); + cfg.set_feature_sample_ratio(1.0); + cfg.set_training_optimization_level(2); + // be careful if decide to store detections in db let segments = ss.get_segments_inside(0, u64::MAX / 2).unwrap(); let has_segments_label = segments @@ -263,19 +315,22 @@ impl AnalyticUnit for PatternAnalyticUnit { anti_patterns.push(xs); } - let records = Array::from_shape_fn((records_raw.len(), FEATURES_SIZE), |(i, j)| { - records_raw[i][j] - }); - - let targets = Array::from_vec(targets_raw.clone()); + let mut train_dv = Vec::new(); + assert_eq!(records_raw.len(), targets_raw.len()); + + for i in 0..train_dv.len() { + let data = Data::new_training_data( + records_raw[i].iter().map(|e| *e as f32).collect(), + 1.0, + if targets_raw[i] { 1.0 } else { -1.0 }, + Some(0.5) + ); + train_dv.push(data); + } - let train = linfa::Dataset::new(records, targets); - let model = Svm::<_, bool>::params() - .pos_neg_weights(50000., 5000.) - .gaussian_kernel(80.0) - .fit(&train) - .unwrap(); + let mut model = GBDT::new(&cfg); + model.fit(&mut train_dv); let avg_pattern_length = pattern_length_size_sum / (&patterns.len() + &anti_patterns.len()); @@ -346,14 +401,13 @@ impl AnalyticUnit for PatternAnalyticUnit { for v in window.iter() { vs.push(*v); } - let fs = get_features(&vs); + let fs = get_features(&vs).iter().map(|e| *e as f32).collect(); let lk = lr.model.lock(); - let p = lk.predict(Array::from_vec(fs.to_vec())); - if p { - 1 - } else { - -1 - } + + let data_t1 = Data::new_test_data(fs, Some(0.5)); + let mut test_dv = Vec::new(); + test_dv.push(data_t1); + lk.predict(&test_dv)[0] }; let score = positive_corr * self.config.correlation_score