hastic
/
hastic
4
1
Fork 0
Code Issues 61 Pull Requests Releases Activity
Browse Source

desision trees experiment

pull/25/head
Alexey Velikiy 3 years ago
parent
27cee51551
commit
42e7da5efd
5 changed files with 294 additions and 209 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      client/package.json
  2. 2
      client/yarn.lock
  3. 387
      server/Cargo.lock
  4. 4
      server/Cargo.toml
  5. 106
      server/src/services/analytic_service/analytic_unit/pattern_analytic_unit.rs

2
client/package.json
Unescape View File

@ -9,7 +9,7 @@
"lint": "vue-cli-service lint"
},
"dependencies": {
"@chartwerk/line-pod": "0.4.4",
"@chartwerk/line-pod": "^0.4.4",
"@chartwerk/scatter-pod": "^0.2.4",
"@kyvg/vue3-notification": "^2.3.4",
"@types/lodash": "^4.14.176",

2
client/yarn.lock
Unescape View File

@ -232,7 +232,7 @@
version "0.1.0"
resolved "https://codeload.github.com/chartwerk/core/tar.gz/532eddbc8ad938091b1d9ec1693cec5eddfdbfc2"
"@chartwerk/line-pod@0.4.4":
"@chartwerk/line-pod@^0.4.4":
version "0.4.4"
resolved "https://registry.yarnpkg.com/@chartwerk/line-pod/-/line-pod-0.4.4.tgz#724152e886c529df97019ba3dfed335ef42b2f57"
integrity sha512-L7RmTzy+TVMueV91E3upVCK/V7t9ZYLwC0tIYhFJxSJjrLOhRMAOLxZdkp7CqxXFj+u+FuFDhY0Dch2nYDvLpg==

387
server/Cargo.lock generated
Unescape View File

@ -37,15 +37,6 @@ version = "1.0.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
[[package]]
name = "approx"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f2a05fd1bd10b2527e20a2cd32d8873d115b8b39fe219ee25f42a8aca6ba278"
dependencies = [
"num-traits 0.2.14",
]
[[package]]
name = "arrayvec"
version = "0.5.2"
@ -74,6 +65,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "autocfg"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
[[package]]
name = "autocfg"
version = "1.0.1"
@ -150,6 +147,12 @@ version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
[[package]]
name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
[[package]]
name = "cfg-if"
version = "1.0.0"
@ -184,6 +187,15 @@ dependencies = [
"vec_map",
]
[[package]]
name = "cloudabi"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
dependencies = [
"bitflags",
]
[[package]]
name = "config"
version = "0.11.0"
@ -256,12 +268,6 @@ dependencies = [
"generic-array",
]
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "fallible-iterator"
version = "0.2.0"
@ -314,6 +320,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "futures"
version = "0.3.17"
@ -368,7 +380,7 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb"
dependencies = [
"autocfg",
"autocfg 1.0.1",
"proc-macro-hack",
"proc-macro2",
"quote",
@ -393,7 +405,7 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481"
dependencies = [
"autocfg",
"autocfg 1.0.1",
"futures-channel",
"futures-core",
"futures-io",
@ -408,6 +420,20 @@ dependencies = [
"slab",
]
[[package]]
name = "gbdt"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74248386ea349f903cee13fae53f41bdae987f74eb798c6cbcb08370c99ccd34"
dependencies = [
"cfg-if 0.1.10",
"rand 0.6.5",
"regex",
"serde 1.0.130",
"serde_derive",
"serde_json",
]
[[package]]
name = "generic-array"
version = "0.14.4"
@ -424,7 +450,7 @@ version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"libc",
"wasi 0.9.0+wasi-snapshot-preview1",
]
@ -435,7 +461,7 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"libc",
"wasi 0.10.0+wasi-snapshot-preview1",
]
@ -488,13 +514,13 @@ dependencies = [
"config",
"fastrand",
"futures",
"gbdt",
"libsqlite3-sys",
"linfa",
"linfa-svm",
"ndarray",
"openssl",
"parking_lot",
"rusqlite",
"rustfft",
"serde 1.0.130",
"serde_json",
"subbeat",
@ -624,7 +650,7 @@ version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
dependencies = [
"autocfg",
"autocfg 1.0.1",
"hashbrown",
]
@ -643,16 +669,7 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
]
[[package]]
name = "itertools"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf"
dependencies = [
"either",
"cfg-if 1.0.0",
]
[[package]]
@ -661,15 +678,6 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "kdtree"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80ee359328fc9087e9e3fc0a4567c4dd27ec69a127d6a70e8d9dd22845b8b1a2"
dependencies = [
"num-traits 0.2.14",
]
[[package]]
name = "lazy_static"
version = "1.4.0"
@ -684,7 +692,7 @@ checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
dependencies = [
"arrayvec",
"bitflags",
"cfg-if",
"cfg-if 1.0.0",
"ryu",
"static_assertions",
]
@ -695,12 +703,6 @@ version = "0.2.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "869d572136620d55835903746bcb5cdc54cb2851fd0aeec53220b4bb65ef3013"
[[package]]
name = "libm"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7d73b3f436185384286bd8098d17ec07c9a7d2388a6599f824d8502b529702a"
[[package]]
name = "libsqlite3-sys"
version = "0.23.1"
@ -712,64 +714,6 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "linfa"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7496612871c85de124d12006109360ed7ffc4740ce3fe3fe35a749e2d7f2f88f"
dependencies = [
"approx",
"ndarray",
"num-traits 0.2.14",
"rand 0.8.4",
"thiserror",
]
[[package]]
name = "linfa-kernel"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6da781111566c8b9867c8e9c0ac69fd72808dda1e111164aa23ece9f15d6a95a"
dependencies = [
"linfa",
"linfa-nn",
"ndarray",
"num-traits 0.2.14",
"serde 1.0.130",
"sprs",
]
[[package]]
name = "linfa-nn"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83748e3f71c8fa7fc490a54d2e33d40b8246a335618159fb313989ff6eb209ef"
dependencies = [
"kdtree",
"linfa",
"ndarray",
"ndarray-stats",
"noisy_float",
"num-traits 0.2.14",
"order-stat",
"thiserror",
]
[[package]]
name = "linfa-svm"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ecfffa7a24c3019dc4c4fa9c58bf8f187ff2fdbbd08b6fee454591c6ec285f3f"
dependencies = [
"linfa",
"linfa-kernel",
"ndarray",
"ndarray-rand",
"num-traits 0.2.14",
"serde 1.0.130",
"thiserror",
]
[[package]]
name = "linked-hash-map"
version = "0.5.4"
@ -791,7 +735,7 @@ version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
]
[[package]]
@ -895,48 +839,11 @@ version = "0.15.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08e854964160a323e65baa19a0b1a027f76d590faba01f05c0cbc3187221a8c9"
dependencies = [
"approx",
"matrixmultiply",
"num-complex",
"num-integer",
"num-traits 0.2.14",
"rawpointer",
"serde 1.0.130",
]
[[package]]
name = "ndarray-rand"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65608f937acc725f5b164dcf40f4f0bc5d67dc268ab8a649d3002606718c4588"
dependencies = [
"ndarray",
"rand 0.8.4",
"rand_distr",
]
[[package]]
name = "ndarray-stats"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22877ad014bafa2f7dcfa5d556b0c7a52b0546cc98061a1ebef6d1834957b069"
dependencies = [
"indexmap",
"itertools",
"ndarray",
"noisy_float",
"num-integer",
"num-traits 0.2.14",
"rand 0.8.4",
]
[[package]]
name = "noisy_float"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "978fe6e6ebc0bf53de533cd456ca2d9de13de13856eda1518a285d7705a213af"
dependencies = [
"num-traits 0.2.14",
]
[[package]]
@ -974,7 +881,7 @@ version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg",
"autocfg 1.0.1",
"num-traits 0.2.14",
]
@ -993,8 +900,7 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
dependencies = [
"autocfg",
"libm",
"autocfg 1.0.1",
]
[[package]]
@ -1026,7 +932,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577"
dependencies = [
"bitflags",
"cfg-if",
"cfg-if 1.0.0",
"foreign-types",
"libc",
"once_cell",
@ -1054,7 +960,7 @@ version = "0.9.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058"
dependencies = [
"autocfg",
"autocfg 1.0.1",
"cc",
"libc",
"openssl-src",
@ -1062,12 +968,6 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "order-stat"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "efa535d5117d3661134dbf1719b6f0ffe06f2375843b13935db186cd094105eb"
[[package]]
name = "parking_lot"
version = "0.11.2"
@ -1085,7 +985,7 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"instant",
"libc",
"redox_syscall",
@ -1143,6 +1043,15 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741"
[[package]]
name = "primal-check"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01419cee72c1a1ca944554e23d83e483e1bccf378753344e881de28b5487511d"
dependencies = [
"num-integer",
]
[[package]]
name = "proc-macro-hack"
version = "0.5.19"
@ -1179,6 +1088,25 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca"
dependencies = [
"autocfg 0.1.7",
"libc",
"rand_chacha 0.1.1",
"rand_core 0.4.2",
"rand_hc 0.1.0",
"rand_isaac",
"rand_jitter",
"rand_os",
"rand_pcg",
"rand_xorshift",
"winapi",
]
[[package]]
name = "rand"
version = "0.7.3"
@ -1204,6 +1132,16 @@ dependencies = [
"rand_hc 0.3.1",
]
[[package]]
name = "rand_chacha"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef"
dependencies = [
"autocfg 0.1.7",
"rand_core 0.3.1",
]
[[package]]
name = "rand_chacha"
version = "0.2.2"
@ -1224,6 +1162,21 @@ dependencies = [
"rand_core 0.6.3",
]
[[package]]
name = "rand_core"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
dependencies = [
"rand_core 0.4.2",
]
[[package]]
name = "rand_core"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]]
name = "rand_core"
version = "0.5.1"
@ -1243,13 +1196,12 @@ dependencies = [
]
[[package]]
name = "rand_distr"
version = "0.4.2"
name = "rand_hc"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "964d548f8e7d12e102ef183a0de7e98180c9f8729f555897a857b96e48122d2f"
checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4"
dependencies = [
"num-traits 0.2.14",
"rand 0.8.4",
"rand_core 0.3.1",
]
[[package]]
@ -1270,12 +1222,74 @@ dependencies = [
"rand_core 0.6.3",
]
[[package]]
name = "rand_isaac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rand_jitter"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b"
dependencies = [
"libc",
"rand_core 0.4.2",
"winapi",
]
[[package]]
name = "rand_os"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071"
dependencies = [
"cloudabi",
"fuchsia-cprng",
"libc",
"rand_core 0.4.2",
"rdrand",
"winapi",
]
[[package]]
name = "rand_pcg"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44"
dependencies = [
"autocfg 0.1.7",
"rand_core 0.4.2",
]
[[package]]
name = "rand_xorshift"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rdrand"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "redox_syscall"
version = "0.2.10"
@ -1338,6 +1352,20 @@ version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2"
[[package]]
name = "rustfft"
version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1d089e5c57521629a59f5f39bca7434849ff89bd6873b521afe389c1c602543"
dependencies = [
"num-complex",
"num-integer",
"num-traits 0.2.14",
"primal-check",
"strength_reduce",
"transpose",
]
[[package]]
name = "ryu"
version = "1.0.5"
@ -1474,7 +1502,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6"
dependencies = [
"block-buffer",
"cfg-if",
"cfg-if 1.0.0",
"cpufeatures",
"digest",
"opaque-debug",
@ -1511,25 +1539,18 @@ dependencies = [
"winapi",
]
[[package]]
name = "sprs"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ea71e48b3eab4c4b153e8e35dcaeac132720809ef68359097b8cb54a18edd70"
dependencies = [
"ndarray",
"num-complex",
"num-traits 0.2.14",
"serde 1.0.130",
"smallvec",
]
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strength_reduce"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3ff2f71c82567c565ba4b3009a9350a96a7269eaa4001ebedae926230bc2254"
[[package]]
name = "strsim"
version = "0.8.0"
@ -1574,7 +1595,7 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"libc",
"rand 0.8.4",
"redox_syscall",
@ -1643,7 +1664,7 @@ version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2c2416fdedca8443ae44b4527de1ea633af61d8f7169ffa6e72c5b53d24efcc"
dependencies = [
"autocfg",
"autocfg 1.0.1",
"bytes",
"libc",
"memchr",
@ -1737,7 +1758,7 @@ version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105"
dependencies = [
"cfg-if",
"cfg-if 1.0.0",
"log",
"pin-project-lite",
"tracing-core",
@ -1752,6 +1773,16 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "transpose"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95f9c900aa98b6ea43aee227fd680550cdec726526aab8ac801549eadb25e39f"
dependencies = [
"num-integer",
"strength_reduce",
]
[[package]]
name = "try-lock"
version = "0.2.3"

4
server/Cargo.toml
Unescape View File

@ -23,11 +23,11 @@ libsqlite3-sys = { version = "*", features = ["bundled"] }
futures = "0.3.17"
chrono = "0.4.19"
linfa = "0.5.0"
linfa-svm = { version="0.5.0", features=["serde"] }
gbdt = "0.1.1"
ndarray = "0.15.3"
bincode = "1.3.3"
async-trait = "0.1.51"
rustfft = "6.0.1"
# TODO: remove this from prod
# plotlib = "0.5.1"

106
server/src/services/analytic_service/analytic_unit/pattern_analytic_unit.rs
Unescape View File

@ -3,10 +3,12 @@ use std::{collections::VecDeque, fmt, sync::Arc};
use futures::future;
use parking_lot::Mutex;
use linfa::prelude::*;
use linfa;
use linfa_svm::Svm;
use gbdt::config::Config;
use gbdt::decision_tree::{Data, DataVec, PredVec};
use gbdt::gradient_boost::GBDT;
use ndarray::Array;
@ -16,13 +18,20 @@ use super::types::{AnalyticUnit, AnalyticUnitConfig, LearningResult, PatternConf
use async_trait::async_trait;
use rustfft::{self, FftPlanner, num_complex::Complex};
// TODO: move to config
const DETECTION_STEP: u64 = 10;
const FFT_LEN: usize = 64;
// TODO: convert to vector
pub const FEATURES_SIZE: usize = 4 + 16 * 2;
pub type Features = Vec<f64>;
#[derive(Clone)]
pub struct LearningResults {
// TODO: replace with RWLock
model: Arc<Mutex<Svm<f64, bool>>>,
model: Arc<Mutex<GBDT>>,
pub learning_train: LearningTrain,
@ -55,9 +64,7 @@ impl fmt::Debug for LearningResults {
}
}
pub const FEATURES_SIZE: usize = 4;
pub type Features = [f64; FEATURES_SIZE];
fn nan_to_zero(n: f64) -> f64 {
if n.is_nan() {
@ -111,12 +118,44 @@ fn get_features(xs: &Vec<f64>) -> Features {
let sd = sum.sqrt();
// TODO: add autocorrelation
// TODO: add FFT
// TODO: add DWT
return [
let mut planner = FftPlanner::<f64>::new();
// TODO: move 128 to config
let fft = planner.plan_fft_forward(FFT_LEN);
let mut c_buffer = vec![Complex{ re: 0.0f64, im: 0.0f64 }; FFT_LEN];
for i in 0..FFT_LEN.min(xs.len()) {
c_buffer[i].re = xs[i];
}
fft.process(&mut c_buffer);
// https://docs.rs/rustfft/6.0.1/rustfft/index.html#normalization
let norm_factor = (FFT_LEN.min(xs.len()) as f64).sqrt();
for i in 0..FFT_LEN.min(xs.len()) {
c_buffer[i] /= norm_factor;
}
return vec![
min, max, mean, sd,
c_buffer[0].re, c_buffer[0].im,
c_buffer[1].re, c_buffer[1].im,
c_buffer[2].re, c_buffer[2].im,
c_buffer[3].re, c_buffer[3].im,
c_buffer[4].re, c_buffer[4].im,
c_buffer[5].re, c_buffer[5].im,
c_buffer[6].re, c_buffer[6].im,
c_buffer[7].re, c_buffer[7].im,
c_buffer[8].re, c_buffer[8].im,
c_buffer[9].re, c_buffer[9].im,
c_buffer[10].re, c_buffer[10].im,
c_buffer[11].re, c_buffer[11].im,
c_buffer[12].re, c_buffer[12].im,
c_buffer[13].re, c_buffer[13].im,
c_buffer[14].re, c_buffer[14].im,
c_buffer[15].re, c_buffer[15].im,
// 0f64,0f64,
// 0f64,0f64,0f64, 0f64
];
@ -202,6 +241,19 @@ impl AnalyticUnit for PatternAnalyticUnit {
}
async fn learn(&mut self, ms: MetricService, ss: SegmentsService) -> LearningResult {
// TODO: move to config
let mut cfg = Config::new();
cfg.set_feature_size(FEATURES_SIZE);
cfg.set_max_depth(3);
cfg.set_iterations(50);
cfg.set_shrinkage(0.1);
cfg.set_loss("LogLikelyhood");
cfg.set_debug(false);
cfg.set_data_sample_ratio(1.0);
cfg.set_feature_sample_ratio(1.0);
cfg.set_training_optimization_level(2);
// be careful if decide to store detections in db
let segments = ss.get_segments_inside(0, u64::MAX / 2).unwrap();
let has_segments_label = segments
@ -263,19 +315,22 @@ impl AnalyticUnit for PatternAnalyticUnit {
anti_patterns.push(xs);
}
let records = Array::from_shape_fn((records_raw.len(), FEATURES_SIZE), |(i, j)| {
records_raw[i][j]
});
let mut train_dv = Vec::new();
assert_eq!(records_raw.len(), targets_raw.len());
let targets = Array::from_vec(targets_raw.clone());
for i in 0..train_dv.len() {
let data = Data::new_training_data(
records_raw[i].iter().map(|e| *e as f32).collect(),
1.0,
if targets_raw[i] { 1.0 } else { -1.0 },
Some(0.5)
);
train_dv.push(data);
}
let train = linfa::Dataset::new(records, targets);
let model = Svm::<_, bool>::params()
.pos_neg_weights(50000., 5000.)
.gaussian_kernel(80.0)
.fit(&train)
.unwrap();
let mut model = GBDT::new(&cfg);
model.fit(&mut train_dv);
let avg_pattern_length = pattern_length_size_sum / (&patterns.len() + &anti_patterns.len());
@ -346,14 +401,13 @@ impl AnalyticUnit for PatternAnalyticUnit {
for v in window.iter() {
vs.push(*v);
}
let fs = get_features(&vs);
let fs = get_features(&vs).iter().map(|e| *e as f32).collect();
let lk = lr.model.lock();
let p = lk.predict(Array::from_vec(fs.to_vec()));
if p {
1
} else {
-1
}
let data_t1 = Data::new_test_data(fs, Some(0.5));
let mut test_dv = Vec::new();
test_dv.push(data_t1);
lk.predict(&test_dv)[0]
};
let score = positive_corr * self.config.correlation_score

Write Preview
Loading…
Cancel
Save