Merge pull request #25 from hastic/anomaly-detector-fix-learning-failire-#23

Anomaly detector fix learning failire #23

client/package.json

@@ -9,7 +9,7 @@
     "lint": "vue-cli-service lint"
   },
   "dependencies": {
-    "@chartwerk/line-pod": "^0.4.4",
+    "@chartwerk/line-pod": "^0.4.6",
     "@chartwerk/scatter-pod": "^0.2.4",
     "@kyvg/vue3-notification": "^2.3.4",
     "@types/lodash": "^4.14.176",

client/yarn.lock

@@ -223,21 +223,21 @@
     "@babel/helper-validator-identifier" "^7.14.9"
     to-fast-properties "^2.0.0"
 
-"@chartwerk/core@^0.3.2":
-  version "0.3.2"
-  resolved "https://registry.yarnpkg.com/@chartwerk/core/-/core-0.3.2.tgz#40187a4d1dda5045effc04c01559ff16c236bca7"
-  integrity sha512-E4Bb2rDBDTNXRM7koSXDwM6IYYNdLnqxJel1y2BmH4p9R0Z+3L0C4B+jYurkiwcnIWdGlHXVFL0E0khPeLypfA==
+"@chartwerk/core@^0.3.3":
+  version "0.3.3"
+  resolved "https://registry.yarnpkg.com/@chartwerk/core/-/core-0.3.3.tgz#f48db1307cb068fe1fefc2089593903643c223bd"
+  integrity sha512-aauWwYLPeMuLF727twW74GW6Uw2yqPXD8vYuFle/4nN4xjKbjWJvU7/9wlF4xh/azrpOrYWdD4l4SD25u8DzUg==
 
 "@chartwerk/core@github:chartwerk/core#532eddbc8ad938091b1d9ec1693cec5eddfdbfc2":
   version "0.1.0"
   resolved "https://codeload.github.com/chartwerk/core/tar.gz/532eddbc8ad938091b1d9ec1693cec5eddfdbfc2"
 
-"@chartwerk/line-pod@^0.4.4":
-  version "0.4.4"
-  resolved "https://registry.yarnpkg.com/@chartwerk/line-pod/-/line-pod-0.4.4.tgz#724152e886c529df97019ba3dfed335ef42b2f57"
-  integrity sha512-L7RmTzy+TVMueV91E3upVCK/V7t9ZYLwC0tIYhFJxSJjrLOhRMAOLxZdkp7CqxXFj+u+FuFDhY0Dch2nYDvLpg==
+"@chartwerk/line-pod@^0.4.6":
+  version "0.4.6"
+  resolved "https://registry.yarnpkg.com/@chartwerk/line-pod/-/line-pod-0.4.6.tgz#b4cd4b48acf2eaacf8a9e2db81c79533908ca567"
+  integrity sha512-hZNsrjM0euDhh2SRr1EZTXrnDXdGQ+rUmBpg3HhFJR70MWrrvJMYjtaRCsP6Xx1z8h3ayGq5ye7yLY9AQD4Y9g==
   dependencies:
-    "@chartwerk/core" "^0.3.2"
+    "@chartwerk/core" "^0.3.3"
 
 "@chartwerk/scatter-pod@^0.2.4":
   version "0.2.4"

server/src/services/analytic_service/analytic_unit/anomaly_analytic_unit.rs

@@ -12,21 +12,22 @@ use chrono::prelude::*;
 
 // TODO: move to config
 const DETECTION_STEP: u64 = 10;
-
-// offset from intex in timrange in seconds
-fn get_value_with_offset(ts: &Vec<(u64, f64)>, index: usize, offset: u64) -> anyhow::Result<f64> {
-    // TODO: implement
-    if index == 0 {
-        return Err(anyhow::format_err!("index should be > 0"));
+const SEASONALITY_ITERATIONS: u64 = 3; // TODO: better name
+
+// timerange offset in seconds backwards from end of ts in assumption that ts has no gaps
+fn get_value_with_offset(ts: &Vec<(u64, f64)>, offset: u64) -> Option<(u64, f64)>{
+    // TODO: remove dependency to DETECTION_STEP
+    
+    let indexes_offset = (offset / DETECTION_STEP) as usize;
+    let n = ts.len() - 1;
+    if n < indexes_offset {
+        return None;
     }
-    return Ok(0.0);
-    // let step =
-    // let index_candidate =
-    // let intex_candidate =
+    let i = n - indexes_offset;
+    return Some(ts[i]);
 }
 
 
-
 struct SARIMA {
     pub ts: Vec<(u64, f64)>,
     pub seasonality: u64,
@@ -42,27 +43,40 @@ impl SARIMA {
 
     pub fn learn(&mut self, ts: &Vec<(u64, f64)>) -> anyhow::Result<()> {
 
+        // TODO: don't count NaNs in model
+        // TODO: add exponental smooting to model
+        // TODO: trend detection
+
         if ts.len() < 2 {
             return Err(anyhow::format_err!("to short timeserie to learn from"));
         }
         // TODO: ensure capacity with seasonality size
-        let res_ts = Vec::<(u64, f64)>::new();
+        let mut res_ts = Vec::<(u64, f64)>::new();
         let from = ts[0].0;
         let to = ts.last().unwrap().0;
-        
-        if to - from != 3 * self.seasonality {
-            return Err(anyhow::format_err!("timeserie to learn from should be 3 * sasonality"));
+        // let s_from = ts[ts.len() - (self.seasonality / DETECTION_STEP) as usize].0;
+
+        if to - from != SEASONALITY_ITERATIONS * self.seasonality {
+            return Err(anyhow::format_err!("timeserie to learn from should be {} * sasonality", SEASONALITY_ITERATIONS));
         }
 
-        // TODO: compute avg based on seasonality
+        let iter_steps = (self.seasonality / DETECTION_STEP) as usize;
+        let mut vts = Vec::new();
+        for k in 0..iter_steps {
+            for si in 0..SEASONALITY_ITERATIONS {
+                vts.push(ts[k + iter_steps * si as usize].1);
+            }
+            let mut vt: f64 = vts.iter().sum();
+            vt /= SEASONALITY_ITERATIONS as f64;
+            res_ts.push((k as u64 * DETECTION_STEP, vt));
+        }
 
         self.ts = res_ts;
 
         return Ok(());
-        
     }
     pub fn predict(&self, timestamp: u64, value: f64) -> (f64, f64, f64) {
-        // TODO: implement
+        // TODO: basic implement based on existing ts
         return (0.0, 0.0, 0.0);
     }
 
@@ -70,11 +84,9 @@ impl SARIMA {
         // TODO: inmplement
     }
 
-    // TODO: don't count NaNs in model
 }
 
 
-
 pub struct AnomalyAnalyticUnit {
     config: AnomalyConfig,
     sarima: Option<SARIMA>,
@@ -139,7 +151,7 @@ impl AnalyticUnit for AnomalyAnalyticUnit {
 
         let utc: DateTime<Utc> = Utc::now();
         let to = utc.timestamp() as u64;
-        let from = to - self.config.seasonality * 3;
+        let from = to - self.config.seasonality * SEASONALITY_ITERATIONS;
 
         let mr = ms.query(from, to, DETECTION_STEP).await.unwrap();
         if mr.data.keys().len() == 0 {
@@ -148,7 +160,9 @@ impl AnalyticUnit for AnomalyAnalyticUnit {
 
         let k = mr.data.keys().nth(0).unwrap();
         let ts = &mr.data[k];
-        sarima.learn(ts);
+        sarima.learn(ts).unwrap();
+
+        self.sarima = Some(sarima);
 
         // TODO: ensure that learning reruns on seasonaliy change
         // TODO: load data to learning