Improve all models

analytics/models/custom_model.py

@@ -15,5 +15,5 @@ class CustomModel(Model):
     def fit(self, dataframe: pd.DataFrame, segments: list, cache: Optional[dict]) -> dict:
         pass
     
-    def predict(self, dataframe, cache: Optional[dict]):
+    def do_predict(self, dataframe: pd.DataFrame) -> list:
         return []

analytics/models/drop_model.py

@@ -10,7 +10,7 @@ import numpy as np
 import pandas as pd
 from typing import Optional
 
-WINDOW_SIZE = 300
+WINDOW_SIZE = 200
 
 class DropModel(Model):
     def __init__(self):
@@ -98,7 +98,7 @@ class DropModel(Model):
 
         return self.state
 
-    def do_predict(self, dataframe: pd.DataFrame):
+    def do_predict(self, dataframe: pd.DataFrame) -> list:
         data = dataframe['value']
         possible_drops = utils.find_drop(data, self.state['DROP_HEIGHT'], self.state['DROP_LENGTH'] + 1)
 
@@ -115,7 +115,7 @@ class DropModel(Model):
         for i in range(1, len(segments)):
             if segments[i] < segments[i - 1] + variance_error:
                 delete_list.append(segments[i])
-        #for item in delete_list:
+        # for item in delete_list:
         #    segments.remove(item)
         delete_list = []
 
@@ -131,7 +131,8 @@ class DropModel(Model):
                     delete_list.append(segment)
             else:
                 delete_list.append(segment)
-        for item in delete_list:
-            segments.remove(item)
+        # TODO: implement filtering
+        # for item in delete_list:
+        #     segments.remove(item)
 
-        return segments
+        return set(segments)

analytics/models/general_model.py

@@ -12,7 +12,7 @@ from scipy.stats import norm
 from typing import Optional
 
 
-WINDOW_SIZE = 350
+WINDOW_SIZE = 150
 
 
 class GeneralModel(Model):
@@ -41,12 +41,11 @@ class GeneralModel(Model):
                 segment_data = data[segment_from_index: segment_to_index + 1]
                 if len(segment_data) == 0:
                     continue
-                self.ipats.append(segment_from_index + int((segment_to_index - segment_from_index) / 2))
+                x = segment_from_index + int((segment_to_index - segment_from_index) / 2)
+                self.ipats.append(x)
+                segment_data = data[x - WINDOW_SIZE : x + WINDOW_SIZE]
                 segment_min = min(segment_data)
                 segment_data = segment_data - segment_min
-                segment_max = max(segment_data)
-                segment_data = segment_data / segment_max
-
                 convolve = scipy.signal.fftconvolve(segment_data, segment_data)
                 convolve_list.append(max(convolve))
 
@@ -57,35 +56,29 @@ class GeneralModel(Model):
 
         return self.state
 
-    def do_predict(self, dataframe: pd.DataFrame):
+    def do_predict(self, dataframe: pd.DataFrame) -> list:
         data = dataframe['value']
         pat_data = data[self.ipats[0] - WINDOW_SIZE: self.ipats[0] + WINDOW_SIZE]
         x = min(pat_data)
         pat_data = pat_data - x
         y = max(pat_data)
-        pat_data = pat_data / y
 
         for i in range(WINDOW_SIZE * 2, len(data)):
             watch_data = data[i - WINDOW_SIZE * 2: i]
             w = min(watch_data)
             watch_data = watch_data - w
-            r = max(watch_data)
-            if r < y:
-                watch_data = watch_data / y
-            else:
-                watch_data = watch_data / r
             conv = scipy.signal.fftconvolve(pat_data, watch_data)
             self.all_conv.append(max(conv))
         all_conv_peaks = utils.peak_finder(self.all_conv, WINDOW_SIZE * 2)
 
         filtered = self.__filter_prediction(all_conv_peaks, data)
+        filtered = set(item + WINDOW_SIZE for item in filtered)
         # TODO: convert from ns to ms more proper way (not dividing by 10^6)
         return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered]
 
     def __filter_prediction(self, segments: list, data: list):
         if len(segments) == 0 or len(self.ipats) == 0:
-            segments = []
-            return segments
+            return []
         delete_list = []
 
         for val in segments:
@@ -95,4 +88,4 @@ class GeneralModel(Model):
         for item in delete_list:
             segments.remove(item)
 
-        return segments
+        return set(segments)

analytics/models/jump_model.py

@@ -12,7 +12,7 @@ from scipy.stats import norm
 from typing import Optional
 
 
-WINDOW_SIZE = 300
+WINDOW_SIZE = 200
 
 class JumpModel(Model):
 
@@ -68,16 +68,16 @@ class JumpModel(Model):
                 jump_height_list.append(jump_height)
                 jump_length = utils.find_jump_length(segment_data, segment_min_line, segment_max_line)
                 jump_length_list.append(jump_length)
-                cen_ind = utils.intersection_segment(flat_segment, segment_median) #finds all interseprions with median
+                cen_ind = utils.intersection_segment(flat_segment.tolist(), segment_median) #finds all interseprions with median
                 #cen_ind =  utils.find_ind_median(segment_median, flat_segment)
                 jump_center = cen_ind[0]
                 segment_cent_index = jump_center - 5 + segment_from_index
                 self.ijumps.append(segment_cent_index)
-                labeled_drop = data[segment_cent_index - WINDOW_SIZE : segment_cent_index + WINDOW_SIZE]
-                labeled_min = min(labeled_drop)
-                for value in labeled_drop:
+                labeled_jump = data[segment_cent_index - WINDOW_SIZE : segment_cent_index + WINDOW_SIZE]
+                labeled_min = min(labeled_jump)
+                for value in labeled_jump:
                     value = value - labeled_min
-                convolve = scipy.signal.fftconvolve(labeled_drop, labeled_drop)
+                convolve = scipy.signal.fftconvolve(labeled_jump, labeled_jump)
                 convolve_list.append(max(convolve))
 
         if len(confidences) > 0:
@@ -102,7 +102,7 @@ class JumpModel(Model):
 
         return self.state
 
-    def do_predict(self, dataframe: pd.DataFrame):
+    def do_predict(self, dataframe: pd.DataFrame) -> list:
         data = dataframe['value']
         possible_jumps = utils.find_jump(data, self.state['JUMP_HEIGHT'], self.state['JUMP_LENGTH'] + 1)
 
@@ -138,7 +138,8 @@ class JumpModel(Model):
         for item in delete_list:
             segments.remove(item)
 
+        # TODO: implement filtering
         #for ijump in self.ijumps:
             #segments.append(ijump)
 
-        return segments
+        return set(segments)

analytics/models/model.py

@@ -13,7 +13,7 @@ class Model(ABC):
         pass
 
     @abstractmethod
-    def do_predict(self, dataframe: DataFrame):
+    def do_predict(self, dataframe: DataFrame) -> list:
         pass
 
     def predict(self, dataframe: DataFrame, cache: Optional[AnalyticUnitCache]) -> dict:
@@ -21,10 +21,7 @@ class Model(ABC):
             self.state = cache
 
         result = self.do_predict(dataframe)
-        result.sort()
 
-        if len(self.segments) > 0:
-            result = [segment for segment in result if not utils.is_intersect(segment, self.segments)]
         return {
             'segments': result,
             'cache': self.state

analytics/models/peak_model.py

@@ -69,8 +69,7 @@ class PeakModel(Model):
     def do_predict(self, dataframe: pd.DataFrame):
         data = dataframe['value']
         window_size = 24
-        all_max_flatten_data = data.rolling(window=window_size).mean()
-        all_maxs = argrelextrema(np.array(all_max_flatten_data), np.greater)[0]
+        all_maxs = argrelextrema(np.array(data), np.greater)[0]
 
         extrema_list = []
         for i in utils.exponential_smoothing(data + self.state['confidence'], 0.02):
@@ -78,16 +77,16 @@ class PeakModel(Model):
 
         segments = []
         for i in all_maxs:
-            if all_max_flatten_data[i] > extrema_list[i]:
+            if data[i] > extrema_list[i]:
                 segments.append(i)
 
         filtered = self.__filter_prediction(segments, data)
         # TODO: convert from ns to ms more proper way (not dividing by 10^6)
         return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered]
 
-    def __filter_prediction(self, segments: list, all_max_flatten_data: list):
+    def __filter_prediction(self, segments: list, data: list) -> list:
         delete_list = []
-        variance_error = int(0.004 * len(all_max_flatten_data))
+        variance_error = int(0.004 * len(data))
         if variance_error > 100:
             variance_error = 100
         for i in range(1, len(segments)):
@@ -100,16 +99,17 @@ class PeakModel(Model):
         if len(segments) == 0 or len(self.ipeaks) == 0:
             return []
 
-        pattern_data = all_max_flatten_data[self.ipeaks[0] - WINDOW_SIZE: self.ipeaks[0] + WINDOW_SIZE]
+        pattern_data = data[self.ipeaks[0] - WINDOW_SIZE: self.ipeaks[0] + WINDOW_SIZE]
         for segment in segments:
             if segment > WINDOW_SIZE:
-                convol_data = all_max_flatten_data[segment - WINDOW_SIZE: segment + WINDOW_SIZE]
+                convol_data = data[segment - WINDOW_SIZE: segment + WINDOW_SIZE]
                 conv = scipy.signal.fftconvolve(pattern_data, convol_data)
                 if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_max'] * 0.8:
                     delete_list.append(segment)
             else:
                 delete_list.append(segment)
-        for item in delete_list:
-            segments.remove(item)
+        # TODO: implement filtering
+        # for item in delete_list:
+        #     segments.remove(item)
 
-        return segments
+        return set(segments)

analytics/models/trough_model.py

@@ -68,8 +68,7 @@ class TroughModel(Model):
     def do_predict(self, dataframe: pd.DataFrame):
         data = dataframe['value']
         window_size = 24
-        all_max_flatten_data = data.rolling(window=window_size).mean()
-        all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0]
+        all_mins = argrelextrema(np.array(data), np.less)[0]
         
         extrema_list = []
         for i in utils.exponential_smoothing(data - self.state['confidence'], 0.02):
@@ -77,16 +76,16 @@ class TroughModel(Model):
 
         segments = []
         for i in all_mins:
-            if all_max_flatten_data[i] < extrema_list[i]:
+            if data[i] < extrema_list[i]:
                 segments.append(i)
         test = dataframe['timestamp'][1].value
         filtered = self.__filter_prediction(segments, data)
         # TODO: convert from ns to ms more proper way (not dividing by 10^6)
         return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered]
 
-    def __filter_prediction(self, segments: list, all_max_flatten_data: list):
+    def __filter_prediction(self, segments: list, data: list) -> list:
         delete_list = []
-        variance_error = int(0.004 * len(all_max_flatten_data))
+        variance_error = int(0.004 * len(data))
         if variance_error > 100:
             variance_error = 100
         for i in range(1, len(segments)):
@@ -100,16 +99,17 @@ class TroughModel(Model):
             segments = []
             return segments
             
-        pattern_data = all_max_flatten_data[self.ipeaks[0] - WINDOW_SIZE : self.ipeaks[0] + WINDOW_SIZE]
+        pattern_data = data[self.ipeaks[0] - WINDOW_SIZE : self.ipeaks[0] + WINDOW_SIZE]
         for segment in segments:
             if segment > WINDOW_SIZE:
-                convol_data = all_max_flatten_data[segment - WINDOW_SIZE : segment + WINDOW_SIZE]
+                convol_data = data[segment - WINDOW_SIZE : segment + WINDOW_SIZE]
                 conv = scipy.signal.fftconvolve(pattern_data, convol_data)
                 if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_max'] * 0.8:
                     delete_list.append(segment)
             else:
                 delete_list.append(segment)
-        for item in delete_list:
-            segments.remove(item)
+        # TODO: implement filtering
+        # for item in delete_list:
+        #     segments.remove(item)
 
-        return segments
+        return set(segments)

analytics/utils/__init__.py

@@ -2,14 +2,6 @@ import numpy as np
 import pandas as pd
 
 
-def is_intersect(target_segment, segments):
-    for segment in segments:
-        start = max(segment['from'], target_segment[0])
-        finish = min(segment['to'], target_segment[1])
-        if start <= finish:
-            return True
-    return False
-
 def exponential_smoothing(series, alpha):
     result = [series[0]]
     for n in range(1, len(series)):

server/src/controllers/analytics_controller.ts

@@ -168,16 +168,16 @@ export async function runPredict(id: AnalyticUnit.AnalyticUnitId) {
 
     let payload = processPredictionResult(id, result);
 
-    // Merging segments
-    if(segments.length > 0 && payload.segments.length > 0) {
-      let lastOldSegment = segments[segments.length - 1];
-      let firstNewSegment = payload.segments[0];
-
-      if(firstNewSegment.from <= lastOldSegment.to) {
-        payload.segments[0].from = lastOldSegment.from;
-        Segment.removeSegments([lastOldSegment.id])
-      }
-    }
+    // TODO: implement segments merging without removing labeled
+    // if(segments.length > 0 && payload.segments.length > 0) {
+    //   let lastOldSegment = segments[segments.length - 1];
+    //   let firstNewSegment = payload.segments[0];
+
+    //   if(firstNewSegment.from <= lastOldSegment.to) {
+    //     payload.segments[0].from = lastOldSegment.from;
+    //     Segment.removeSegments([lastOldSegment.id])
+    //   }
+    // }
 
     Segment.insertSegments(payload.segments);
     AnalyticUnitCache.setData(id, payload.cache);