Split out models from detectors #98 (#101)

* Create abstract model class

* Move detectors/*_detector -> models/*_model

* Update Model class

* Change detectors to models and move fields to self.state

* Use models instead of detectors in PatternDetector

* Update inits in detectors/ and models/

* Add types to resolve_model_by_pattern

* Add types to abstract Model class

analytics/detectors/__init__.py

@@ -1,5 +1,3 @@
-from detectors.general_detector import GeneralDetector
 from detectors.pattern_detector import PatternDetector
-from detectors.peaks_detector import PeaksDetector
+# TODO: do something with general detector
-from detectors.step_detector import StepDetector
+from detectors.general_detector import GeneralDetector
-from detectors.jump_detector import JumpDetector

analytics/detectors/pattern_detector.py

@@ -1,4 +1,4 @@
-import detectors
+import models
 import utils
 
 from grafana_data_provider import GrafanaDataProvider
@@ -16,13 +16,13 @@ logger = logging.getLogger('analytic_toolset')
 
 
 
-def resolve_detector_by_pattern(pattern):
+def resolve_model_by_pattern(pattern: str) -> models.Model:
     if pattern == 'peak':
-        return detectors.PeaksDetector()
+        return models.PeaksModel()
     if pattern == 'drop':
-        return detectors.StepDetector()
+        return models.StepModel()
     if pattern == 'jump':
-        return detectors.JumpDetector()
+        return models.JumpModel()
     raise ValueError('Unknown pattern "%s"' % pattern)
 
 
@@ -53,7 +53,7 @@ class PatternDetector:
         self.__load_model(pattern_type)
 
     async def learn(self, segments):
-        self.model = resolve_detector_by_pattern(self.pattern_type)
+        self.model = resolve_model_by_pattern(self.pattern_type)
         window_size = 200
 
         dataframe = self.data_prov.get_dataframe()
@@ -109,5 +109,5 @@ class PatternDetector:
         logger.info("Load model '%s'" % self.analytic_unit_id)
         model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_type + ".m")
         if os.path.exists(model_filename):
-            self.model = resolve_detector_by_pattern(pattern)
+            self.model = resolve_model_by_pattern(pattern)
             self.model.load(model_filename)

analytics/models/__init__.py

@@ -0,0 +1,4 @@
+from models.model import Model
+from models.step_model import StepModel
+from models.peaks_model import PeaksModel
+from models.jump_model import JumpModel

analytics/models/jump_model.py

@@ -1,22 +1,26 @@
+from models import Model
+
 import utils
 import numpy as np
-import pickle
 import scipy.signal
 from scipy.fftpack import fft
 from scipy.signal import argrelextrema
 import math
 
+
 WINDOW_SIZE = 120
 
-class JumpDetector:
+class JumpModel(Model):
 
     def __init__(self):
-        self.segments = []
+        super()
-        self.confidence = 1.5
+        self.state = {
-        self.convolve_max = WINDOW_SIZE
+            'confidence': 1.5,
-        self.size = 50
+            'convolve_max': WINDOW_SIZE
+        }
     
     async def fit(self, dataframe, segments):
+        self.segments = segments
         #self.alpha_finder()
         data = dataframe['value']
         confidences = []
@@ -59,14 +63,14 @@ class JumpDetector:
 
 
         if len(confidences) > 0:
-            self.confidence = min(confidences)
+            self.state['confidence'] = min(confidences)
         else:
-            self.confidence = 1.5
+            self.state['confidence'] = 1.5
 
         if len(convolve_list) > 0:
-            self.convolve_max = max(convolve_list)
+            self.state['convolve_max'] = max(convolve_list)
         else:
-            self.convolve_max = WINDOW_SIZE # макс метрика свертки равна отступу(WINDOW_SIZE), вау!
+            self.state['convolve_max'] = WINDOW_SIZE # макс метрика свертки равна отступу(WINDOW_SIZE), вау!
     
     async def predict(self, dataframe):
         data = dataframe['value']
@@ -82,10 +86,10 @@ class JumpDetector:
         window_size = 24
         all_max_flatten_data = data.rolling(window=window_size).mean()
         all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0]        
-        possible_jumps = utils.find_all_jumps(all_max_flatten_data, 50, self.confidence)
+        possible_jumps = utils.find_all_jumps(all_max_flatten_data, 50, self.state['confidence'])
 
         '''
-        for i in utils.exponential_smoothing(data + self.confidence, 0.02):
+        for i in utils.exponential_smoothing(data + self.state['confidence'], 0.02):
             extrema_list.append(i)
 
         segments = []
@@ -117,24 +121,13 @@ class JumpDetector:
         for segment in segments:
             convol_data = all_max_flatten_data[segment - WINDOW_SIZE : segment + WINDOW_SIZE]
             conv = scipy.signal.fftconvolve(pattern_data, convol_data)
-            if max(conv) > self.convolve_max * 1.1 or max(conv) < self.convolve_max * 0.9:
+            if max(conv) > self.state['convolve_max'] * 1.1 or max(conv) < self.state['convolve_max'] * 0.9:
                 delete_list.append(segment)
         for item in delete_list:
             segments.remove(item)
 
         return segments
 
-    def save(self, model_filename):
-        with open(model_filename, 'wb') as file:
-            pickle.dump((self.confidence, self.convolve_max), file)
-
-    def load(self, model_filename):
-        try:
-            with open(model_filename, 'rb') as file:
-                (self.confidence, self.convolve_max) = pickle.load(file)
-        except:
-            pass
-
     def alpha_finder(self, data):
         """
         поиск альфы для логистической сигмоиды

analytics/models/model.py

@@ -0,0 +1,30 @@
+from abc import ABC, abstractmethod
+from pandas import DataFrame
+import pickle
+
+class Model(ABC):
+    
+    def __init__(self):
+        """
+        Variables which are obtained as a result of fit() method 
+        should be stored in self.state dict
+        in order to be saved in model file
+        """
+        self.state = {}
+        self.segments = []
+
+    @abstractmethod
+    async def fit(self, dataframe: DataFrame, segments: list):
+        pass
+
+    @abstractmethod
+    async def predict(self, dataframe: DataFrame) -> list:
+        pass
+
+    def save(self, model_filename: str):
+        with open(model_filename, 'wb') as file:
+            pickle.dump(self.state, file)
+
+    def load(self, model_filename: str):
+        with open(model_filename, 'rb') as f:
+            self.state = pickle.load(f)

analytics/models/peaks_model.py

@@ -1,11 +1,14 @@
+from models import Model
+
 import utils
 from scipy import signal
 import numpy as np
 
 
-class PeaksDetector:
+class PeaksModel(Model):
+
     def __init__(self):
-        pass
+        super()
 
     async def fit(self, dataset, contamination=0.005):
         pass
@@ -53,13 +56,3 @@ class PeaksDetector:
 
         result = utils.find_steps(data, 0.1)
         return [(dataframe.index[x], dataframe.index[x + window_size]) for x in result]
-
-    def save(self, model_filename):
-        pass
-        # with open(model_filename, 'wb') as file:
-        #     pickle.dump((self.clf, self.scaler), file)
-
-    def load(self, model_filename):
-        pass
-        # with open(model_filename, 'rb') as file:
-        #     self.clf, self.scaler = pickle.load(file)

analytics/models/step_model.py

@@ -1,3 +1,5 @@
+from models import Model
+
 import scipy.signal
 from scipy.fftpack import fft
 from scipy.signal import argrelextrema
@@ -7,14 +9,18 @@ import numpy as np
 import pickle
 
 
-class StepDetector:
+class StepModel(Model):
 
     def __init__(self):
+        super()
         self.segments = []
-        self.confidence = 1.5
+        self.state = {
-        self.convolve_max = 570000
+            'confidence': 1.5,
+            'convolve_max': 570000
+        }
 
     async def fit(self, dataframe, segments):
+        self.segments = segments
         data = dataframe['value']
         confidences = []
         convolve_list = []
@@ -32,14 +38,14 @@ class StepDetector:
                 convolve_list.append(max(convolve))
 
         if len(confidences) > 0:
-            self.confidence = min(confidences)
+            self.state['confidence'] = min(confidences)
         else:
-            self.confidence = 1.5
+            self.state['confidence'] = 1.5
 
         if len(convolve_list) > 0:
-            self.convolve_max = max(convolve_list)
+            self.state['convolve_max'] = max(convolve_list)
         else:
-            self.convolve_max = 570000
+            self.state['convolve_max'] = 570000
 
     async def predict(self, dataframe):
         data = dataframe['value']
@@ -57,7 +63,7 @@ class StepDetector:
         all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0]
         extrema_list = []
 
-        for i in utils.exponential_smoothing(data - self.confidence, 0.03):
+        for i in utils.exponential_smoothing(data - self.state['confidence'], 0.03):
             extrema_list.append(i)
 
         segments = []
@@ -83,20 +89,9 @@ class StepDetector:
         for segment in segments:
             convol_data = all_max_flatten_data[segment - 120 : segment + 120]
             conv = scipy.signal.fftconvolve(pattern_data, convol_data)
-            if max(conv) > self.convolve_max * 1.1 or max(conv) < self.convolve_max * 0.9:
+            if max(conv) > self.state['convolve_max'] * 1.1 or max(conv) < self.state['convolve_max'] * 0.9:
                 delete_list.append(segment)
         for item in delete_list:
             segments.remove(item)
 
         return segments
-
-    def save(self, model_filename):
-        with open(model_filename, 'wb') as file:
-            pickle.dump((self.confidence, self.convolve_max), file)
-
-    def load(self, model_filename):
-        try:
-            with open(model_filename, 'rb') as file:
-                (self.confidence, self.convolve_max) = pickle.load(file)
-        except:
-            pass