From b86da0fcaedc45147455b110a332ca49bce1197c Mon Sep 17 00:00:00 2001 From: rozetko Date: Sat, 28 Jul 2018 12:46:40 +0300 Subject: [PATCH] Split out models from detectors #98 (#101) * Create abstract model class * Move detectors/*_detector -> models/*_model * Update Model class * Change detectors to models and move fields to self.state * Use models instead of detectors in PatternDetector * Update inits in detectors/ and models/ * Add types to resolve_model_by_pattern * Add types to abstract Model class --- analytics/detectors/__init__.py | 6 +-- analytics/detectors/pattern_detector.py | 14 +++---- analytics/models/__init__.py | 4 ++ .../jump_detector.py => models/jump_model.py} | 41 ++++++++----------- analytics/models/model.py | 30 ++++++++++++++ .../peaks_model.py} | 17 +++----- .../step_detector.py => models/step_model.py} | 35 +++++++--------- 7 files changed, 80 insertions(+), 67 deletions(-) create mode 100644 analytics/models/__init__.py rename analytics/{detectors/jump_detector.py => models/jump_model.py} (83%) create mode 100644 analytics/models/model.py rename analytics/{detectors/peaks_detector.py => models/peaks_model.py} (82%) rename analytics/{detectors/step_detector.py => models/step_model.py} (78%) diff --git a/analytics/detectors/__init__.py b/analytics/detectors/__init__.py index ef168e8..145dc6d 100644 --- a/analytics/detectors/__init__.py +++ b/analytics/detectors/__init__.py @@ -1,5 +1,3 @@ -from detectors.general_detector import GeneralDetector from detectors.pattern_detector import PatternDetector -from detectors.peaks_detector import PeaksDetector -from detectors.step_detector import StepDetector -from detectors.jump_detector import JumpDetector +# TODO: do something with general detector +from detectors.general_detector import GeneralDetector diff --git a/analytics/detectors/pattern_detector.py b/analytics/detectors/pattern_detector.py index 541ee4c..31ecec6 100644 --- a/analytics/detectors/pattern_detector.py +++ b/analytics/detectors/pattern_detector.py @@ -1,4 +1,4 @@ -import detectors +import models import utils from grafana_data_provider import GrafanaDataProvider @@ -16,13 +16,13 @@ logger = logging.getLogger('analytic_toolset') -def resolve_detector_by_pattern(pattern): +def resolve_model_by_pattern(pattern: str) -> models.Model: if pattern == 'peak': - return detectors.PeaksDetector() + return models.PeaksModel() if pattern == 'drop': - return detectors.StepDetector() + return models.StepModel() if pattern == 'jump': - return detectors.JumpDetector() + return models.JumpModel() raise ValueError('Unknown pattern "%s"' % pattern) @@ -53,7 +53,7 @@ class PatternDetector: self.__load_model(pattern_type) async def learn(self, segments): - self.model = resolve_detector_by_pattern(self.pattern_type) + self.model = resolve_model_by_pattern(self.pattern_type) window_size = 200 dataframe = self.data_prov.get_dataframe() @@ -109,5 +109,5 @@ class PatternDetector: logger.info("Load model '%s'" % self.analytic_unit_id) model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_type + ".m") if os.path.exists(model_filename): - self.model = resolve_detector_by_pattern(pattern) + self.model = resolve_model_by_pattern(pattern) self.model.load(model_filename) diff --git a/analytics/models/__init__.py b/analytics/models/__init__.py new file mode 100644 index 0000000..7989a6c --- /dev/null +++ b/analytics/models/__init__.py @@ -0,0 +1,4 @@ +from models.model import Model +from models.step_model import StepModel +from models.peaks_model import PeaksModel +from models.jump_model import JumpModel diff --git a/analytics/detectors/jump_detector.py b/analytics/models/jump_model.py similarity index 83% rename from analytics/detectors/jump_detector.py rename to analytics/models/jump_model.py index 93672f3..c0007ff 100644 --- a/analytics/detectors/jump_detector.py +++ b/analytics/models/jump_model.py @@ -1,22 +1,26 @@ +from models import Model + import utils import numpy as np -import pickle import scipy.signal from scipy.fftpack import fft from scipy.signal import argrelextrema import math + WINDOW_SIZE = 120 -class JumpDetector: +class JumpModel(Model): def __init__(self): - self.segments = [] - self.confidence = 1.5 - self.convolve_max = WINDOW_SIZE - self.size = 50 + super() + self.state = { + 'confidence': 1.5, + 'convolve_max': WINDOW_SIZE + } async def fit(self, dataframe, segments): + self.segments = segments #self.alpha_finder() data = dataframe['value'] confidences = [] @@ -59,14 +63,14 @@ class JumpDetector: if len(confidences) > 0: - self.confidence = min(confidences) + self.state['confidence'] = min(confidences) else: - self.confidence = 1.5 + self.state['confidence'] = 1.5 if len(convolve_list) > 0: - self.convolve_max = max(convolve_list) + self.state['convolve_max'] = max(convolve_list) else: - self.convolve_max = WINDOW_SIZE # макс метрика свертки равна отступу(WINDOW_SIZE), вау! + self.state['convolve_max'] = WINDOW_SIZE # макс метрика свертки равна отступу(WINDOW_SIZE), вау! async def predict(self, dataframe): data = dataframe['value'] @@ -82,10 +86,10 @@ class JumpDetector: window_size = 24 all_max_flatten_data = data.rolling(window=window_size).mean() all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0] - possible_jumps = utils.find_all_jumps(all_max_flatten_data, 50, self.confidence) + possible_jumps = utils.find_all_jumps(all_max_flatten_data, 50, self.state['confidence']) ''' - for i in utils.exponential_smoothing(data + self.confidence, 0.02): + for i in utils.exponential_smoothing(data + self.state['confidence'], 0.02): extrema_list.append(i) segments = [] @@ -117,24 +121,13 @@ class JumpDetector: for segment in segments: convol_data = all_max_flatten_data[segment - WINDOW_SIZE : segment + WINDOW_SIZE] conv = scipy.signal.fftconvolve(pattern_data, convol_data) - if max(conv) > self.convolve_max * 1.1 or max(conv) < self.convolve_max * 0.9: + if max(conv) > self.state['convolve_max'] * 1.1 or max(conv) < self.state['convolve_max'] * 0.9: delete_list.append(segment) for item in delete_list: segments.remove(item) return segments - def save(self, model_filename): - with open(model_filename, 'wb') as file: - pickle.dump((self.confidence, self.convolve_max), file) - - def load(self, model_filename): - try: - with open(model_filename, 'rb') as file: - (self.confidence, self.convolve_max) = pickle.load(file) - except: - pass - def alpha_finder(self, data): """ поиск альфы для логистической сигмоиды diff --git a/analytics/models/model.py b/analytics/models/model.py new file mode 100644 index 0000000..74adfae --- /dev/null +++ b/analytics/models/model.py @@ -0,0 +1,30 @@ +from abc import ABC, abstractmethod +from pandas import DataFrame +import pickle + +class Model(ABC): + + def __init__(self): + """ + Variables which are obtained as a result of fit() method + should be stored in self.state dict + in order to be saved in model file + """ + self.state = {} + self.segments = [] + + @abstractmethod + async def fit(self, dataframe: DataFrame, segments: list): + pass + + @abstractmethod + async def predict(self, dataframe: DataFrame) -> list: + pass + + def save(self, model_filename: str): + with open(model_filename, 'wb') as file: + pickle.dump(self.state, file) + + def load(self, model_filename: str): + with open(model_filename, 'rb') as f: + self.state = pickle.load(f) diff --git a/analytics/detectors/peaks_detector.py b/analytics/models/peaks_model.py similarity index 82% rename from analytics/detectors/peaks_detector.py rename to analytics/models/peaks_model.py index 2c4d497..9bcec9c 100644 --- a/analytics/detectors/peaks_detector.py +++ b/analytics/models/peaks_model.py @@ -1,11 +1,14 @@ +from models import Model + import utils from scipy import signal import numpy as np -class PeaksDetector: +class PeaksModel(Model): + def __init__(self): - pass + super() async def fit(self, dataset, contamination=0.005): pass @@ -53,13 +56,3 @@ class PeaksDetector: result = utils.find_steps(data, 0.1) return [(dataframe.index[x], dataframe.index[x + window_size]) for x in result] - - def save(self, model_filename): - pass - # with open(model_filename, 'wb') as file: - # pickle.dump((self.clf, self.scaler), file) - - def load(self, model_filename): - pass - # with open(model_filename, 'rb') as file: - # self.clf, self.scaler = pickle.load(file) diff --git a/analytics/detectors/step_detector.py b/analytics/models/step_model.py similarity index 78% rename from analytics/detectors/step_detector.py rename to analytics/models/step_model.py index 7a2ed44..69bd879 100644 --- a/analytics/detectors/step_detector.py +++ b/analytics/models/step_model.py @@ -1,3 +1,5 @@ +from models import Model + import scipy.signal from scipy.fftpack import fft from scipy.signal import argrelextrema @@ -7,14 +9,18 @@ import numpy as np import pickle -class StepDetector: +class StepModel(Model): def __init__(self): + super() self.segments = [] - self.confidence = 1.5 - self.convolve_max = 570000 + self.state = { + 'confidence': 1.5, + 'convolve_max': 570000 + } async def fit(self, dataframe, segments): + self.segments = segments data = dataframe['value'] confidences = [] convolve_list = [] @@ -32,14 +38,14 @@ class StepDetector: convolve_list.append(max(convolve)) if len(confidences) > 0: - self.confidence = min(confidences) + self.state['confidence'] = min(confidences) else: - self.confidence = 1.5 + self.state['confidence'] = 1.5 if len(convolve_list) > 0: - self.convolve_max = max(convolve_list) + self.state['convolve_max'] = max(convolve_list) else: - self.convolve_max = 570000 + self.state['convolve_max'] = 570000 async def predict(self, dataframe): data = dataframe['value'] @@ -57,7 +63,7 @@ class StepDetector: all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0] extrema_list = [] - for i in utils.exponential_smoothing(data - self.confidence, 0.03): + for i in utils.exponential_smoothing(data - self.state['confidence'], 0.03): extrema_list.append(i) segments = [] @@ -83,20 +89,9 @@ class StepDetector: for segment in segments: convol_data = all_max_flatten_data[segment - 120 : segment + 120] conv = scipy.signal.fftconvolve(pattern_data, convol_data) - if max(conv) > self.convolve_max * 1.1 or max(conv) < self.convolve_max * 0.9: + if max(conv) > self.state['convolve_max'] * 1.1 or max(conv) < self.state['convolve_max'] * 0.9: delete_list.append(segment) for item in delete_list: segments.remove(item) return segments - - def save(self, model_filename): - with open(model_filename, 'wb') as file: - pickle.dump((self.confidence, self.convolve_max), file) - - def load(self, model_filename): - try: - with open(model_filename, 'rb') as file: - (self.confidence, self.convolve_max) = pickle.load(file) - except: - pass