You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
121 lines
4.1 KiB
121 lines
4.1 KiB
![]()
6 years ago
|
from detectors.step_detector import StepDetector
|
||
|
from detectors.peaks_detector import PeaksDetector
|
||
|
|
||
![]()
6 years ago
|
from data_provider import DataProvider
|
||
![]()
6 years ago
|
|
||
![]()
6 years ago
|
import logging
|
||
6 years ago
|
from urllib.parse import urlparse
|
||
![]()
6 years ago
|
import os.path
|
||
|
import json
|
||
![]()
6 years ago
|
import config
|
||
|
|
||
![]()
6 years ago
|
import pandas as pd
|
||
|
|
||
![]()
6 years ago
|
|
||
![]()
6 years ago
|
logger = logging.getLogger('analytic_toolset')
|
||
|
|
||
|
|
||
|
def segments_box(segments):
|
||
|
max_time = 0
|
||
|
min_time = float("inf")
|
||
|
for segment in segments:
|
||
|
min_time = min(min_time, segment['start'])
|
||
|
max_time = max(max_time, segment['finish'])
|
||
|
min_time = pd.to_datetime(min_time, unit='ms')
|
||
|
max_time = pd.to_datetime(max_time, unit='ms')
|
||
|
return min_time, max_time
|
||
|
|
||
|
|
||
|
class PatternDetectionModel:
|
||
|
|
||
![]()
6 years ago
|
def __init__(self, analytic_unit_id, pattern_type):
|
||
![]()
6 years ago
|
self.analytic_unit_id = analytic_unit_id
|
||
![]()
6 years ago
|
self.pattern_type = pattern_type
|
||
![]()
6 years ago
|
|
||
|
self.__load_anomaly_config()
|
||
6 years ago
|
|
||
|
parsedUrl = urlparse(self.anomaly_config['panelUrl'])
|
||
|
origin = parsedUrl.scheme + '://' + parsedUrl.netloc
|
||
|
|
||
|
datasource = self.anomaly_config['datasource']
|
||
![]()
6 years ago
|
metric_name = self.anomaly_config['metric']['targets'][0]
|
||
|
|
||
![]()
6 years ago
|
target_filename = os.path.join(config.METRICS_FOLDER, metric_name + ".json")
|
||
6 years ago
|
datasource['origin'] = origin
|
||
![]()
6 years ago
|
dataset_filename = os.path.join(config.DATASET_FOLDER, metric_name + ".csv")
|
||
![]()
6 years ago
|
|
||
|
with open(target_filename, 'r') as file:
|
||
|
target = json.load(file)
|
||
|
|
||
6 years ago
|
self.data_prov = DataProvider(datasource, target, dataset_filename)
|
||
![]()
6 years ago
|
|
||
|
self.model = None
|
||
![]()
6 years ago
|
self.__load_model(pattern_type)
|
||
![]()
6 years ago
|
|
||
|
def learn(self, segments):
|
||
![]()
6 years ago
|
self.model = self.__create_model(self.pattern_type)
|
||
![]()
6 years ago
|
window_size = 200
|
||
|
|
||
|
dataframe = self.data_prov.get_dataframe()
|
||
|
|
||
|
segments = self.data_prov.transform_anomalies(segments)
|
||
![]()
6 years ago
|
# TODO: pass only part of dataframe that has segments
|
||
![]()
6 years ago
|
self.model.fit(dataframe, segments)
|
||
|
self.__save_model()
|
||
|
return 0
|
||
|
|
||
|
def predict(self, last_prediction_time):
|
||
|
if self.model is None:
|
||
|
return [], last_prediction_time
|
||
|
|
||
|
window_size = 100
|
||
|
last_prediction_time = pd.to_datetime(last_prediction_time, unit='ms')
|
||
|
|
||
|
start_index = self.data_prov.get_upper_bound(last_prediction_time)
|
||
|
start_index = max(0, start_index - window_size)
|
||
|
dataframe = self.data_prov.get_data_range(start_index)
|
||
|
|
||
|
predicted_indexes = self.model.predict(dataframe)
|
||
|
predicted_indexes = [(x, y) for (x, y) in predicted_indexes if x >= start_index and y >= start_index]
|
||
|
|
||
|
predicted_times = self.data_prov.inverse_transform_indexes(predicted_indexes)
|
||
|
segments = []
|
||
|
for time_value in predicted_times:
|
||
|
ts1 = int(time_value[0].timestamp() * 1000)
|
||
|
ts2 = int(time_value[1].timestamp() * 1000)
|
||
|
segments.append({
|
||
![]()
6 years ago
|
'start': min(ts1, ts2),
|
||
|
'finish': max(ts1, ts2)
|
||
![]()
6 years ago
|
})
|
||
|
|
||
|
last_dataframe_time = dataframe.iloc[- 1]['timestamp']
|
||
|
last_prediction_time = int(last_dataframe_time.timestamp() * 1000)
|
||
|
return segments, last_prediction_time
|
||
|
# return predicted_anomalies, last_prediction_time
|
||
|
|
||
|
def synchronize_data(self):
|
||
|
self.data_prov.synchronize()
|
||
|
|
||
6 years ago
|
def __create_model(self, pattern):
|
||
|
if pattern == "peaks":
|
||
![]()
6 years ago
|
return PeaksDetector()
|
||
6 years ago
|
if pattern == "jumps" or pattern == "drops":
|
||
|
return StepDetector(pattern)
|
||
![]()
6 years ago
|
raise ValueError('Unknown pattern "%s"' % pattern)
|
||
![]()
6 years ago
|
|
||
|
def __load_anomaly_config(self):
|
||
![]()
6 years ago
|
with open(os.path.join(config.ANALYTIC_UNITS_FOLDER, self.analytic_unit_id + ".json"), 'r') as config_file:
|
||
![]()
6 years ago
|
self.anomaly_config = json.load(config_file)
|
||
|
|
||
|
def __save_model(self):
|
||
![]()
6 years ago
|
logger.info("Save model '%s'" % self.analytic_unit_id)
|
||
|
model_filename = os.path.join(config.MODELS_FOLDER, self.analytic_unit_id + ".m")
|
||
![]()
6 years ago
|
self.model.save(model_filename)
|
||
|
|
||
6 years ago
|
def __load_model(self, pattern):
|
||
![]()
6 years ago
|
logger.info("Load model '%s'" % self.analytic_unit_id)
|
||
![]()
6 years ago
|
model_filename = os.path.join(config.MODELS_FOLDER, self.pattern_type + ".m")
|
||
![]()
6 years ago
|
if os.path.exists(model_filename):
|
||
6 years ago
|
self.model = self.__create_model(pattern)
|
||
|
self.model.load(model_filename)
|