You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
152 lines
5.8 KiB
152 lines
5.8 KiB
![]()
6 years ago
|
from data_provider import DataProvider
|
||
|
from data_preprocessor import data_preprocessor
|
||
|
import pandas as pd
|
||
|
import logging
|
||
![]()
6 years ago
|
from urllib.parse import urlparse
|
||
![]()
6 years ago
|
import config
|
||
![]()
6 years ago
|
import os.path
|
||
|
import json
|
||
![]()
6 years ago
|
|
||
![]()
6 years ago
|
|
||
|
logger = logging.getLogger('analytic_toolset')
|
||
|
|
||
|
|
||
|
def anomalies_to_timestamp(anomalies):
|
||
|
for anomaly in anomalies:
|
||
|
anomaly['start'] = int(anomaly['start'].timestamp() * 1000)
|
||
|
anomaly['finish'] = int(anomaly['finish'].timestamp() * 1000)
|
||
|
return anomalies
|
||
|
|
||
|
|
||
![]()
6 years ago
|
class GeneralDetector:
|
||
![]()
6 years ago
|
|
||
|
def __init__(self, anomaly_name):
|
||
|
self.anomaly_name = anomaly_name
|
||
|
self.load_anomaly_config()
|
||
|
|
||
![]()
6 years ago
|
parsedUrl = urlparse(self.anomaly_config['panelUrl'])
|
||
|
origin = parsedUrl.scheme + '://' + parsedUrl.netloc
|
||
|
|
||
|
datasource = self.anomaly_config['datasource']
|
||
|
datasource['origin'] = origin
|
||
![]()
6 years ago
|
metric_name = self.anomaly_config['metric']['targets'][0]
|
||
|
|
||
![]()
6 years ago
|
target_filename = os.path.join(config.METRICS_FOLDER, metric_name + ".json")
|
||
![]()
6 years ago
|
|
||
![]()
6 years ago
|
dataset_filename = os.path.join(config.DATASET_FOLDER, metric_name + ".csv")
|
||
|
augmented_path = os.path.join(config.DATASET_FOLDER, metric_name + "_augmented.csv")
|
||
![]()
6 years ago
|
|
||
|
with open(target_filename, 'r') as file:
|
||
|
target = json.load(file)
|
||
|
|
||
![]()
6 years ago
|
self.data_prov = DataProvider(datasource, target, dataset_filename)
|
||
![]()
6 years ago
|
self.preprocessor = data_preprocessor(self.data_prov, augmented_path)
|
||
|
self.model = None
|
||
|
|
||
|
self.__load_model()
|
||
|
|
||
|
def anomalies_box(self, anomalies):
|
||
|
max_time = 0
|
||
|
min_time = float("inf")
|
||
|
for anomaly in anomalies:
|
||
|
max_time = max(max_time, anomaly['finish'])
|
||
|
min_time = min(min_time, anomaly['start'])
|
||
|
min_time = pd.to_datetime(min_time, unit='ms')
|
||
|
max_time = pd.to_datetime(max_time, unit='ms')
|
||
|
return min_time, max_time
|
||
|
|
||
|
def learn(self, anomalies):
|
||
|
logger.info("Start to learn for anomaly_name='%s'" % self.anomaly_name)
|
||
|
|
||
|
confidence = 0.02
|
||
|
dataframe = self.data_prov.get_dataframe()
|
||
|
start_index, stop_index = 0, len(dataframe)
|
||
|
if len(anomalies) > 0:
|
||
|
confidence = 0.0
|
||
|
min_time, max_time = self.anomalies_box(anomalies)
|
||
6 years ago
|
dataframe = dataframe[dataframe['timestamp'] <= max_time]
|
||
|
dataframe = dataframe[dataframe['timestamp'] >= min_time]
|
||
![]()
6 years ago
|
|
||
|
train_augmented = self.preprocessor.get_augmented_data(
|
||
6 years ago
|
dataframe.index[0],
|
||
|
dataframe.index[-1],
|
||
![]()
6 years ago
|
anomalies
|
||
|
)
|
||
|
|
||
|
self.model = self.create_algorithm()
|
||
|
self.model.fit(train_augmented, confidence)
|
||
|
if len(anomalies) > 0:
|
||
![]()
6 years ago
|
last_dataframe_time = dataframe.iloc[-1]['timestamp']
|
||
![]()
6 years ago
|
last_prediction_time = int(last_dataframe_time.timestamp() * 1000)
|
||
|
else:
|
||
|
last_prediction_time = 0
|
||
|
|
||
|
self.__save_model()
|
||
|
logger.info("Learning is finished for anomaly_name='%s'" % self.anomaly_name)
|
||
|
return last_prediction_time
|
||
|
|
||
|
def predict(self, last_prediction_time):
|
||
|
logger.info("Start to predict for anomaly type='%s'" % self.anomaly_name)
|
||
|
last_prediction_time = pd.to_datetime(last_prediction_time, unit='ms')
|
||
|
|
||
|
start_index = self.data_prov.get_upper_bound(last_prediction_time)
|
||
|
stop_index = self.data_prov.size()
|
||
|
|
||
|
last_prediction_time = int(last_prediction_time.timestamp() * 1000)
|
||
|
|
||
|
predicted_anomalies = []
|
||
|
if start_index < stop_index:
|
||
|
max_chunk_size = 50000
|
||
|
predicted = pd.Series()
|
||
|
for index in range(start_index, stop_index, max_chunk_size):
|
||
|
chunk_start = index
|
||
|
chunk_finish = min(index + max_chunk_size, stop_index)
|
||
|
predict_augmented = self.preprocessor.get_augmented_data(chunk_start, chunk_finish)
|
||
|
|
||
|
assert(len(predict_augmented) == chunk_finish - chunk_start)
|
||
|
|
||
|
predicted_current = self.model.predict(predict_augmented)
|
||
|
predicted = pd.concat([predicted, predicted_current])
|
||
|
predicted_anomalies = self.preprocessor.inverse_transform_anomalies(predicted)
|
||
|
|
||
|
last_row = self.data_prov.get_data_range(stop_index - 1, stop_index)
|
||
|
|
||
|
last_dataframe_time = last_row.iloc[0]['timestamp']
|
||
|
predicted_anomalies = anomalies_to_timestamp(predicted_anomalies)
|
||
|
last_prediction_time = int(last_dataframe_time.timestamp() * 1000)
|
||
|
|
||
|
logger.info("Predicting is finished for anomaly type='%s'" % self.anomaly_name)
|
||
|
return predicted_anomalies, last_prediction_time
|
||
|
|
||
|
def synchronize_data(self):
|
||
|
self.data_prov.synchronize()
|
||
|
self.preprocessor.set_data_provider(self.data_prov)
|
||
|
self.preprocessor.synchronize()
|
||
|
|
||
|
def load_anomaly_config(self):
|
||
![]()
6 years ago
|
with open(os.path.join(config.ANALYTIC_UNITS_FOLDER, self.anomaly_name + ".json"), 'r') as config_file:
|
||
![]()
6 years ago
|
self.anomaly_config = json.load(config_file)
|
||
|
|
||
|
def get_anomalies(self):
|
||
![]()
6 years ago
|
labeled_anomalies_file = os.path.join(config.ANALYTIC_UNITS_FOLDER, self.anomaly_name + "_labeled.json")
|
||
![]()
6 years ago
|
if not os.path.exists(labeled_anomalies_file):
|
||
|
return []
|
||
|
with open(labeled_anomalies_file) as file:
|
||
|
return json.load(file)
|
||
|
|
||
|
def create_algorithm(self):
|
||
|
from supervised_algorithm import supervised_algorithm
|
||
|
return supervised_algorithm()
|
||
|
|
||
|
def __save_model(self):
|
||
|
logger.info("Save model '%s'" % self.anomaly_name)
|
||
![]()
6 years ago
|
model_filename = os.path.join(config.MODELS_FOLDER, self.anomaly_name + ".m")
|
||
![]()
6 years ago
|
self.model.save(model_filename)
|
||
|
|
||
|
def __load_model(self):
|
||
|
logger.info("Load model '%s'" % self.anomaly_name)
|
||
![]()
6 years ago
|
model_filename = os.path.join(config.MODELS_FOLDER, self.anomaly_name + ".m")
|
||
![]()
6 years ago
|
if os.path.exists(model_filename):
|
||
|
self.model = self.create_algorithm()
|
||
|
self.model.load(model_filename)
|