diff --git a/analytics/models/custom_model.py b/analytics/models/custom_model.py index b0c78e7..f1005ef 100644 --- a/analytics/models/custom_model.py +++ b/analytics/models/custom_model.py @@ -15,5 +15,5 @@ class CustomModel(Model): def fit(self, dataframe: pd.DataFrame, segments: list, cache: Optional[dict]) -> dict: pass - def predict(self, dataframe, cache: Optional[dict]): + def do_predict(self, dataframe: pd.DataFrame) -> list: return [] diff --git a/analytics/models/drop_model.py b/analytics/models/drop_model.py index ca97d3a..d126abe 100644 --- a/analytics/models/drop_model.py +++ b/analytics/models/drop_model.py @@ -10,7 +10,7 @@ import numpy as np import pandas as pd from typing import Optional -WINDOW_SIZE = 300 +WINDOW_SIZE = 200 class DropModel(Model): def __init__(self): @@ -98,7 +98,7 @@ class DropModel(Model): return self.state - def do_predict(self, dataframe: pd.DataFrame): + def do_predict(self, dataframe: pd.DataFrame) -> list: data = dataframe['value'] possible_drops = utils.find_drop(data, self.state['DROP_HEIGHT'], self.state['DROP_LENGTH'] + 1) @@ -115,7 +115,7 @@ class DropModel(Model): for i in range(1, len(segments)): if segments[i] < segments[i - 1] + variance_error: delete_list.append(segments[i]) - #for item in delete_list: + # for item in delete_list: # segments.remove(item) delete_list = [] @@ -131,7 +131,8 @@ class DropModel(Model): delete_list.append(segment) else: delete_list.append(segment) - for item in delete_list: - segments.remove(item) + # TODO: implement filtering + # for item in delete_list: + # segments.remove(item) - return segments + return set(segments) diff --git a/analytics/models/general_model.py b/analytics/models/general_model.py index 42ba460..2719e00 100644 --- a/analytics/models/general_model.py +++ b/analytics/models/general_model.py @@ -12,7 +12,7 @@ from scipy.stats import norm from typing import Optional -WINDOW_SIZE = 350 +WINDOW_SIZE = 150 class GeneralModel(Model): @@ -41,12 +41,11 @@ class GeneralModel(Model): segment_data = data[segment_from_index: segment_to_index + 1] if len(segment_data) == 0: continue - self.ipats.append(segment_from_index + int((segment_to_index - segment_from_index) / 2)) + x = segment_from_index + int((segment_to_index - segment_from_index) / 2) + self.ipats.append(x) + segment_data = data[x - WINDOW_SIZE : x + WINDOW_SIZE] segment_min = min(segment_data) segment_data = segment_data - segment_min - segment_max = max(segment_data) - segment_data = segment_data / segment_max - convolve = scipy.signal.fftconvolve(segment_data, segment_data) convolve_list.append(max(convolve)) @@ -57,35 +56,29 @@ class GeneralModel(Model): return self.state - def do_predict(self, dataframe: pd.DataFrame): + def do_predict(self, dataframe: pd.DataFrame) -> list: data = dataframe['value'] pat_data = data[self.ipats[0] - WINDOW_SIZE: self.ipats[0] + WINDOW_SIZE] x = min(pat_data) pat_data = pat_data - x y = max(pat_data) - pat_data = pat_data / y for i in range(WINDOW_SIZE * 2, len(data)): watch_data = data[i - WINDOW_SIZE * 2: i] w = min(watch_data) watch_data = watch_data - w - r = max(watch_data) - if r < y: - watch_data = watch_data / y - else: - watch_data = watch_data / r conv = scipy.signal.fftconvolve(pat_data, watch_data) self.all_conv.append(max(conv)) all_conv_peaks = utils.peak_finder(self.all_conv, WINDOW_SIZE * 2) filtered = self.__filter_prediction(all_conv_peaks, data) + filtered = set(item + WINDOW_SIZE for item in filtered) # TODO: convert from ns to ms more proper way (not dividing by 10^6) return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered] def __filter_prediction(self, segments: list, data: list): if len(segments) == 0 or len(self.ipats) == 0: - segments = [] - return segments + return [] delete_list = [] for val in segments: @@ -95,4 +88,4 @@ class GeneralModel(Model): for item in delete_list: segments.remove(item) - return segments + return set(segments) diff --git a/analytics/models/jump_model.py b/analytics/models/jump_model.py index badf453..fcf4389 100644 --- a/analytics/models/jump_model.py +++ b/analytics/models/jump_model.py @@ -12,7 +12,7 @@ from scipy.stats import norm from typing import Optional -WINDOW_SIZE = 300 +WINDOW_SIZE = 200 class JumpModel(Model): @@ -68,16 +68,16 @@ class JumpModel(Model): jump_height_list.append(jump_height) jump_length = utils.find_jump_length(segment_data, segment_min_line, segment_max_line) jump_length_list.append(jump_length) - cen_ind = utils.intersection_segment(flat_segment, segment_median) #finds all interseprions with median + cen_ind = utils.intersection_segment(flat_segment.tolist(), segment_median) #finds all interseprions with median #cen_ind = utils.find_ind_median(segment_median, flat_segment) jump_center = cen_ind[0] segment_cent_index = jump_center - 5 + segment_from_index self.ijumps.append(segment_cent_index) - labeled_drop = data[segment_cent_index - WINDOW_SIZE : segment_cent_index + WINDOW_SIZE] - labeled_min = min(labeled_drop) - for value in labeled_drop: + labeled_jump = data[segment_cent_index - WINDOW_SIZE : segment_cent_index + WINDOW_SIZE] + labeled_min = min(labeled_jump) + for value in labeled_jump: value = value - labeled_min - convolve = scipy.signal.fftconvolve(labeled_drop, labeled_drop) + convolve = scipy.signal.fftconvolve(labeled_jump, labeled_jump) convolve_list.append(max(convolve)) if len(confidences) > 0: @@ -102,7 +102,7 @@ class JumpModel(Model): return self.state - def do_predict(self, dataframe: pd.DataFrame): + def do_predict(self, dataframe: pd.DataFrame) -> list: data = dataframe['value'] possible_jumps = utils.find_jump(data, self.state['JUMP_HEIGHT'], self.state['JUMP_LENGTH'] + 1) @@ -138,7 +138,8 @@ class JumpModel(Model): for item in delete_list: segments.remove(item) + # TODO: implement filtering #for ijump in self.ijumps: #segments.append(ijump) - return segments + return set(segments) diff --git a/analytics/models/model.py b/analytics/models/model.py index b37a4f6..f76b616 100644 --- a/analytics/models/model.py +++ b/analytics/models/model.py @@ -13,7 +13,7 @@ class Model(ABC): pass @abstractmethod - def do_predict(self, dataframe: DataFrame): + def do_predict(self, dataframe: DataFrame) -> list: pass def predict(self, dataframe: DataFrame, cache: Optional[AnalyticUnitCache]) -> dict: @@ -21,10 +21,7 @@ class Model(ABC): self.state = cache result = self.do_predict(dataframe) - result.sort() - if len(self.segments) > 0: - result = [segment for segment in result if not utils.is_intersect(segment, self.segments)] return { 'segments': result, 'cache': self.state diff --git a/analytics/models/peak_model.py b/analytics/models/peak_model.py index 799d86b..9b6e6d9 100644 --- a/analytics/models/peak_model.py +++ b/analytics/models/peak_model.py @@ -69,8 +69,7 @@ class PeakModel(Model): def do_predict(self, dataframe: pd.DataFrame): data = dataframe['value'] window_size = 24 - all_max_flatten_data = data.rolling(window=window_size).mean() - all_maxs = argrelextrema(np.array(all_max_flatten_data), np.greater)[0] + all_maxs = argrelextrema(np.array(data), np.greater)[0] extrema_list = [] for i in utils.exponential_smoothing(data + self.state['confidence'], 0.02): @@ -78,16 +77,16 @@ class PeakModel(Model): segments = [] for i in all_maxs: - if all_max_flatten_data[i] > extrema_list[i]: + if data[i] > extrema_list[i]: segments.append(i) filtered = self.__filter_prediction(segments, data) # TODO: convert from ns to ms more proper way (not dividing by 10^6) return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered] - def __filter_prediction(self, segments: list, all_max_flatten_data: list): + def __filter_prediction(self, segments: list, data: list) -> list: delete_list = [] - variance_error = int(0.004 * len(all_max_flatten_data)) + variance_error = int(0.004 * len(data)) if variance_error > 100: variance_error = 100 for i in range(1, len(segments)): @@ -100,16 +99,17 @@ class PeakModel(Model): if len(segments) == 0 or len(self.ipeaks) == 0: return [] - pattern_data = all_max_flatten_data[self.ipeaks[0] - WINDOW_SIZE: self.ipeaks[0] + WINDOW_SIZE] + pattern_data = data[self.ipeaks[0] - WINDOW_SIZE: self.ipeaks[0] + WINDOW_SIZE] for segment in segments: if segment > WINDOW_SIZE: - convol_data = all_max_flatten_data[segment - WINDOW_SIZE: segment + WINDOW_SIZE] + convol_data = data[segment - WINDOW_SIZE: segment + WINDOW_SIZE] conv = scipy.signal.fftconvolve(pattern_data, convol_data) if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_max'] * 0.8: delete_list.append(segment) else: delete_list.append(segment) - for item in delete_list: - segments.remove(item) + # TODO: implement filtering + # for item in delete_list: + # segments.remove(item) - return segments + return set(segments) diff --git a/analytics/models/trough_model.py b/analytics/models/trough_model.py index f270edd..201abcf 100644 --- a/analytics/models/trough_model.py +++ b/analytics/models/trough_model.py @@ -68,8 +68,7 @@ class TroughModel(Model): def do_predict(self, dataframe: pd.DataFrame): data = dataframe['value'] window_size = 24 - all_max_flatten_data = data.rolling(window=window_size).mean() - all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0] + all_mins = argrelextrema(np.array(data), np.less)[0] extrema_list = [] for i in utils.exponential_smoothing(data - self.state['confidence'], 0.02): @@ -77,16 +76,16 @@ class TroughModel(Model): segments = [] for i in all_mins: - if all_max_flatten_data[i] < extrema_list[i]: + if data[i] < extrema_list[i]: segments.append(i) test = dataframe['timestamp'][1].value filtered = self.__filter_prediction(segments, data) # TODO: convert from ns to ms more proper way (not dividing by 10^6) return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered] - def __filter_prediction(self, segments: list, all_max_flatten_data: list): + def __filter_prediction(self, segments: list, data: list) -> list: delete_list = [] - variance_error = int(0.004 * len(all_max_flatten_data)) + variance_error = int(0.004 * len(data)) if variance_error > 100: variance_error = 100 for i in range(1, len(segments)): @@ -100,16 +99,17 @@ class TroughModel(Model): segments = [] return segments - pattern_data = all_max_flatten_data[self.ipeaks[0] - WINDOW_SIZE : self.ipeaks[0] + WINDOW_SIZE] + pattern_data = data[self.ipeaks[0] - WINDOW_SIZE : self.ipeaks[0] + WINDOW_SIZE] for segment in segments: if segment > WINDOW_SIZE: - convol_data = all_max_flatten_data[segment - WINDOW_SIZE : segment + WINDOW_SIZE] + convol_data = data[segment - WINDOW_SIZE : segment + WINDOW_SIZE] conv = scipy.signal.fftconvolve(pattern_data, convol_data) if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_max'] * 0.8: delete_list.append(segment) else: delete_list.append(segment) - for item in delete_list: - segments.remove(item) + # TODO: implement filtering + # for item in delete_list: + # segments.remove(item) - return segments + return set(segments) diff --git a/analytics/utils/__init__.py b/analytics/utils/__init__.py index e24b87f..89e6ab5 100644 --- a/analytics/utils/__init__.py +++ b/analytics/utils/__init__.py @@ -2,14 +2,6 @@ import numpy as np import pandas as pd -def is_intersect(target_segment, segments): - for segment in segments: - start = max(segment['from'], target_segment[0]) - finish = min(segment['to'], target_segment[1]) - if start <= finish: - return True - return False - def exponential_smoothing(series, alpha): result = [series[0]] for n in range(1, len(series)): diff --git a/server/src/controllers/analytics_controller.ts b/server/src/controllers/analytics_controller.ts index 09b6abc..3681bc6 100644 --- a/server/src/controllers/analytics_controller.ts +++ b/server/src/controllers/analytics_controller.ts @@ -168,16 +168,16 @@ export async function runPredict(id: AnalyticUnit.AnalyticUnitId) { let payload = processPredictionResult(id, result); - // Merging segments - if(segments.length > 0 && payload.segments.length > 0) { - let lastOldSegment = segments[segments.length - 1]; - let firstNewSegment = payload.segments[0]; - - if(firstNewSegment.from <= lastOldSegment.to) { - payload.segments[0].from = lastOldSegment.from; - Segment.removeSegments([lastOldSegment.id]) - } - } + // TODO: implement segments merging without removing labeled + // if(segments.length > 0 && payload.segments.length > 0) { + // let lastOldSegment = segments[segments.length - 1]; + // let firstNewSegment = payload.segments[0]; + + // if(firstNewSegment.from <= lastOldSegment.to) { + // payload.segments[0].from = lastOldSegment.from; + // Segment.removeSegments([lastOldSegment.id]) + // } + // } Segment.insertSegments(payload.segments); AnalyticUnitCache.setData(id, payload.cache);