hastic
/
hastic-server
4
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity
Browse Source

Improve all models

pull/1/head
rozetko 6 years ago
parent
8c80148ed3
commit
d1ca327b0d
9 changed files with 56 additions and 72 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      analytics/models/custom_model.py
  2. 13
      analytics/models/drop_model.py
  3. 23
      analytics/models/general_model.py
  4. 17
      analytics/models/jump_model.py
  5. 5
      analytics/models/model.py
  6. 20
      analytics/models/peak_model.py
  7. 20
      analytics/models/trough_model.py
  8. 8
      analytics/utils/__init__.py
  9. 20
      server/src/controllers/analytics_controller.ts

2
analytics/models/custom_model.py
Unescape View File

@ -15,5 +15,5 @@ class CustomModel(Model):
def fit(self, dataframe: pd.DataFrame, segments: list, cache: Optional[dict]) -> dict: def fit(self, dataframe: pd.DataFrame, segments: list, cache: Optional[dict]) -> dict:
pass pass
def predict(self, dataframe, cache: Optional[dict]): def do_predict(self, dataframe: pd.DataFrame) -> list:
return [] return []

13
analytics/models/drop_model.py
Unescape View File

@ -10,7 +10,7 @@ import numpy as np
import pandas as pd import pandas as pd
from typing import Optional from typing import Optional
WINDOW_SIZE = 300 WINDOW_SIZE = 200
class DropModel(Model): class DropModel(Model):
def __init__(self): def __init__(self):
@ -98,7 +98,7 @@ class DropModel(Model):
return self.state return self.state
def do_predict(self, dataframe: pd.DataFrame): def do_predict(self, dataframe: pd.DataFrame) -> list:
data = dataframe['value'] data = dataframe['value']
possible_drops = utils.find_drop(data, self.state['DROP_HEIGHT'], self.state['DROP_LENGTH'] + 1) possible_drops = utils.find_drop(data, self.state['DROP_HEIGHT'], self.state['DROP_LENGTH'] + 1)
@ -115,7 +115,7 @@ class DropModel(Model):
for i in range(1, len(segments)): for i in range(1, len(segments)):
if segments[i] < segments[i - 1] + variance_error: if segments[i] < segments[i - 1] + variance_error:
delete_list.append(segments[i]) delete_list.append(segments[i])
#for item in delete_list: # for item in delete_list:
# segments.remove(item) # segments.remove(item)
delete_list = [] delete_list = []
@ -131,7 +131,8 @@ class DropModel(Model):
delete_list.append(segment) delete_list.append(segment)
else: else:
delete_list.append(segment) delete_list.append(segment)
for item in delete_list: # TODO: implement filtering
segments.remove(item) # for item in delete_list:
# segments.remove(item)
return segments return set(segments)

23
analytics/models/general_model.py
Unescape View File

@ -12,7 +12,7 @@ from scipy.stats import norm
from typing import Optional from typing import Optional
WINDOW_SIZE = 350 WINDOW_SIZE = 150
class GeneralModel(Model): class GeneralModel(Model):
@ -41,12 +41,11 @@ class GeneralModel(Model):
segment_data = data[segment_from_index: segment_to_index + 1] segment_data = data[segment_from_index: segment_to_index + 1]
if len(segment_data) == 0: if len(segment_data) == 0:
continue continue
self.ipats.append(segment_from_index + int((segment_to_index - segment_from_index) / 2)) x = segment_from_index + int((segment_to_index - segment_from_index) / 2)
self.ipats.append(x)
segment_data = data[x - WINDOW_SIZE : x + WINDOW_SIZE]
segment_min = min(segment_data) segment_min = min(segment_data)
segment_data = segment_data - segment_min segment_data = segment_data - segment_min
segment_max = max(segment_data)
segment_data = segment_data / segment_max
convolve = scipy.signal.fftconvolve(segment_data, segment_data) convolve = scipy.signal.fftconvolve(segment_data, segment_data)
convolve_list.append(max(convolve)) convolve_list.append(max(convolve))
@ -57,35 +56,29 @@ class GeneralModel(Model):
return self.state return self.state
def do_predict(self, dataframe: pd.DataFrame): def do_predict(self, dataframe: pd.DataFrame) -> list:
data = dataframe['value'] data = dataframe['value']
pat_data = data[self.ipats[0] - WINDOW_SIZE: self.ipats[0] + WINDOW_SIZE] pat_data = data[self.ipats[0] - WINDOW_SIZE: self.ipats[0] + WINDOW_SIZE]
x = min(pat_data) x = min(pat_data)
pat_data = pat_data - x pat_data = pat_data - x
y = max(pat_data) y = max(pat_data)
pat_data = pat_data / y
for i in range(WINDOW_SIZE * 2, len(data)): for i in range(WINDOW_SIZE * 2, len(data)):
watch_data = data[i - WINDOW_SIZE * 2: i] watch_data = data[i - WINDOW_SIZE * 2: i]
w = min(watch_data) w = min(watch_data)
watch_data = watch_data - w watch_data = watch_data - w
r = max(watch_data)
if r < y:
watch_data = watch_data / y
else:
watch_data = watch_data / r
conv = scipy.signal.fftconvolve(pat_data, watch_data) conv = scipy.signal.fftconvolve(pat_data, watch_data)
self.all_conv.append(max(conv)) self.all_conv.append(max(conv))
all_conv_peaks = utils.peak_finder(self.all_conv, WINDOW_SIZE * 2) all_conv_peaks = utils.peak_finder(self.all_conv, WINDOW_SIZE * 2)
filtered = self.__filter_prediction(all_conv_peaks, data) filtered = self.__filter_prediction(all_conv_peaks, data)
filtered = set(item + WINDOW_SIZE for item in filtered)
# TODO: convert from ns to ms more proper way (not dividing by 10^6) # TODO: convert from ns to ms more proper way (not dividing by 10^6)
return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered] return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered]
def __filter_prediction(self, segments: list, data: list): def __filter_prediction(self, segments: list, data: list):
if len(segments) == 0 or len(self.ipats) == 0: if len(segments) == 0 or len(self.ipats) == 0:
segments = [] return []
return segments
delete_list = [] delete_list = []
for val in segments: for val in segments:
@ -95,4 +88,4 @@ class GeneralModel(Model):
for item in delete_list: for item in delete_list:
segments.remove(item) segments.remove(item)
return segments return set(segments)

17
analytics/models/jump_model.py
Unescape View File

@ -12,7 +12,7 @@ from scipy.stats import norm
from typing import Optional from typing import Optional
WINDOW_SIZE = 300 WINDOW_SIZE = 200
class JumpModel(Model): class JumpModel(Model):
@ -68,16 +68,16 @@ class JumpModel(Model):
jump_height_list.append(jump_height) jump_height_list.append(jump_height)
jump_length = utils.find_jump_length(segment_data, segment_min_line, segment_max_line) jump_length = utils.find_jump_length(segment_data, segment_min_line, segment_max_line)
jump_length_list.append(jump_length) jump_length_list.append(jump_length)
cen_ind = utils.intersection_segment(flat_segment, segment_median) #finds all interseprions with median cen_ind = utils.intersection_segment(flat_segment.tolist(), segment_median) #finds all interseprions with median
#cen_ind = utils.find_ind_median(segment_median, flat_segment) #cen_ind = utils.find_ind_median(segment_median, flat_segment)
jump_center = cen_ind[0] jump_center = cen_ind[0]
segment_cent_index = jump_center - 5 + segment_from_index segment_cent_index = jump_center - 5 + segment_from_index
self.ijumps.append(segment_cent_index) self.ijumps.append(segment_cent_index)
labeled_drop = data[segment_cent_index - WINDOW_SIZE : segment_cent_index + WINDOW_SIZE] labeled_jump = data[segment_cent_index - WINDOW_SIZE : segment_cent_index + WINDOW_SIZE]
labeled_min = min(labeled_drop) labeled_min = min(labeled_jump)
for value in labeled_drop: for value in labeled_jump:
value = value - labeled_min value = value - labeled_min
convolve = scipy.signal.fftconvolve(labeled_drop, labeled_drop) convolve = scipy.signal.fftconvolve(labeled_jump, labeled_jump)
convolve_list.append(max(convolve)) convolve_list.append(max(convolve))
if len(confidences) > 0: if len(confidences) > 0:
@ -102,7 +102,7 @@ class JumpModel(Model):
return self.state return self.state
def do_predict(self, dataframe: pd.DataFrame): def do_predict(self, dataframe: pd.DataFrame) -> list:
data = dataframe['value'] data = dataframe['value']
possible_jumps = utils.find_jump(data, self.state['JUMP_HEIGHT'], self.state['JUMP_LENGTH'] + 1) possible_jumps = utils.find_jump(data, self.state['JUMP_HEIGHT'], self.state['JUMP_LENGTH'] + 1)
@ -138,7 +138,8 @@ class JumpModel(Model):
for item in delete_list: for item in delete_list:
segments.remove(item) segments.remove(item)
# TODO: implement filtering
#for ijump in self.ijumps: #for ijump in self.ijumps:
#segments.append(ijump) #segments.append(ijump)
return segments return set(segments)

5
analytics/models/model.py
Unescape View File

@ -13,7 +13,7 @@ class Model(ABC):
pass pass
@abstractmethod @abstractmethod
def do_predict(self, dataframe: DataFrame): def do_predict(self, dataframe: DataFrame) -> list:
pass pass
def predict(self, dataframe: DataFrame, cache: Optional[AnalyticUnitCache]) -> dict: def predict(self, dataframe: DataFrame, cache: Optional[AnalyticUnitCache]) -> dict:
@ -21,10 +21,7 @@ class Model(ABC):
self.state = cache self.state = cache
result = self.do_predict(dataframe) result = self.do_predict(dataframe)
result.sort()
if len(self.segments) > 0:
result = [segment for segment in result if not utils.is_intersect(segment, self.segments)]
return { return {
'segments': result, 'segments': result,
'cache': self.state 'cache': self.state

20
analytics/models/peak_model.py
Unescape View File

@ -69,8 +69,7 @@ class PeakModel(Model):
def do_predict(self, dataframe: pd.DataFrame): def do_predict(self, dataframe: pd.DataFrame):
data = dataframe['value'] data = dataframe['value']
window_size = 24 window_size = 24
all_max_flatten_data = data.rolling(window=window_size).mean() all_maxs = argrelextrema(np.array(data), np.greater)[0]
all_maxs = argrelextrema(np.array(all_max_flatten_data), np.greater)[0]
extrema_list = [] extrema_list = []
for i in utils.exponential_smoothing(data + self.state['confidence'], 0.02): for i in utils.exponential_smoothing(data + self.state['confidence'], 0.02):
@ -78,16 +77,16 @@ class PeakModel(Model):
segments = [] segments = []
for i in all_maxs: for i in all_maxs:
if all_max_flatten_data[i] > extrema_list[i]: if data[i] > extrema_list[i]:
segments.append(i) segments.append(i)
filtered = self.__filter_prediction(segments, data) filtered = self.__filter_prediction(segments, data)
# TODO: convert from ns to ms more proper way (not dividing by 10^6) # TODO: convert from ns to ms more proper way (not dividing by 10^6)
return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered] return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered]
def __filter_prediction(self, segments: list, all_max_flatten_data: list): def __filter_prediction(self, segments: list, data: list) -> list:
delete_list = [] delete_list = []
variance_error = int(0.004 * len(all_max_flatten_data)) variance_error = int(0.004 * len(data))
if variance_error > 100: if variance_error > 100:
variance_error = 100 variance_error = 100
for i in range(1, len(segments)): for i in range(1, len(segments)):
@ -100,16 +99,17 @@ class PeakModel(Model):
if len(segments) == 0 or len(self.ipeaks) == 0: if len(segments) == 0 or len(self.ipeaks) == 0:
return [] return []
pattern_data = all_max_flatten_data[self.ipeaks[0] - WINDOW_SIZE: self.ipeaks[0] + WINDOW_SIZE] pattern_data = data[self.ipeaks[0] - WINDOW_SIZE: self.ipeaks[0] + WINDOW_SIZE]
for segment in segments: for segment in segments:
if segment > WINDOW_SIZE: if segment > WINDOW_SIZE:
convol_data = all_max_flatten_data[segment - WINDOW_SIZE: segment + WINDOW_SIZE] convol_data = data[segment - WINDOW_SIZE: segment + WINDOW_SIZE]
conv = scipy.signal.fftconvolve(pattern_data, convol_data) conv = scipy.signal.fftconvolve(pattern_data, convol_data)
if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_max'] * 0.8: if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_max'] * 0.8:
delete_list.append(segment) delete_list.append(segment)
else: else:
delete_list.append(segment) delete_list.append(segment)
for item in delete_list: # TODO: implement filtering
segments.remove(item) # for item in delete_list:
# segments.remove(item)
return segments return set(segments)

20
analytics/models/trough_model.py
Unescape View File

@ -68,8 +68,7 @@ class TroughModel(Model):
def do_predict(self, dataframe: pd.DataFrame): def do_predict(self, dataframe: pd.DataFrame):
data = dataframe['value'] data = dataframe['value']
window_size = 24 window_size = 24
all_max_flatten_data = data.rolling(window=window_size).mean() all_mins = argrelextrema(np.array(data), np.less)[0]
all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0]
extrema_list = [] extrema_list = []
for i in utils.exponential_smoothing(data - self.state['confidence'], 0.02): for i in utils.exponential_smoothing(data - self.state['confidence'], 0.02):
@ -77,16 +76,16 @@ class TroughModel(Model):
segments = [] segments = []
for i in all_mins: for i in all_mins:
if all_max_flatten_data[i] < extrema_list[i]: if data[i] < extrema_list[i]:
segments.append(i) segments.append(i)
test = dataframe['timestamp'][1].value test = dataframe['timestamp'][1].value
filtered = self.__filter_prediction(segments, data) filtered = self.__filter_prediction(segments, data)
# TODO: convert from ns to ms more proper way (not dividing by 10^6) # TODO: convert from ns to ms more proper way (not dividing by 10^6)
return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered] return [(dataframe['timestamp'][x - 1].value / 1000000, dataframe['timestamp'][x + 1].value / 1000000) for x in filtered]
def __filter_prediction(self, segments: list, all_max_flatten_data: list): def __filter_prediction(self, segments: list, data: list) -> list:
delete_list = [] delete_list = []
variance_error = int(0.004 * len(all_max_flatten_data)) variance_error = int(0.004 * len(data))
if variance_error > 100: if variance_error > 100:
variance_error = 100 variance_error = 100
for i in range(1, len(segments)): for i in range(1, len(segments)):
@ -100,16 +99,17 @@ class TroughModel(Model):
segments = [] segments = []
return segments return segments
pattern_data = all_max_flatten_data[self.ipeaks[0] - WINDOW_SIZE : self.ipeaks[0] + WINDOW_SIZE] pattern_data = data[self.ipeaks[0] - WINDOW_SIZE : self.ipeaks[0] + WINDOW_SIZE]
for segment in segments: for segment in segments:
if segment > WINDOW_SIZE: if segment > WINDOW_SIZE:
convol_data = all_max_flatten_data[segment - WINDOW_SIZE : segment + WINDOW_SIZE] convol_data = data[segment - WINDOW_SIZE : segment + WINDOW_SIZE]
conv = scipy.signal.fftconvolve(pattern_data, convol_data) conv = scipy.signal.fftconvolve(pattern_data, convol_data)
if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_max'] * 0.8: if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_max'] * 0.8:
delete_list.append(segment) delete_list.append(segment)
else: else:
delete_list.append(segment) delete_list.append(segment)
for item in delete_list: # TODO: implement filtering
segments.remove(item) # for item in delete_list:
# segments.remove(item)
return segments return set(segments)

8
analytics/utils/__init__.py
Unescape View File

@ -2,14 +2,6 @@ import numpy as np
import pandas as pd import pandas as pd
def is_intersect(target_segment, segments):
for segment in segments:
start = max(segment['from'], target_segment[0])
finish = min(segment['to'], target_segment[1])
if start <= finish:
return True
return False
def exponential_smoothing(series, alpha): def exponential_smoothing(series, alpha):
result = [series[0]] result = [series[0]]
for n in range(1, len(series)): for n in range(1, len(series)):

20
server/src/controllers/analytics_controller.ts
Unescape View File

@ -168,16 +168,16 @@ export async function runPredict(id: AnalyticUnit.AnalyticUnitId) {
let payload = processPredictionResult(id, result); let payload = processPredictionResult(id, result);
// Merging segments // TODO: implement segments merging without removing labeled
if(segments.length > 0 && payload.segments.length > 0) { // if(segments.length > 0 && payload.segments.length > 0) {
let lastOldSegment = segments[segments.length - 1]; // let lastOldSegment = segments[segments.length - 1];
let firstNewSegment = payload.segments[0]; // let firstNewSegment = payload.segments[0];
if(firstNewSegment.from <= lastOldSegment.to) { // if(firstNewSegment.from <= lastOldSegment.to) {
payload.segments[0].from = lastOldSegment.from; // payload.segments[0].from = lastOldSegment.from;
Segment.removeSegments([lastOldSegment.id]) // Segment.removeSegments([lastOldSegment.id])
} // }
} // }
Segment.insertSegments(payload.segments); Segment.insertSegments(payload.segments);
AnalyticUnitCache.setData(id, payload.cache); AnalyticUnitCache.setData(id, payload.cache);

Write Preview
Loading…
Cancel
Save