hastic
/
hastic-server
4
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity
Browse Source

Jump detector v4

pull/1/head
Alexandr Velikiy 6 years ago committed by Alexey Velikiy
parent
15cdaf18b9
commit
5fcc7846d3
2 changed files with 52 additions and 46 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 60
      analytics/detectors/jump_detector.py
  2. 38
      analytics/utils/__init__.py

60
analytics/detectors/jump_detector.py
Unescape View File

@ -6,13 +6,15 @@ from scipy.fftpack import fft
from scipy.signal import argrelextrema from scipy.signal import argrelextrema
import math import math
WINDOW_SIZE = 120
class JumpDetector: class JumpDetector:
def __init__(self): def __init__(self):
self.segments = [] self.segments = []
self.confidence = 1.5 self.confidence = 1.5
self.convolve_max = 120 self.convolve_max = WINDOW_SIZE
self.size = 50
async def fit(self, dataframe, segments): async def fit(self, dataframe, segments):
#self.alpha_finder() #self.alpha_finder()
@ -27,36 +29,23 @@ class JumpDetector:
confidences.append(0.20 * (segment_max - segment_min)) confidences.append(0.20 * (segment_max - segment_min))
flat_segment = segment_data.rolling(window=4).mean() #сглаживаем сегмент flat_segment = segment_data.rolling(window=4).mean() #сглаживаем сегмент
kde_segment = flat_data.dropna().plot.kde() # distribution density kde_segment = flat_data.dropna().plot.kde() # distribution density
ax = flat_data.dropna().plot.kde() ax_list = kde_segment.get_lines()[0].get_xydata() #take coordinates of kde
ax_list = ax.get_lines()[0].get_xydata() mids = argrelextrema(np.array(ax_list), np.less)[0]
mids = argrelextrema(np.array(ax_list), np.less)[0]
maxs = argrelextrema(np.array(ax_list), np.greater)[0] maxs = argrelextrema(np.array(ax_list), np.greater)[0]
min_peak = maxs[0] min_peak = maxs[0]
max_peak = maxs[1] max_peak = maxs[1]
min_line = ax_list[min_peak, 0] min_line = ax_list[min_peak, 0]
max_line = ax_list[max_peak, 0] max_line = ax_list[max_peak, 0]
sigm_heidht = max_line - min_line sigm_heidht = max_line - min_line
pat_sigm = utils.logistic_sigmoid(-120, 120, 1, sigm_heidht) pat_sigm = utils.logistic_sigmoid(-WINDOW_SIZE, WINDOW_SIZE, 1, sigm_heidht)
for i in range(0, len(pat_sigm)): for i in range(0, len(pat_sigm)):
pat_sigm[i] = pat_sigm[i] + min_line pat_sigm[i] = pat_sigm[i] + min_line
cen_ind = utils.intersection_segment(flat_segment, mids[0]) cen_ind = utils.intersection_segment(flat_segment, mids[0]) #finds all interseprions with median
c = [] c = [] # choose the correct one interseption by convolve
for i in range(len(cen_ind)): jump_center = utils.find_jump_center(cen_ind)
x = cen_ind[i]
cx = scipy.signal.fftconvolve(pat_sigm, flat_data[x-120:x+120]) segment_cent_index = jump_center - 4
c.append(cx[240]) labeled_drop = data[segment_cent_index - WINDOW_SIZE : segment_cent_index + WINDOW_SIZE]
# в идеале нужно посмотреть гистограмму сегмента и выбрать среднее значение,
# далее от него брать + -120
segment_summ = 0
for val in flat_segment:
segment_summ += val
segment_mid = segment_summ / len(flat_segment) #посчитать нормально среднее значение/медиану
for ind in range(1, len(flat_segment) - 1):
if flat_segment[ind + 1] > segment_mid and flat_segment[ind - 1] < segment_mid:
flat_mid_index = ind # найти пересечение средней и графика, получить его индекс
segment_mid_index = flat_mid_index - 5
labeled_drop = data[segment_mid_index - 120 : segment_mid_index + 120]
labeled_min = min(labeled_drop) labeled_min = min(labeled_drop)
for value in labeled_drop: # обрезаем for value in labeled_drop: # обрезаем
value = value - labeled_min value = value - labeled_min
@ -65,7 +54,9 @@ class JumpDetector:
value = value / labeled_max value = value / labeled_max
convolve = scipy.signal.fftconvolve(labeled_drop, labeled_drop) convolve = scipy.signal.fftconvolve(labeled_drop, labeled_drop)
convolve_list.append(max(convolve)) # сворачиваем паттерн convolve_list.append(max(convolve)) # сворачиваем паттерн
# плюс надо впихнуть сюда логистическую сигмоиду и поиск альфы # TODO: add convolve with alpha sigmoid
# TODO: add size of jump rize
if len(confidences) > 0: if len(confidences) > 0:
self.confidence = min(confidences) self.confidence = min(confidences)
@ -75,7 +66,7 @@ class JumpDetector:
if len(convolve_list) > 0: if len(convolve_list) > 0:
self.convolve_max = max(convolve_list) self.convolve_max = max(convolve_list)
else: else:
self.convolve_max = 120 # макс метрика свертки равна отступу(120), вау! self.convolve_max = WINDOW_SIZE # макс метрика свертки равна отступу(WINDOW_SIZE), вау!
async def predict(self, dataframe): async def predict(self, dataframe):
data = dataframe['value'] data = dataframe['value']
@ -90,10 +81,10 @@ class JumpDetector:
async def __predict(self, data): async def __predict(self, data):
window_size = 24 window_size = 24
all_max_flatten_data = data.rolling(window=window_size).mean() all_max_flatten_data = data.rolling(window=window_size).mean()
all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0] all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0]
extrema_list = [] possible_jumps = utils.find_all_jumps(all_max_flatten_data, 50, self.confidence)
# добавить все пересечения экспоненты со сглаженным графиком
'''
for i in utils.exponential_smoothing(data + self.confidence, 0.02): for i in utils.exponential_smoothing(data + self.confidence, 0.02):
extrema_list.append(i) extrema_list.append(i)
@ -101,8 +92,9 @@ class JumpDetector:
for i in all_mins: for i in all_mins:
if all_max_flatten_data[i] > extrema_list[i]: if all_max_flatten_data[i] > extrema_list[i]:
segments.append(i - window_size) segments.append(i - window_size)
'''
return [(x - 1, x + 1) for x in self.__filter_prediction(segments, all_max_flatten_data)] return [(x - 1, x + 1) for x in self.__filter_prediction(possible_jumps, all_max_flatten_data)]
def __filter_prediction(self, segments, all_max_flatten_data): def __filter_prediction(self, segments, all_max_flatten_data):
delete_list = [] delete_list = []
@ -116,10 +108,14 @@ class JumpDetector:
segments.remove(item) segments.remove(item)
# изменить секонд делит лист, сделать для свертки с сигмоидой # изменить секонд делит лист, сделать для свертки с сигмоидой
# !!!!!!!!
# написать фильтрацию паттернов-джампов! посмотерть каждый сегмент, обрезать его
# отнормировать, сравнить с выбранным патерном.
# !!!!!!!!
delete_list = [] delete_list = []
pattern_data = all_max_flatten_data[segments[0] - 120 : segments[0] + 120] pattern_data = all_max_flatten_data[segments[0] - WINDOW_SIZE : segments[0] + WINDOW_SIZE]
for segment in segments: for segment in segments:
convol_data = all_max_flatten_data[segment - 120 : segment + 120] convol_data = all_max_flatten_data[segment - WINDOW_SIZE : segment + WINDOW_SIZE]
conv = scipy.signal.fftconvolve(pattern_data, convol_data) conv = scipy.signal.fftconvolve(pattern_data, convol_data)
if max(conv) > self.convolve_max * 1.1 or max(conv) < self.convolve_max * 0.9: if max(conv) > self.convolve_max * 1.1 or max(conv) < self.convolve_max * 0.9:
delete_list.append(segment) delete_list.append(segment)

38
analytics/utils/__init__.py
Unescape View File

@ -61,27 +61,27 @@ def segments_box(segments):
return min_time, max_time return min_time, max_time
def intersection_segment(data, median): def intersection_segment(data, median):
"""
Finds all intersections between flatten data and median
"""
cen_ind = [] cen_ind = []
for i in range(1, len(data)-1): for i in range(1, len(data)-1):
if data[i - 1] < median and data[i + 1] > median: if data[i - 1] < median and data[i + 1] > median:
cen_ind.append(i) cen_ind.append(i)
del_ind = [] del_ind = []
for i in range(1,len(cen_ind)): for i in range(1, len(cen_ind)):
if cen_ind[i] == cen_ind[i - 1] + 1: if cen_ind[i] == cen_ind[i - 1] + 1:
del_ind.append(i - 1) del_ind.append(i - 1)
del_ind = del_ind[::-1]
for i in del_ind: return [x for (idx, x) in enumerate(cen_ind) if idx not in del_ind]
del cen_ind[i]
return cen_ind
def logistic_sigmoid(self, x1, x2, alpha, height): def logistic_sigmoid_distribution(self, x1, x2, alpha, height):
distribution = [] return map(lambda x: logistic_sigmoid(x, alpha, height), range(x1, x2))
for i in range(x1, x2):
F = 1 * height / (1 + math.exp(-i * alpha))
distribution.append(F)
return distribution
def findOneJump(data, x, size, height, err): def logistic_sigmoid(x, alpha, height):
return height / (1 + math.exp(-x * alpha))
def find_one_jump(data, x, size, height, err):
l = [] l = []
for i in range(x + 1, x + size): for i in range(x + 1, x + size):
if (data[i] > data[x] and data[x + size] > data[x] + height): if (data[i] > data[x] and data[x + size] > data[x] + height):
@ -91,10 +91,20 @@ def findOneJump(data, x, size, height, err):
else: else:
return 0 return 0
def findAllJumps(data, size, height): def find_all_jumps(data, size, height):
possible_jump_list = [] possible_jump_list = []
for i in range(len(data - size)): for i in range(len(data - size)):
x = findOneJump(data, i, size, height, 0.9) x = find_one_jump(data, i, size, height, 0.9)
if x > 0: if x > 0:
possible_jump_list.append(x) possible_jump_list.append(x)
return possible_jump_list return possible_jump_list
def find_jump_center(cen_ind):
jump_center = cen_ind[0]
for i in range(len(cen_ind)):
x = cen_ind[i]
cx = scipy.signal.fftconvolve(pat_sigm, flat_data[x - WINDOW_SIZE : x + WINDOW_SIZE])
c.append(cx[2 * WINDOW_SIZE])
if i > 0 and cx > c[i - 1]:
jump_center = x
return jump_center

Write Preview
Loading…
Cancel
Save