hastic
/
hastic-server
4
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity
Browse Source

Jump detector v4

pull/1/head
Alexandr Velikiy 6 years ago committed by Alexey Velikiy
parent
15cdaf18b9
commit
5fcc7846d3
2 changed files with 52 additions and 46 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 54
      analytics/detectors/jump_detector.py
  2. 36
      analytics/utils/__init__.py

54
analytics/detectors/jump_detector.py
Unescape View File

@ -6,13 +6,15 @@ from scipy.fftpack import fft
from scipy.signal import argrelextrema
import math
WINDOW_SIZE = 120
class JumpDetector:
def __init__(self):
self.segments = []
self.confidence = 1.5
self.convolve_max = 120
self.convolve_max = WINDOW_SIZE
self.size = 50
async def fit(self, dataframe, segments):
#self.alpha_finder()
@ -27,8 +29,7 @@ class JumpDetector:
confidences.append(0.20 * (segment_max - segment_min))
flat_segment = segment_data.rolling(window=4).mean() #сглаживаем сегмент
kde_segment = flat_data.dropna().plot.kde() # distribution density
ax = flat_data.dropna().plot.kde()
ax_list = ax.get_lines()[0].get_xydata()
ax_list = kde_segment.get_lines()[0].get_xydata() #take coordinates of kde
mids = argrelextrema(np.array(ax_list), np.less)[0]
maxs = argrelextrema(np.array(ax_list), np.greater)[0]
min_peak = maxs[0]
@ -36,27 +37,15 @@ class JumpDetector:
min_line = ax_list[min_peak, 0]
max_line = ax_list[max_peak, 0]
sigm_heidht = max_line - min_line
pat_sigm = utils.logistic_sigmoid(-120, 120, 1, sigm_heidht)
pat_sigm = utils.logistic_sigmoid(-WINDOW_SIZE, WINDOW_SIZE, 1, sigm_heidht)
for i in range(0, len(pat_sigm)):
pat_sigm[i] = pat_sigm[i] + min_line
cen_ind = utils.intersection_segment(flat_segment, mids[0])
c = []
for i in range(len(cen_ind)):
x = cen_ind[i]
cx = scipy.signal.fftconvolve(pat_sigm, flat_data[x-120:x+120])
c.append(cx[240])
# в идеале нужно посмотреть гистограмму сегмента и выбрать среднее значение,
# далее от него брать + -120
segment_summ = 0
for val in flat_segment:
segment_summ += val
segment_mid = segment_summ / len(flat_segment) #посчитать нормально среднее значение/медиану
for ind in range(1, len(flat_segment) - 1):
if flat_segment[ind + 1] > segment_mid and flat_segment[ind - 1] < segment_mid:
flat_mid_index = ind # найти пересечение средней и графика, получить его индекс
segment_mid_index = flat_mid_index - 5
labeled_drop = data[segment_mid_index - 120 : segment_mid_index + 120]
cen_ind = utils.intersection_segment(flat_segment, mids[0]) #finds all interseprions with median
c = [] # choose the correct one interseption by convolve
jump_center = utils.find_jump_center(cen_ind)
segment_cent_index = jump_center - 4
labeled_drop = data[segment_cent_index - WINDOW_SIZE : segment_cent_index + WINDOW_SIZE]
labeled_min = min(labeled_drop)
for value in labeled_drop: # обрезаем
value = value - labeled_min
@ -65,7 +54,9 @@ class JumpDetector:
value = value / labeled_max
convolve = scipy.signal.fftconvolve(labeled_drop, labeled_drop)
convolve_list.append(max(convolve)) # сворачиваем паттерн
# плюс надо впихнуть сюда логистическую сигмоиду и поиск альфы
# TODO: add convolve with alpha sigmoid
# TODO: add size of jump rize
if len(confidences) > 0:
self.confidence = min(confidences)
@ -75,7 +66,7 @@ class JumpDetector:
if len(convolve_list) > 0:
self.convolve_max = max(convolve_list)
else:
self.convolve_max = 120 # макс метрика свертки равна отступу(120), вау!
self.convolve_max = WINDOW_SIZE # макс метрика свертки равна отступу(WINDOW_SIZE), вау!
async def predict(self, dataframe):
data = dataframe['value']
@ -91,9 +82,9 @@ class JumpDetector:
window_size = 24
all_max_flatten_data = data.rolling(window=window_size).mean()
all_mins = argrelextrema(np.array(all_max_flatten_data), np.less)[0]
extrema_list = []
# добавить все пересечения экспоненты со сглаженным графиком
possible_jumps = utils.find_all_jumps(all_max_flatten_data, 50, self.confidence)
'''
for i in utils.exponential_smoothing(data + self.confidence, 0.02):
extrema_list.append(i)
@ -101,8 +92,9 @@ class JumpDetector:
for i in all_mins:
if all_max_flatten_data[i] > extrema_list[i]:
segments.append(i - window_size)
'''
return [(x - 1, x + 1) for x in self.__filter_prediction(segments, all_max_flatten_data)]
return [(x - 1, x + 1) for x in self.__filter_prediction(possible_jumps, all_max_flatten_data)]
def __filter_prediction(self, segments, all_max_flatten_data):
delete_list = []
@ -116,10 +108,14 @@ class JumpDetector:
segments.remove(item)
# изменить секонд делит лист, сделать для свертки с сигмоидой
# !!!!!!!!
# написать фильтрацию паттернов-джампов! посмотерть каждый сегмент, обрезать его
# отнормировать, сравнить с выбранным патерном.
# !!!!!!!!
delete_list = []
pattern_data = all_max_flatten_data[segments[0] - 120 : segments[0] + 120]
pattern_data = all_max_flatten_data[segments[0] - WINDOW_SIZE : segments[0] + WINDOW_SIZE]
for segment in segments:
convol_data = all_max_flatten_data[segment - 120 : segment + 120]
convol_data = all_max_flatten_data[segment - WINDOW_SIZE : segment + WINDOW_SIZE]
conv = scipy.signal.fftconvolve(pattern_data, convol_data)
if max(conv) > self.convolve_max * 1.1 or max(conv) < self.convolve_max * 0.9:
delete_list.append(segment)

36
analytics/utils/__init__.py
Unescape View File

@ -61,6 +61,9 @@ def segments_box(segments):
return min_time, max_time
def intersection_segment(data, median):
"""
Finds all intersections between flatten data and median
"""
cen_ind = []
for i in range(1, len(data)-1):
if data[i - 1] < median and data[i + 1] > median:
@ -69,19 +72,16 @@ def intersection_segment(data, median):
for i in range(1, len(cen_ind)):
if cen_ind[i] == cen_ind[i - 1] + 1:
del_ind.append(i - 1)
del_ind = del_ind[::-1]
for i in del_ind:
del cen_ind[i]
return cen_ind
def logistic_sigmoid(self, x1, x2, alpha, height):
distribution = []
for i in range(x1, x2):
F = 1 * height / (1 + math.exp(-i * alpha))
distribution.append(F)
return distribution
return [x for (idx, x) in enumerate(cen_ind) if idx not in del_ind]
def logistic_sigmoid_distribution(self, x1, x2, alpha, height):
return map(lambda x: logistic_sigmoid(x, alpha, height), range(x1, x2))
def findOneJump(data, x, size, height, err):
def logistic_sigmoid(x, alpha, height):
return height / (1 + math.exp(-x * alpha))
def find_one_jump(data, x, size, height, err):
l = []
for i in range(x + 1, x + size):
if (data[i] > data[x] and data[x + size] > data[x] + height):
@ -91,10 +91,20 @@ def findOneJump(data, x, size, height, err):
else:
return 0
def findAllJumps(data, size, height):
def find_all_jumps(data, size, height):
possible_jump_list = []
for i in range(len(data - size)):
x = findOneJump(data, i, size, height, 0.9)
x = find_one_jump(data, i, size, height, 0.9)
if x > 0:
possible_jump_list.append(x)
return possible_jump_list
def find_jump_center(cen_ind):
jump_center = cen_ind[0]
for i in range(len(cen_ind)):
x = cen_ind[i]
cx = scipy.signal.fftconvolve(pat_sigm, flat_data[x - WINDOW_SIZE : x + WINDOW_SIZE])
c.append(cx[2 * WINDOW_SIZE])
if i > 0 and cx > c[i - 1]:
jump_center = x
return jump_center

Write Preview
Loading…
Cancel
Save