Browse Source

Anti-segments in general model #142 (#185)

pull/1/head
Alexandr Velikiy 6 years ago committed by rozetko
parent
commit
b79abf5299
  1. 2
      analytics/models/drop_model.py
  2. 30
      analytics/models/general_model.py
  3. 2
      analytics/models/jump_model.py
  4. 2
      analytics/models/peak_model.py
  5. 2
      analytics/models/trough_model.py

2
analytics/models/drop_model.py

@ -172,7 +172,7 @@ class DropModel(Model):
conv = scipy.signal.fftconvolve(convol_data, pattern_data) conv = scipy.signal.fftconvolve(convol_data, pattern_data)
if conv[self.state['WINDOW_SIZE']*2] > self.state['convolve_max'] * 1.2 or conv[self.state['WINDOW_SIZE']*2] < self.state['convolve_min'] * 0.8: if conv[self.state['WINDOW_SIZE']*2] > self.state['convolve_max'] * 1.2 or conv[self.state['WINDOW_SIZE']*2] < self.state['convolve_min'] * 0.8:
delete_list.append(segment) delete_list.append(segment)
if max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98:
delete_list.append(segment) delete_list.append(segment)
else: else:
delete_list.append(segment) delete_list.append(segment)

30
analytics/models/general_model.py

@ -23,6 +23,8 @@ class GeneralModel(Model):
'convolve_max': 240, 'convolve_max': 240,
'convolve_min': 200, 'convolve_min': 200,
'WINDOW_SIZE': 240, 'WINDOW_SIZE': 240,
'conv_del_min': 100,
'conv_del_max': 120,
} }
self.all_conv = [] self.all_conv = []
@ -38,7 +40,7 @@ class GeneralModel(Model):
segment_data = data[segment_from_index: segment_to_index + 1] segment_data = data[segment_from_index: segment_to_index + 1]
if len(segment_data) == 0: if len(segment_data) == 0:
continue continue
x = segment_from_index + int((segment_to_index - segment_from_index) / 2) x = segment_from_index + math.ceil((segment_to_index - segment_from_index) / 2)
self.ipats.append(x) self.ipats.append(x)
segment_data = data[x - self.state['WINDOW_SIZE'] : x + self.state['WINDOW_SIZE']] segment_data = data[x - self.state['WINDOW_SIZE'] : x + self.state['WINDOW_SIZE']]
segment_min = min(segment_data) segment_min = min(segment_data)
@ -54,6 +56,20 @@ class GeneralModel(Model):
convolve_list.append(max(auto_convolve)) convolve_list.append(max(auto_convolve))
convolve_list.append(max(convolve_data)) convolve_list.append(max(convolve_data))
del_conv_list = []
for segment in segments:
if segment['deleted']:
segment_from_index = utils.timestamp_to_index(dataframe, pd.to_datetime(segment['from'], unit='ms'))
segment_to_index = utils.timestamp_to_index(dataframe, pd.to_datetime(segment['to'], unit='ms'))
segment_data = data[segment_from_index: segment_to_index + 1]
if len(segment_data) == 0:
continue
del_mid_index = segment_from_index + math.ceil((segment_to_index - segment_from_index) / 2)
deleted_pat = data[del_mid_index - self.state['WINDOW_SIZE']: del_mid_index + self.state['WINDOW_SIZE'] + 1]
deleted_pat = deleted_pat - min(deleted_pat)
del_conv_pat = scipy.signal.fftconvolve(deleted_pat, self.model_gen)
del_conv_list.append(max(del_conv_pat))
if len(convolve_list) > 0: if len(convolve_list) > 0:
self.state['convolve_max'] = float(max(convolve_list)) self.state['convolve_max'] = float(max(convolve_list))
else: else:
@ -64,6 +80,16 @@ class GeneralModel(Model):
else: else:
self.state['convolve_min'] = self.state['WINDOW_SIZE'] / 3 self.state['convolve_min'] = self.state['WINDOW_SIZE'] / 3
if len(del_conv_list) > 0:
self.state['conv_del_min'] = float(min(del_conv_list))
else:
self.state['conv_del_min'] = self.state['WINDOW_SIZE']
if len(del_conv_list) > 0:
self.state['conv_del_max'] = float(max(del_conv_list))
else:
self.state['conv_del_max'] = self.state['WINDOW_SIZE']
def do_predict(self, dataframe: pd.DataFrame) -> list: def do_predict(self, dataframe: pd.DataFrame) -> list:
data = dataframe['value'] data = dataframe['value']
pat_data = self.model_gen pat_data = self.model_gen
@ -88,6 +114,8 @@ class GeneralModel(Model):
for val in segments: for val in segments:
if self.all_conv[val] < self.state['convolve_min'] * 0.8: if self.all_conv[val] < self.state['convolve_min'] * 0.8:
delete_list.append(val) delete_list.append(val)
elif (self.all_conv[val] < self.state['conv_del_max'] * 1.02 and self.all_conv[val] > self.state['conv_del_min'] * 0.98):
delete_list.append(val)
for item in delete_list: for item in delete_list:
segments.remove(item) segments.remove(item)

2
analytics/models/jump_model.py

@ -173,7 +173,7 @@ class JumpModel(Model):
conv = scipy.signal.fftconvolve(convol_data, pattern_data) conv = scipy.signal.fftconvolve(convol_data, pattern_data)
if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_min'] * 0.8: if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_min'] * 0.8:
delete_list.append(segment) delete_list.append(segment)
if max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98:
delete_list.append(segment) delete_list.append(segment)
else: else:
delete_list.append(segment) delete_list.append(segment)

2
analytics/models/peak_model.py

@ -134,7 +134,7 @@ class PeakModel(Model):
conv = scipy.signal.fftconvolve(convol_data, pattern_data) conv = scipy.signal.fftconvolve(convol_data, pattern_data)
if max(conv) > self.state['convolve_max'] * 1.05 or max(conv) < self.state['convolve_min'] * 0.95: if max(conv) > self.state['convolve_max'] * 1.05 or max(conv) < self.state['convolve_min'] * 0.95:
delete_list.append(segment) delete_list.append(segment)
if max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98:
delete_list.append(segment) delete_list.append(segment)
else: else:
delete_list.append(segment) delete_list.append(segment)

2
analytics/models/trough_model.py

@ -136,7 +136,7 @@ class TroughModel(Model):
conv = scipy.signal.fftconvolve(convol_data, pattern_data) conv = scipy.signal.fftconvolve(convol_data, pattern_data)
if max(conv) > self.state['convolve_max'] * 1.1 or max(conv) < self.state['convolve_min'] * 0.9: if max(conv) > self.state['convolve_max'] * 1.1 or max(conv) < self.state['convolve_min'] * 0.9:
delete_list.append(segment) delete_list.append(segment)
if max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98:
delete_list.append(segment) delete_list.append(segment)
else: else:
delete_list.append(segment) delete_list.append(segment)

Loading…
Cancel
Save