Browse Source

Move data cropping to the models #335 (#336)

pull/1/head
Alexandr Velikiy 6 years ago committed by rozetko
parent
commit
5ab3ff64dd
  1. 3
      analytics/analytics/analytic_unit_manager.py
  2. 6
      analytics/analytics/models/drop_model.py
  3. 6
      analytics/analytics/models/general_model.py
  4. 6
      analytics/analytics/models/jump_model.py
  5. 6
      analytics/analytics/models/peak_model.py
  6. 6
      analytics/analytics/models/trough_model.py
  7. 6
      analytics/analytics/utils/common.py
  8. 8
      analytics/tests/test_dataset.py

3
analytics/analytics/analytic_unit_manager.py

@ -33,9 +33,6 @@ def prepare_data(data: list):
data = pd.DataFrame(data, columns=['timestamp', 'value']) data = pd.DataFrame(data, columns=['timestamp', 'value'])
data['timestamp'] = pd.to_datetime(data['timestamp'], unit='ms') data['timestamp'] = pd.to_datetime(data['timestamp'], unit='ms')
data.fillna(value = np.nan, inplace = True) data.fillna(value = np.nan, inplace = True)
if not np.isnan(data['value'].min()):
data['value'] = data['value'] - min(data['value'])
return data return data

6
analytics/analytics/models/drop_model.py

@ -28,7 +28,8 @@ class DropModel(Model):
} }
def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None: def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None:
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
confidences = [] confidences = []
convolve_list = [] convolve_list = []
drop_height_list = [] drop_height_list = []
@ -101,7 +102,8 @@ class DropModel(Model):
self.state['conv_del_max'] = self.state['WINDOW_SIZE'] self.state['conv_del_max'] = self.state['WINDOW_SIZE']
def do_detect(self, dataframe: pd.DataFrame) -> list: def do_detect(self, dataframe: pd.DataFrame) -> list:
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
possible_drops = utils.find_drop(data, self.state['DROP_HEIGHT'], self.state['DROP_LENGTH'] + 1) possible_drops = utils.find_drop(data, self.state['DROP_HEIGHT'], self.state['DROP_LENGTH'] + 1)
return self.__filter_detection(possible_drops, data) return self.__filter_detection(possible_drops, data)

6
analytics/analytics/models/general_model.py

@ -28,7 +28,8 @@ class GeneralModel(Model):
self.all_conv = [] self.all_conv = []
def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None: def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None:
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
convolve_list = [] convolve_list = []
patterns_list = [] patterns_list = []
for segment in segments: for segment in segments:
@ -79,7 +80,8 @@ class GeneralModel(Model):
self.state['conv_del_max'] = self.state['WINDOW_SIZE'] self.state['conv_del_max'] = self.state['WINDOW_SIZE']
def do_detect(self, dataframe: pd.DataFrame) -> list: def do_detect(self, dataframe: pd.DataFrame) -> list:
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
pat_data = self.model_gen pat_data = self.model_gen
y = max(pat_data) y = max(pat_data)

6
analytics/analytics/models/jump_model.py

@ -29,7 +29,8 @@ class JumpModel(Model):
} }
def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None: def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None:
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
confidences = [] confidences = []
convolve_list = [] convolve_list = []
jump_height_list = [] jump_height_list = []
@ -102,7 +103,8 @@ class JumpModel(Model):
self.state['conv_del_max'] = self.state['WINDOW_SIZE'] self.state['conv_del_max'] = self.state['WINDOW_SIZE']
def do_detect(self, dataframe: pd.DataFrame) -> list: def do_detect(self, dataframe: pd.DataFrame) -> list:
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
possible_jumps = utils.find_jump(data, self.state['JUMP_HEIGHT'], self.state['JUMP_LENGTH'] + 1) possible_jumps = utils.find_jump(data, self.state['JUMP_HEIGHT'], self.state['JUMP_LENGTH'] + 1)
return self.__filter_detection(possible_jumps, data) return self.__filter_detection(possible_jumps, data)

6
analytics/analytics/models/peak_model.py

@ -28,7 +28,8 @@ class PeakModel(Model):
} }
def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None: def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None:
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
confidences = [] confidences = []
convolve_list = [] convolve_list = []
patterns_list = [] patterns_list = []
@ -87,7 +88,8 @@ class PeakModel(Model):
self.state['conv_del_max'] = self.state['WINDOW_SIZE'] self.state['conv_del_max'] = self.state['WINDOW_SIZE']
def do_detect(self, dataframe: pd.DataFrame): def do_detect(self, dataframe: pd.DataFrame):
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
window_size = int(len(data)/SMOOTHING_COEFF) #test ws on flat data window_size = int(len(data)/SMOOTHING_COEFF) #test ws on flat data
all_maxs = argrelextrema(np.array(data), np.greater)[0] all_maxs = argrelextrema(np.array(data), np.greater)[0]

6
analytics/analytics/models/trough_model.py

@ -28,7 +28,8 @@ class TroughModel(Model):
} }
def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None: def do_fit(self, dataframe: pd.DataFrame, segments: list) -> None:
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
confidences = [] confidences = []
convolve_list = [] convolve_list = []
patterns_list = [] patterns_list = []
@ -88,7 +89,8 @@ class TroughModel(Model):
self.state['conv_del_max'] = self.state['WINDOW_SIZE'] self.state['conv_del_max'] = self.state['WINDOW_SIZE']
def do_detect(self, dataframe: pd.DataFrame): def do_detect(self, dataframe: pd.DataFrame):
data = dataframe['value'] data = utils.cut_dataframe(dataframe)
data = data['value']
window_size = int(len(data)/SMOOTHING_COEFF) #test ws on flat data window_size = int(len(data)/SMOOTHING_COEFF) #test ws on flat data
all_mins = argrelextrema(np.array(data), np.less)[0] all_mins = argrelextrema(np.array(data), np.less)[0]

6
analytics/analytics/utils/common.py

@ -273,4 +273,8 @@ def pattern_intersection(segment_data: list, median: float, pattern_type: str) -
return [x for (idx, x) in enumerate(center_index) if idx not in delete_index] return [x for (idx, x) in enumerate(center_index) if idx not in delete_index]
def cut_dataframe(data: pd.DataFrame) -> pd.DataFrame:
data_min = data['value'].min()
if not np.isnan(data_min) and data_min > 0:
data['value'] = data['value'] - data_min
return data

8
analytics/tests/test_dataset.py

@ -121,10 +121,10 @@ class TestDataset(unittest.TestCase):
data_none = [[1523889000000, None], [1523889000001, None], [1523889000002, None]] data_none = [[1523889000000, None], [1523889000001, None], [1523889000002, None]]
return_data_nan = prepare_data(data_nan) return_data_nan = prepare_data(data_nan)
return_data_none = prepare_data(data_none) return_data_none = prepare_data(data_none)
for item in return_data_nan: for item in return_data_nan.value:
self.assertTrue(np.isnan(item.value)) self.assertTrue(np.isnan(item))
for item in return_data_none: for item in return_data_none.value:
self.assertTrue(np.isnan(item.value)) self.assertTrue(np.isnan(item))
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()

Loading…
Cancel
Save