From 4912026113a252835f6491388c2e9d8c174aad04 Mon Sep 17 00:00:00 2001 From: Alexandr Velikiy <39257464+VargBurz@users.noreply.github.com> Date: Thu, 25 Oct 2018 11:50:42 +0300 Subject: [PATCH] Fix models for choosing the "wrong" patterns #195 (#202) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * jump and drop not crash * fix logiс of try/except block in drops left one block * fix logic of try/except in jump left one block --- analytics/.vscode/settings.json | 2 +- analytics/models/drop_model.py | 38 +++++++++++++++++++------------ analytics/models/jump_model.py | 40 ++++++++++++++++++++------------- analytics/models/peak_model.py | 1 + 4 files changed, 51 insertions(+), 30 deletions(-) diff --git a/analytics/.vscode/settings.json b/analytics/.vscode/settings.json index 0d3f9d2..5184a3b 100644 --- a/analytics/.vscode/settings.json +++ b/analytics/.vscode/settings.json @@ -8,4 +8,4 @@ "files.exclude": { "**/__pycache__/": true } -} \ No newline at end of file +} diff --git a/analytics/models/drop_model.py b/analytics/models/drop_model.py index 0c09858..130447f 100644 --- a/analytics/models/drop_model.py +++ b/analytics/models/drop_model.py @@ -47,19 +47,24 @@ class DropModel(Model): confidences.append(0.20 * (segment_max - segment_min)) flat_segment = segment_data.rolling(window = 5).mean() pdf = gaussian_kde(flat_segment.dropna()) + max_drop = max(flat_segment.dropna()) + min_drop = min(flat_segment.dropna()) x = np.linspace(flat_segment.dropna().min() - 1, flat_segment.dropna().max() + 1, len(flat_segment.dropna())) y = pdf(x) - ax_list = [] - for i in range(len(x)): - ax_list.append([x[i], y[i]]) + ax_list = list(zip(x, y)) ax_list = np.array(ax_list, np.float32) antipeaks_kde = argrelextrema(np.array(ax_list), np.less)[0] peaks_kde = argrelextrema(np.array(ax_list), np.greater)[0] - min_peak_index = peaks_kde[0] - max_peak_index = peaks_kde[1] - segment_median = ax_list[antipeaks_kde[0], 0] - segment_min_line = ax_list[min_peak_index, 0] - segment_max_line = ax_list[max_peak_index, 0] + try: + min_peak_index = peaks_kde[0] + segment_min_line = ax_list[min_peak_index, 0] + max_peak_index = peaks_kde[1] + segment_max_line = ax_list[max_peak_index, 0] + segment_median = ax_list[antipeaks_kde[0], 0] + except IndexError: + segment_max_line = max_drop + segment_min_line = min_drop + segment_median = (max_drop - min_drop) / 2 + min_drop drop_height = 0.95 * (segment_max_line - segment_min_line) drop_height_list.append(drop_height) drop_length = utils.find_drop_length(segment_data, segment_min_line, segment_max_line) @@ -94,9 +99,7 @@ class DropModel(Model): pdf = gaussian_kde(flat_segment_dropna) x = np.linspace(flat_segment_dropna.min() - 1, flat_segment_dropna.max() + 1, len(flat_segment_dropna)) y = pdf(x) - ax_list = [] - for i in range(len(x)): - ax_list.append([x[i], y[i]]) + ax_list = list(zip(x, y)) ax_list = np.array(ax_list, np.float32) antipeaks_kde = argrelextrema(np.array(ax_list), np.less)[0] segment_median = ax_list[antipeaks_kde[0], 0] @@ -170,9 +173,16 @@ class DropModel(Model): if segment > self.state['WINDOW_SIZE'] and segment < (len(data) - self.state['WINDOW_SIZE']): convol_data = data[segment - self.state['WINDOW_SIZE'] : segment + self.state['WINDOW_SIZE'] + 1] conv = scipy.signal.fftconvolve(convol_data, pattern_data) - if conv[self.state['WINDOW_SIZE']*2] > self.state['convolve_max'] * 1.2 or conv[self.state['WINDOW_SIZE']*2] < self.state['convolve_min'] * 0.8: - delete_list.append(segment) - elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: + upper_bound = self.state['convolve_max'] * 1.2 + lower_bound = self.state['convolve_min'] * 0.8 + delete_up_bound = self.state['conv_del_max'] * 1.02 + delete_low_bound = self.state['conv_del_min'] * 0.98 + try: + if max(conv) > upper_bound or max(conv) < lower_bound: + delete_list.append(segment) + elif max(conv) < delete_up_bound and max(conv) > delete_low_bound: + delete_list.append(segment) + except ValueError: delete_list.append(segment) else: delete_list.append(segment) diff --git a/analytics/models/jump_model.py b/analytics/models/jump_model.py index 7f43b19..8f8f8cb 100644 --- a/analytics/models/jump_model.py +++ b/analytics/models/jump_model.py @@ -47,20 +47,25 @@ class JumpModel(Model): confidences.append(0.20 * (segment_max - segment_min)) flat_segment = segment_data.rolling(window = 5).mean() flat_segment_dropna = flat_segment.dropna() + min_jump = min(flat_segment_dropna) + max_jump = max(flat_segment_dropna) pdf = gaussian_kde(flat_segment_dropna) x = np.linspace(flat_segment_dropna.min() - 1, flat_segment_dropna.max() + 1, len(flat_segment_dropna)) y = pdf(x) - ax_list = [] - for i in range(len(x)): - ax_list.append([x[i], y[i]]) + ax_list = list(zip(x, y)) ax_list = np.array(ax_list, np.float32) antipeaks_kde = argrelextrema(np.array(ax_list), np.less)[0] peaks_kde = argrelextrema(np.array(ax_list), np.greater)[0] - min_peak_index = peaks_kde[0] - max_peak_index = peaks_kde[1] - segment_median = ax_list[antipeaks_kde[0], 0] - segment_min_line = ax_list[min_peak_index, 0] - segment_max_line = ax_list[max_peak_index, 0] + try: + min_peak_index = peaks_kde[0] + segment_min_line = ax_list[min_peak_index, 0] + max_peak_index = peaks_kde[1] + segment_max_line = ax_list[max_peak_index, 0] + segment_median = ax_list[antipeaks_kde[0], 0] + except IndexError: + segment_max_line = max_jump + segment_min_line = min_jump + segment_median = (max_jump - min_jump) / 2 + min_jump jump_height = 0.95 * (segment_max_line - segment_min_line) jump_height_list.append(jump_height) jump_length = utils.find_jump_length(segment_data, segment_min_line, segment_max_line) @@ -95,9 +100,7 @@ class JumpModel(Model): pdf = gaussian_kde(flat_segment_dropna) x = np.linspace(flat_segment_dropna.min() - 1, flat_segment_dropna.max() + 1, len(flat_segment_dropna)) y = pdf(x) - ax_list = [] - for i in range(len(x)): - ax_list.append([x[i], y[i]]) + ax_list = list(zip(x, y)) ax_list = np.array(ax_list, np.float32) antipeaks_kde = argrelextrema(np.array(ax_list), np.less)[0] segment_median = ax_list[antipeaks_kde[0], 0] @@ -125,7 +128,7 @@ class JumpModel(Model): self.state['convolve_min'] = self.state['WINDOW_SIZE'] if len(jump_height_list) > 0: - self.state['JUMP_HEIGHT'] = int(min(jump_height_list)) + self.state['JUMP_HEIGHT'] = float(min(jump_height_list)) else: self.state['JUMP_HEIGHT'] = 1 @@ -167,13 +170,20 @@ class JumpModel(Model): delete_list = [] pattern_data = self.model_jump + upper_bound = self.state['convolve_max'] * 1.2 + lower_bound = self.state['convolve_min'] * 0.8 + delete_up_bound = self.state['conv_del_max'] * 1.02 + delete_low_bound = self.state['conv_del_min'] * 0.98 for segment in segments: if segment > self.state['WINDOW_SIZE'] and segment < (len(data) - self.state['WINDOW_SIZE']): convol_data = data[segment - self.state['WINDOW_SIZE'] : segment + self.state['WINDOW_SIZE'] + 1] conv = scipy.signal.fftconvolve(convol_data, pattern_data) - if max(conv) > self.state['convolve_max'] * 1.2 or max(conv) < self.state['convolve_min'] * 0.8: - delete_list.append(segment) - elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: + try: + if max(conv) > upper_bound or max(conv) < lower_bound: + delete_list.append(segment) + elif max(conv) < delete_up_bound and max(conv) > delete_low_bound: + delete_list.append(segment) + except ValueError: delete_list.append(segment) else: delete_list.append(segment) diff --git a/analytics/models/peak_model.py b/analytics/models/peak_model.py index a924ee6..179668a 100644 --- a/analytics/models/peak_model.py +++ b/analytics/models/peak_model.py @@ -135,6 +135,7 @@ class PeakModel(Model): if max(conv) > self.state['convolve_max'] * 1.05 or max(conv) < self.state['convolve_min'] * 0.95: delete_list.append(segment) elif max(conv) < self.state['conv_del_max'] * 1.02 and max(conv) > self.state['conv_del_min'] * 0.98: + print("this must be deleted: {0}, index: {1}".format(max(conv), segment)) delete_list.append(segment) else: delete_list.append(segment)