Improve drops algorithm

analytics/step_detector.py

@@ -1,5 +1,7 @@
 import numpy as np
 import pickle
+import scipy.signal
+from scipy.fftpack import fft
 from scipy.signal import argrelextrema
 
 def is_intersect(target_segment, segments):
@@ -22,21 +24,34 @@ class StepDetector:
         self.pattern = pattern
         self.segments = []
         self.confidence = 1.5
+        self.convolve_max = 570000
 
     def fit(self, dataframe, segments):
         data = dataframe['value']
         confidences = []
+        convolve_list = []
         for segment in segments:
             if segment['labeled']:
                 segment_data = data[segment['start'] : segment['finish'] + 1]
                 segment_min = min(segment_data)
                 segment_max = max(segment_data)
                 confidences.append(0.24 * (segment_max - segment_min))
+                flat_segment = segment_data.rolling(window=5).mean()
+                
+                segment_min_index = flat_segment.idxmin() - 5
+                labeled_drop = data[segment_min_index - 60 : segment_min_index + 60]
+                convolve = scipy.signal.fftconvolve(labeled_drop, labeled_drop)
+                convolve_list.append(max(convolve))
+
         if len(confidences) > 0:
             self.confidence = min(confidences)
         else:
             self.confidence = 1.5
 
+        if len(convolve_list) > 0:
+            self.convolve_max = max(convolve_list)
+        else:
+            self.convolve_max = 570000
 
     def predict(self, dataframe):
         data = dataframe['value']
@@ -66,15 +81,22 @@ class StepDetector:
 
     def __filter_prediction(self, segments, all_max_flatten_data):
         delete_list = []
-        for i in segments:
-            new_data = all_max_flatten_data[i-50:i+250]
-            min_value = 100
-            for val in new_data:
-                if val < min_value:
-                    min_value = val
-            if all_max_flatten_data[i] > min_value:
-                delete_list.append(i)
+        variance_error = int(0.004 * len(all_max_flatten_data))
+        if variance_error > 200:
+            variance_error = 200
+        for i in range(1, len(segments)):
+            if segments[i] < segments[i - 1] + variance_error:
+                delete_list.append(segments[i])
+        for item in delete_list:
+            segments.remove(item)
 
+        delete_list = []
+        pattern_data = all_max_flatten_data[segments[0] - 60 : segments[0] + 60]
+        for segment in segments:
+            convol_data = all_max_flatten_data[segment - 60 : segment + 60]
+            conv = scipy.signal.fftconvolve(pattern_data, convol_data)
+            if max(conv) > self.convolve_max * 1.05:
+                delete_list.append(segment)
         for item in delete_list:
             segments.remove(item)
 
@@ -82,11 +104,11 @@ class StepDetector:
 
     def save(self, model_filename):
         with open(model_filename, 'wb') as file:
-            pickle.dump((self.confidence), file)
+            pickle.dump((self.confidence, self.convolve_max), file)
 
     def load(self, model_filename):
         try:
             with open(model_filename, 'rb') as file:
-                self.confidence = pickle.load(file)
+                (self.confidence, self.convolve_max) = pickle.load(file)
         except:
             pass