hastic-server/analytics/step_detector.py

import numpy as np
import pickle


def find_segments(array, threshold):
    segments     = []
    above_points = np.where(array > threshold, 1, 0)
    ap_dif       = np.diff(above_points)
    cross_ups    = np.where(ap_dif == 1)[0]
    cross_dns    = np.where(ap_dif == -1)[0]
    for upi, dni in zip(cross_ups,cross_dns):
        segments.append((upi, dni))
    return segments


def is_intersect(target_segment, segments):
    for segment in segments:
        start = max(segment['start'], target_segment[0])
        finish = min(segment['finish'], target_segment[1])
        if start <= finish:
            return True
    return False


def calc_intersections(segments, finded_segments):
    intersections = 0
    labeled = 0
    for segment in segments:
        if not segment['labeled']:
            continue

        labeled += 1
        intersect = False
        for finded_segment in finded_segments:
            start = max(segment['start'], finded_segment[0])
            finish = min(segment['finish'], finded_segment[1])
            if start <= finish:
                intersect = True
                break
        if intersect:
            intersections += 1
    return intersections, labeled


def cost_function(segments, finded_segments):
    intersections, labeled = calc_intersections(segments, finded_segments)
    return intersections == labeled


def compress_segments(segments):
    result = []
    for segment in segments:
        if len(result) == 0 or result[len(result) - 1][1] < segment[0]:
            result.append(segment)
        else:
            result[len(result) - 1] = (result[len(result) - 1][0], segment[1])
    return result


class StepDetector:
    def __init__(self, pattern):
        self.pattern = pattern
        self.mean = None
        self.window_size = None
        self.corr_max = None
        self.threshold = None
        self.segments = []

    def fit(self, dataframe, segments, contamination=0.01):
        array = dataframe['value'].as_matrix()
        self.mean = array.mean()
        self.segments = segments

        norm_data = (array - self.mean)

        self.__optimize(norm_data, segments, contamination)

        # print(self.threshold)

        # import matplotlib.pyplot as plt
        # fig, ax = plt.subplots(figsize=[18, 16])
        # ax = fig.add_subplot(2, 1, 1)
        # ax.plot(array)
        # ax = fig.add_subplot(2, 1, 2, sharex=ax)
        # ax.plot(corr_res)
        # plt.show()

        # #print(R.size)
        # # Nw = 20
        # # result = R[Nw,Nw:-1]
        # # result[0] = 0
        # #ax.plot(result)
        # #print(len(data))
        # #print(len(R))
        #
        # print(self.window_size)
        # print(self.threshold)

    def predict(self, dataframe):
        array = dataframe['value'].as_matrix()

        norm_data = (array - self.mean)

        step_size = self.window_size // 2
        pattern = np.concatenate([[-1] * step_size, [1] * step_size])
        corr_res = np.correlate(norm_data, pattern, mode='valid') / self.window_size
        corr_res = np.concatenate((np.zeros(step_size), corr_res, np.zeros(step_size)))

        corr_res /= self.corr_max

        result = self.__predict(corr_res, self.threshold)

        # import matplotlib.pyplot as plt
        # fig, ax = plt.subplots(figsize=[18, 16])
        # ax = fig.add_subplot(2, 1, 1)
        # ax.plot(array[:70000])
        # ax = fig.add_subplot(2, 1, 2, sharex=ax)
        # ax.plot(corr_res[:70000])
        # plt.show()

        result.sort()
        result = compress_segments(result)

        if len(self.segments) > 0:
            result = [segment for segment in result if not is_intersect(segment, self.segments)]
        return result

    def __optimize(self, data, segments, contamination):
        window_size = 10
        mincost = None
        while window_size < 100:
            # print(window_size)
            cost = self.__optimize_threshold(data, window_size, segments, contamination)
            if mincost is None or cost < mincost:
                mincost = cost
                self.window_size = window_size
            window_size = int(window_size * 1.2)
        self.__optimize_threshold(data, self.window_size, segments, contamination)

    def __optimize_threshold(self, data, window_size, segments, contamination):
        step_size = window_size // 2
        pattern = np.concatenate([[-1] * step_size, [1] * step_size])
        corr_res = np.correlate(data, pattern, mode='same') / window_size
        corr_res = np.concatenate((np.zeros(step_size), corr_res, np.zeros(step_size)))
        self.corr_max = corr_res.max()
        corr_res /= self.corr_max
        N = 20
        lower = 0.
        upper = 1.
        cost = 0
        for i in range(0, N):
            self.threshold = 0.5 * (lower + upper)
            result = self.__predict(corr_res, self.threshold)

            if len(segments) > 0:
                intersections, labeled = calc_intersections(segments, result)
                good = intersections == labeled
                cost = len(result)
            else:
                total_sum = 0
                for segment in result:
                    total_sum += (segment[1] - segment[0])
                good = total_sum > len(data) * contamination
                cost = -self.threshold

            if good:
                lower = self.threshold
            else:
                upper = self.threshold

        return cost

    def __predict(self, data, threshold):
        segments = find_segments(data, threshold)
        segments += find_segments(data * -1, threshold)
        #segments -= 1
        return [(x - 1, y - 1) for (x, y) in segments]

    def save(self, model_filename):
        with open(model_filename, 'wb') as file:
            pickle.dump((self.mean, self.window_size, self.corr_max, self.threshold), file)

    def load(self, model_filename):
        try:
            with open(model_filename, 'rb') as file:
                self.mean, self.window_size, self.corr_max, self.threshold = pickle.load(file)
        except:
            pass