diff --git a/analytics/analytics/analytic_types/cache.py b/analytics/analytics/analytic_types/cache.py index efe3261..a21dc11 100644 --- a/analytics/analytics/analytic_types/cache.py +++ b/analytics/analytics/analytic_types/cache.py @@ -1,7 +1,7 @@ from typing import Optional, List, Dict from analytic_types.segment import AnomalyDetectorSegment -from analytic_types.detector_typing import Bound +from analytic_types.detector import Bound from utils.meta import JSONClass, SerializableList diff --git a/analytics/analytics/analytic_types/detector_typing.py b/analytics/analytics/analytic_types/detector.py similarity index 100% rename from analytics/analytics/analytic_types/detector_typing.py rename to analytics/analytics/analytic_types/detector.py diff --git a/analytics/analytics/analytic_unit_worker.py b/analytics/analytics/analytic_unit_worker.py index c94961c..ad8b00f 100644 --- a/analytics/analytics/analytic_unit_worker.py +++ b/analytics/analytics/analytic_unit_worker.py @@ -9,7 +9,7 @@ import utils from utils import get_intersected_chunks, get_chunks, prepare_data from analytic_types import ModelCache, TimeSeries -from analytic_types.detector_typing import DetectionResult +from analytic_types.detector import DetectionResult logger = logging.getLogger('AnalyticUnitWorker') diff --git a/analytics/analytics/detectors/anomaly_detector.py b/analytics/analytics/detectors/anomaly_detector.py index 21ee9b7..7885d01 100644 --- a/analytics/analytics/detectors/anomaly_detector.py +++ b/analytics/analytics/detectors/anomaly_detector.py @@ -7,7 +7,7 @@ from typing import Optional, Union, List, Tuple, Generator import operator from analytic_types import AnalyticUnitId, ModelCache -from analytic_types.detector_typing import DetectionResult, ProcessingResult, Bound +from analytic_types.detector import DetectionResult, ProcessingResult, Bound from analytic_types.data_bucket import DataBucket from analytic_types.segment import Segment, AnomalyDetectorSegment from analytic_types.cache import AnomalyCache diff --git a/analytics/analytics/detectors/detector.py b/analytics/analytics/detectors/detector.py index 8521890..b6fbcdf 100644 --- a/analytics/analytics/detectors/detector.py +++ b/analytics/analytics/detectors/detector.py @@ -3,7 +3,7 @@ from pandas import DataFrame from typing import Optional, Union, List from analytic_types import ModelCache, TimeSeries, AnalyticUnitId -from analytic_types.detector_typing import DetectionResult, ProcessingResult +from analytic_types.detector import DetectionResult, ProcessingResult from analytic_types.segment import Segment diff --git a/analytics/analytics/detectors/pattern_detector.py b/analytics/analytics/detectors/pattern_detector.py index e1f21a5..3e3a949 100644 --- a/analytics/analytics/detectors/pattern_detector.py +++ b/analytics/analytics/detectors/pattern_detector.py @@ -11,7 +11,7 @@ from detectors import Detector from analytic_types.data_bucket import DataBucket from utils import convert_pd_timestamp_to_ms from analytic_types import AnalyticUnitId, ModelCache -from analytic_types.detector_typing import DetectionResult +from analytic_types.detector import DetectionResult from analytic_types.segment import Segment import utils diff --git a/analytics/analytics/detectors/threshold_detector.py b/analytics/analytics/detectors/threshold_detector.py index aa8c67b..385bd02 100644 --- a/analytics/analytics/detectors/threshold_detector.py +++ b/analytics/analytics/detectors/threshold_detector.py @@ -6,7 +6,7 @@ import numpy as np from typing import Optional, List from analytic_types import ModelCache, AnalyticUnitId -from analytic_types.detector_typing import DetectionResult, ProcessingResult +from analytic_types.detector import DetectionResult, ProcessingResult from analytic_types.segment import Segment from detectors import ProcessingDetector from time import time diff --git a/analytics/analytics/utils/common.py b/analytics/analytics/utils/common.py index 96a3a91..50bb9e0 100644 --- a/analytics/analytics/utils/common.py +++ b/analytics/analytics/utils/common.py @@ -17,6 +17,7 @@ from analytic_types.segment import Segment SHIFT_FACTOR = 0.05 CONFIDENCE_FACTOR = 0.5 SMOOTHING_FACTOR = 5 +MEASUREMENT_ERROR = 0.05 def exponential_smoothing(series: pd.Series, alpha: float, last_smoothed_value: Optional[float] = None) -> pd.Series: @@ -391,7 +392,7 @@ def find_parameters(segment_data: pd.Series, segment_from_index: int, pat_type: segment = flat_segment.dropna() segment_median, segment_max_line, segment_min_line = utils.get_distribution_density(segment) height = 0.95 * (segment_max_line - segment_min_line) - length = utils.find_length(segment_data, segment_min_line, segment_max_line, pat_type) + length = utils.get_pattern_length(segment_data, segment_min_line, segment_max_line, pat_type) return height, length def find_pattern_center(segment_data: pd.Series, segment_from_index: int, pattern_type: str): @@ -404,14 +405,15 @@ def find_pattern_center(segment_data: pd.Series, segment_from_index: int, patter segment_cent_index = math.ceil((len(segment_data)) / 2) return segment_cent_index -def find_length(segment_data: pd.Series, segment_min_line: float, segment_max_line: float, pat_type: str) -> int: - x_abscissa = np.arange(0, len(segment_data)) +def get_pattern_length(segment_data: pd.Series, segment_min_line: float, segment_max_line: float, pat_type: str) -> int: + # TODO: move function to jump & drop merged model segment_max = max(segment_data) segment_min = min(segment_data) + # TODO: use better way if segment_min_line <= segment_min: - segment_min_line = segment_min * 1.05 + segment_min_line = segment_min * (1 + MEASUREMENT_ERROR) if segment_max_line >= segment_max: - segment_max_line = segment_max * 0.95 + segment_max_line = segment_max * (1 - MEASUREMENT_ERROR) min_line = [] max_line = [] for i in range(len(segment_data)): diff --git a/analytics/analytics/utils/meta.py b/analytics/analytics/utils/meta.py index c2771a1..59116a9 100644 --- a/analytics/analytics/utils/meta.py +++ b/analytics/analytics/utils/meta.py @@ -1,6 +1,6 @@ from inspect import signature, Parameter from functools import wraps -from typing import Optional +from typing import Optional, List import re @@ -76,6 +76,6 @@ def JSONClass(target_class): target_class.from_json = from_json return target_class -class SerializableList(list): +class SerializableList(List[dict]): def to_json(self): return list(map(lambda s: s.to_json(), self)) diff --git a/analytics/scripts/build_dist.sh b/analytics/scripts/build-dist.sh similarity index 100% rename from analytics/scripts/build_dist.sh rename to analytics/scripts/build-dist.sh diff --git a/analytics/tests/test_detectors.py b/analytics/tests/test_detectors.py index e568dfe..01a811c 100644 --- a/analytics/tests/test_detectors.py +++ b/analytics/tests/test_detectors.py @@ -2,7 +2,7 @@ import unittest import pandas as pd from detectors import pattern_detector, threshold_detector, anomaly_detector -from analytic_types.detector_typing import DetectionResult, ProcessingResult, Bound +from analytic_types.detector import DetectionResult, ProcessingResult, Bound from analytic_types.segment import Segment from tests.test_dataset import create_dataframe, create_list_of_timestamps from utils import convert_pd_timestamp_to_ms diff --git a/server/src/index.ts b/server/src/index.ts index 0125c52..3c672df 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -10,7 +10,7 @@ import * as ProcessService from './services/process_service'; import { HASTIC_PORT, PACKAGE_VERSION, GIT_INFO, ZMQ_CONNECTION_STRING, HASTIC_INSTANCE_NAME } from './config'; -import { applyDBMigrations } from './migrations'; +import { applyDBMigrations } from './services/data_service/migrations'; import * as Koa from 'koa'; import * as Router from 'koa-router'; diff --git a/server/src/models/analytic_units/db.ts b/server/src/models/analytic_units/db.ts index 2ec603b..3463994 100644 --- a/server/src/models/analytic_units/db.ts +++ b/server/src/models/analytic_units/db.ts @@ -41,7 +41,7 @@ export async function create(unit: AnalyticUnit): Promise { return db.insertOne(obj); } -export async function insertMany(analyticUnits: any[]): Promise { +export async function insertMany(analyticUnits: AnalyticUnit[]): Promise { return db.insertMany(analyticUnits); } diff --git a/server/src/models/panel_model.ts b/server/src/models/grafana_panel_model.ts similarity index 86% rename from server/src/models/panel_model.ts rename to server/src/models/grafana_panel_model.ts index 0869b21..d1c74e4 100644 --- a/server/src/models/panel_model.ts +++ b/server/src/models/grafana_panel_model.ts @@ -3,14 +3,14 @@ import * as AnalyticUnitCache from '../models/analytic_unit_cache_model'; import * as DetectionSpan from '../models/detection_model'; import * as Segment from '../models/segment_model'; -export type PanelTemplate = { +export type GrafanaPanelTemplate = { analyticUnits: AnalyticUnit.AnalyticUnit[], caches: AnalyticUnitCache.AnalyticUnitCache[], detectionSpans: DetectionSpan.DetectionSpan[], segments: Segment.Segment[] } -export type TemplateVariables = { +export type GrafanaTemplateVariables = { grafanaUrl: string, panelId: string, datasourceUrl: string diff --git a/server/src/routes/panel_router.ts b/server/src/routes/panel_router.ts index e20514f..6334ab9 100644 --- a/server/src/routes/panel_router.ts +++ b/server/src/routes/panel_router.ts @@ -1,10 +1,10 @@ -import { PanelTemplate, TemplateVariables } from '../models/panel_model'; -import { exportPanel, importPanel } from '../services/export_service'; +import { GrafanaPanelTemplate, GrafanaTemplateVariables } from '../models/grafana_panel_model'; +import { exportPanel, importPanel } from '../services/grafana_service'; import * as Router from 'koa-router'; -async function exportPanelTemplate(ctx: Router.IRouterContext) { +async function exportGrafanaPanelTemplate(ctx: Router.IRouterContext) { let panelId = ctx.request.query.panelId; if(panelId === undefined) { throw new Error('Cannot export analytic units with undefined panelId'); @@ -14,10 +14,10 @@ async function exportPanelTemplate(ctx: Router.IRouterContext) { ctx.response.body = panelTemplate; } -async function importPanelTemplate(ctx: Router.IRouterContext) { +async function importGrafanaPanelTemplate(ctx: Router.IRouterContext) { const { panelTemplate, templateVariables } = ctx.request.body as { - panelTemplate: PanelTemplate, - templateVariables: TemplateVariables + panelTemplate: GrafanaPanelTemplate, + templateVariables: GrafanaTemplateVariables }; // TODO: move to model @@ -50,5 +50,5 @@ async function importPanelTemplate(ctx: Router.IRouterContext) { export var router = new Router(); -router.get('/template', exportPanelTemplate); -router.post('/template', importPanelTemplate); +router.get('/template', exportGrafanaPanelTemplate); +router.post('/template', importGrafanaPanelTemplate); diff --git a/server/src/services/data_service.ts b/server/src/services/data_service/index.ts similarity index 97% rename from server/src/services/data_service.ts rename to server/src/services/data_service/index.ts index 8d71d53..49dd374 100644 --- a/server/src/services/data_service.ts +++ b/server/src/services/data_service/index.ts @@ -1,5 +1,5 @@ -import { getDbQueryWrapper, dbCollection, DBType } from './data_layer'; -import * as config from '../config'; +import { getDbQueryWrapper, dbCollection, DBType } from '../data_layer'; +import * as config from '../../config'; import * as nedb from 'nedb'; import * as fs from 'fs'; diff --git a/server/src/migrations.ts b/server/src/services/data_service/migrations.ts similarity index 98% rename from server/src/migrations.ts rename to server/src/services/data_service/migrations.ts index eda9abd..806f179 100644 --- a/server/src/migrations.ts +++ b/server/src/services/data_service/migrations.ts @@ -7,7 +7,7 @@ Note: do not import code from other modules here because it can be changed */ -import { Collection, makeDBQ } from './services/data_service'; +import { Collection, makeDBQ } from './index'; import * as _ from 'lodash'; diff --git a/server/src/services/export_service.ts b/server/src/services/grafana_service.ts similarity index 84% rename from server/src/services/export_service.ts rename to server/src/services/grafana_service.ts index 2449228..4960321 100644 --- a/server/src/services/export_service.ts +++ b/server/src/services/grafana_service.ts @@ -1,4 +1,4 @@ -import { PanelTemplate, TemplateVariables } from '../models/panel_model'; +import { GrafanaPanelTemplate, GrafanaTemplateVariables } from '../models/grafana_panel_model'; import * as AnalyticUnit from '../models/analytic_units'; import * as AnalyticUnitCache from '../models/analytic_unit_cache_model'; @@ -6,7 +6,7 @@ import * as DetectionSpan from '../models/detection_model'; import * as Segment from '../models/segment_model'; -export async function exportPanel(panelId: string): Promise { +export async function exportPanel(panelId: string): Promise { const analyticUnits = await AnalyticUnit.findMany({ panelId }); const analyticUnitIds = analyticUnits.map(analyticUnit => analyticUnit.id); const analyticUnitTemplates = analyticUnits.map(analyticUnit => analyticUnit.toTemplate()); @@ -26,8 +26,8 @@ export async function exportPanel(panelId: string): Promise { } export async function importPanel( - panelTemplate: PanelTemplate, - variables: TemplateVariables + panelTemplate: GrafanaPanelTemplate, + variables: GrafanaTemplateVariables ): Promise { panelTemplate.analyticUnits.forEach(analyticUnit => { analyticUnit.grafanaUrl = variables.grafanaUrl;