From a364e66899cdc53d12fb47ba720943d19fe880c4 Mon Sep 17 00:00:00 2001 From: rozetko Date: Wed, 5 Feb 2020 01:01:53 +0300 Subject: [PATCH] Export analytic units (#833) --- server/src/index.ts | 4 ++- .../src/models/analytic_unit_cache_model.ts | 12 ++++++++ .../analytic_units/analytic_unit_model.ts | 21 ++++++++++++-- server/src/models/analytic_units/index.ts | 6 +++- server/src/models/analytic_units/types.ts | 24 +++++++++++++++ server/src/models/analytic_units/utils.ts | 2 +- server/src/models/detection_model.ts | 10 +++++++ server/src/models/segment_model.ts | 8 +++++ server/src/routes/panel_router.ts | 18 ++++++++++++ server/src/services/export_service.ts | 29 +++++++++++++++++++ server/src/types.ts | 7 +++++ 11 files changed, 135 insertions(+), 6 deletions(-) create mode 100644 server/src/routes/panel_router.ts create mode 100644 server/src/services/export_service.ts create mode 100644 server/src/types.ts diff --git a/server/src/index.ts b/server/src/index.ts index 28a731c..0125c52 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,7 +1,8 @@ import { router as analyticUnitsRouter } from './routes/analytic_units_router'; import { router as segmentsRouter } from './routes/segments_router'; import { router as dataRouter } from './routes/data_router'; -import { router as detectionsRouter } from './routes/detections_router'; +import { router as detectionsRouter } from './routes/detections_router'; +import { router as panelRouter } from './routes/panel_router'; import * as AnalyticsController from './controllers/analytics_controller'; @@ -57,6 +58,7 @@ async function init() { rootRouter.use('/segments', segmentsRouter.routes(), segmentsRouter.allowedMethods()); rootRouter.use('/query', dataRouter.routes(), dataRouter.allowedMethods()); rootRouter.use('/detections', detectionsRouter.routes(), detectionsRouter.allowedMethods()); + rootRouter.use('/panels', panelRouter.routes(), panelRouter.allowedMethods()); rootRouter.get('/', async (ctx) => { const activeWebhooks = await AnalyticsController.getActiveWebhooks(); diff --git a/server/src/models/analytic_unit_cache_model.ts b/server/src/models/analytic_unit_cache_model.ts index a698c93..2a7e28a 100644 --- a/server/src/models/analytic_unit_cache_model.ts +++ b/server/src/models/analytic_unit_cache_model.ts @@ -8,6 +8,10 @@ const db = makeDBQ(Collection.ANALYTIC_UNIT_CACHES); // TODO: count milliseconds in index from dataset const MILLISECONDS_IN_INDEX = 60000; +type FindManyQuery = { + _id: { $in: AnalyticUnitId[] } +}; + export class AnalyticUnitCache { public constructor( public id: AnalyticUnitId, @@ -65,6 +69,14 @@ export async function findById(id: AnalyticUnitId): Promise { + let caches = await db.findMany(query); + if(caches === null) { + return []; + } + return caches.map(cache => AnalyticUnitCache.fromObject(cache)); +} + export async function create(id: AnalyticUnitId): Promise { let cache = new AnalyticUnitCache(id); return db.insertOne(cache.toObject()); diff --git a/server/src/models/analytic_units/analytic_unit_model.ts b/server/src/models/analytic_units/analytic_unit_model.ts index 78e7d4d..1b9d715 100644 --- a/server/src/models/analytic_units/analytic_unit_model.ts +++ b/server/src/models/analytic_units/analytic_unit_model.ts @@ -1,7 +1,12 @@ -import { AnalyticUnitId, AnalyticUnitStatus, DetectorType } from './types'; +import { + AnalyticUnitId, AnalyticUnitStatus, DetectorType, + SerializedAnalyticUnit, SerializedPanelAnalyticUnit +} from './types'; import { Metric } from 'grafana-datasource-kit'; +import * as _ from 'lodash'; + export abstract class AnalyticUnit { public learningAfterUpdateRequired = false; @@ -38,7 +43,7 @@ export abstract class AnalyticUnit { } } - public toObject() { + public toObject(): SerializedAnalyticUnit { let metric; if(this.metric !== undefined) { metric = this.metric.toObject(); @@ -63,7 +68,7 @@ export abstract class AnalyticUnit { }; } - public toPanelObject() { + public toPanelObject(): SerializedPanelAnalyticUnit { return { id: this.id, name: this.name, @@ -77,6 +82,16 @@ export abstract class AnalyticUnit { }; } + public toTemplate(): SerializedAnalyticUnit { + const obj = _.cloneDeep(this.toObject()); + + obj.grafanaUrl = '${GRAFANA_URL}'; + obj.panelId = '${PANEL_ID}'; + obj.metric.datasource.url = '${DATASOURCE_URL}'; + + return obj; + } + get analyticProps () { return {}; } diff --git a/server/src/models/analytic_units/index.ts b/server/src/models/analytic_units/index.ts index 1381735..5530817 100644 --- a/server/src/models/analytic_units/index.ts +++ b/server/src/models/analytic_units/index.ts @@ -1,5 +1,8 @@ import { createAnalyticUnitFromObject } from './utils'; -import { AnalyticUnitId, AnalyticUnitStatus, DetectorType, ANALYTIC_UNIT_TYPES } from './types'; +import { + AnalyticUnitId, AnalyticUnitStatus, DetectorType, ANALYTIC_UNIT_TYPES, + SerializedAnalyticUnit, SerializedPanelAnalyticUnit +} from './types'; import { AnalyticUnit } from './analytic_unit_model'; import { PatternAnalyticUnit } from './pattern_analytic_unit_model'; import { ThresholdAnalyticUnit, Condition } from './threshold_analytic_unit_model'; @@ -19,6 +22,7 @@ import { export { AnalyticUnit, PatternAnalyticUnit, ThresholdAnalyticUnit, AnomalyAnalyticUnit, + SerializedAnalyticUnit, SerializedPanelAnalyticUnit, AnalyticUnitId, AnalyticUnitStatus, Bound, DetectorType, ANALYTIC_UNIT_TYPES, createAnalyticUnitFromObject, Condition, findById, findMany, diff --git a/server/src/models/analytic_units/types.ts b/server/src/models/analytic_units/types.ts index e95eaa9..2d6b040 100644 --- a/server/src/models/analytic_units/types.ts +++ b/server/src/models/analytic_units/types.ts @@ -1,3 +1,5 @@ +import { Omit } from '../../types'; + import { Metric } from 'grafana-datasource-kit'; @@ -71,3 +73,25 @@ export enum DetectorType { ANOMALY = 'anomaly', THRESHOLD = 'threshold' }; + +export type SerializedPanelAnalyticUnit = { + id: AnalyticUnitId; + name: string; + type: string; + alert: boolean; + labeledColor?: string; + deletedColor?: string; + detectorType?: DetectorType; + visible?: boolean; + collapsed?: boolean; +} + +export type SerializedAnalyticUnit = Omit & { + grafanaUrl: string; + panelId: string; + metric?: Metric; + _id?: AnalyticUnitId; + lastDetectionTime?: number; + status?: AnalyticUnitStatus; + error?: string; +} diff --git a/server/src/models/analytic_units/utils.ts b/server/src/models/analytic_units/utils.ts index 7b867ba..1ef43df 100644 --- a/server/src/models/analytic_units/utils.ts +++ b/server/src/models/analytic_units/utils.ts @@ -1,4 +1,4 @@ -import { DetectorType, ANALYTIC_UNIT_TYPES } from './types'; +import { DetectorType } from './types'; import { AnalyticUnit } from './analytic_unit_model'; import { PatternAnalyticUnit } from './pattern_analytic_unit_model'; import { AnomalyAnalyticUnit } from './anomaly_analytic_unit_model'; diff --git a/server/src/models/detection_model.ts b/server/src/models/detection_model.ts index 1d5479d..d12bfb1 100644 --- a/server/src/models/detection_model.ts +++ b/server/src/models/detection_model.ts @@ -106,6 +106,16 @@ export async function findMany(id: AnalyticUnitId, query?: FindManyQuery): Promi return spans.map(DetectionSpan.fromObject); } +// TODO: maybe it could have a better name +export async function findByAnalyticUnitIds(analyticUnitIds: AnalyticUnitId[]): Promise { + const spans = await db.findMany({ analyticUnitId: { $in: analyticUnitIds } }); + + if(spans === null) { + return []; + } + return spans.map(DetectionSpan.fromObject); +} + export async function getIntersectedSpans( analyticUnitId: AnalyticUnitId, from: number, diff --git a/server/src/models/segment_model.ts b/server/src/models/segment_model.ts index b672781..c1bf8c9 100644 --- a/server/src/models/segment_model.ts +++ b/server/src/models/segment_model.ts @@ -106,6 +106,14 @@ export async function findMany(id: AnalyticUnitId, query: FindManyQuery): Promis return segs.map(Segment.fromObject); } +export async function findByAnalyticUnitIds(analyticUnitIds: AnalyticUnitId[]): Promise { + const segments = await db.findMany({ analyticUnitId: { $in: analyticUnitIds } }); + + if(segments === null) { + return []; + } + return segments.map(Segment.fromObject); +} /** * If `from` and `to` are defined: @returns segments intersected with `[from; to]` diff --git a/server/src/routes/panel_router.ts b/server/src/routes/panel_router.ts new file mode 100644 index 0000000..a867107 --- /dev/null +++ b/server/src/routes/panel_router.ts @@ -0,0 +1,18 @@ +import { exportPanel } from '../services/export_service'; + +import * as Router from 'koa-router'; + + +async function getPanelTemplate(ctx: Router.IRouterContext) { + let panelId = ctx.request.query.panelId; + if(panelId === undefined) { + throw new Error('Cannot export analytic units with undefined panelId'); + } + + const json = await exportPanel(panelId); + ctx.response.body = json; +} + +export var router = new Router(); + +router.get('/template', getPanelTemplate); diff --git a/server/src/services/export_service.ts b/server/src/services/export_service.ts new file mode 100644 index 0000000..d8264c8 --- /dev/null +++ b/server/src/services/export_service.ts @@ -0,0 +1,29 @@ +import * as AnalyticUnit from '../models/analytic_units'; +import * as AnalyticUnitCache from '../models/analytic_unit_cache_model'; +import * as DetectionSpan from '../models/detection_model'; +import * as Segment from '../models/segment_model'; + + +export async function exportPanel(panelId: string): Promise<{ + analyticUnitTemplates: any[], + caches: AnalyticUnitCache.AnalyticUnitCache[], + detectionSpans: DetectionSpan.DetectionSpan[], + segments: Segment.Segment[] +}> { + const analyticUnits = await AnalyticUnit.findMany({ panelId }); + const analyticUnitIds = analyticUnits.map(analyticUnit => analyticUnit.id); + const analyticUnitTemplates = analyticUnits.map(analyticUnit => analyticUnit.toTemplate()); + + const [caches, detectionSpans, segments] = await Promise.all([ + AnalyticUnitCache.findMany({ _id: { $in: analyticUnitIds } }), + DetectionSpan.findByAnalyticUnitIds(analyticUnitIds), + Segment.findByAnalyticUnitIds(analyticUnitIds) + ]); + + return { + analyticUnitTemplates, + caches, + detectionSpans, + segments + }; +} diff --git a/server/src/types.ts b/server/src/types.ts new file mode 100644 index 0000000..848c2cc --- /dev/null +++ b/server/src/types.ts @@ -0,0 +1,7 @@ +// We should remove Omit definition when we'll be updating TypeScript +// It was introduced in TS v3.5.1: +// https://devblogs.microsoft.com/typescript/announcing-typescript-3-5/#the-omit-helper-type +/** + * Construct a type with the properties of T except for those in type K. +*/ +export type Omit = { [P in Exclude]: T[P]; }