From 015f819958226232eccde3f5f16f28eac07cace9 Mon Sep 17 00:00:00 2001 From: Evgeny Smyshlyaev Date: Wed, 20 Mar 2019 15:00:27 +0300 Subject: [PATCH] Synchronize data with server #455 (#461) Synchronize data with server #455 (#461) --- server/src/config.ts | 3 +- .../src/controllers/analytics_controller.ts | 13 ++-- server/src/index.ts | 5 +- server/src/migrations/0.3.2-beta.ts | 35 ++++++++++ server/src/models/analytic_unit_model.ts | 54 ++++++++++++--- server/src/models/panel_model.ts | 67 ------------------- server/src/routes/analytic_units_router.ts | 43 ++++-------- server/src/routes/panel_router.ts | 43 ------------ server/src/services/alert_service.ts | 2 +- server/src/services/analytics_service.ts | 1 + server/src/services/data_puller.ts | 8 +-- server/src/services/data_service.ts | 5 +- server/src/services/notification_service.ts | 2 +- 13 files changed, 115 insertions(+), 166 deletions(-) create mode 100644 server/src/migrations/0.3.2-beta.ts delete mode 100644 server/src/models/panel_model.ts delete mode 100644 server/src/routes/panel_router.ts diff --git a/server/src/config.ts b/server/src/config.ts index a1f61bd..7e0ddf0 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -14,9 +14,8 @@ export const ANALYTICS_PATH = path.join(__dirname, '../../analytics'); export const DATA_PATH = path.join(__dirname, '../../data'); export const ANALYTIC_UNITS_DATABASE_PATH = path.join(DATA_PATH, 'analytic_units.db'); -export const SEGMENTS_DATABASE_PATH = path.join(DATA_PATH, 'segments.db'); export const ANALYTIC_UNIT_CACHES_DATABASE_PATH = path.join(DATA_PATH, 'analytic_unit_caches.db'); -export const PANELS_DATABASE_PATH = path.join(DATA_PATH, 'panels.db'); +export const SEGMENTS_DATABASE_PATH = path.join(DATA_PATH, 'segments.db'); export const THRESHOLD_DATABASE_PATH = path.join(DATA_PATH, 'treshold.db'); diff --git a/server/src/controllers/analytics_controller.ts b/server/src/controllers/analytics_controller.ts index 50aa25e..69e9676 100644 --- a/server/src/controllers/analytics_controller.ts +++ b/server/src/controllers/analytics_controller.ts @@ -123,11 +123,11 @@ async function query(analyticUnit: AnalyticUnit.AnalyticUnit, detector: Analytic } console.log(`query time range: from ${new Date(range.from)} to ${new Date(range.to)}`); - let panelUrl; + let grafanaUrl; if(GRAFANA_URL !== null) { - panelUrl = GRAFANA_URL; + grafanaUrl = GRAFANA_URL; } else { - panelUrl = analyticUnit.panelUrl; + grafanaUrl = analyticUnit.grafanaUrl; } let data; @@ -135,7 +135,7 @@ async function query(analyticUnit: AnalyticUnit.AnalyticUnit, detector: Analytic try { const queryResult = await queryByMetric( analyticUnit.metric, - panelUrl, + grafanaUrl, range.from, range.to, HASTIC_API_KEY @@ -284,7 +284,10 @@ export async function runDetect(id: AnalyticUnit.AnalyticUnitId) { } export async function remove(analyticUnitId: AnalyticUnit.AnalyticUnitId) { - await cancelAnalyticsTask(analyticUnitId); + // We don't await for analytics task cancellation here + // If we add await, the rest function will be executed only when analytics becomes up + + cancelAnalyticsTask(analyticUnitId); if(dataPuller !== undefined) { dataPuller.deleteUnit(analyticUnitId); diff --git a/server/src/index.ts b/server/src/index.ts index d632f05..9b9ac2d 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,6 +1,5 @@ import { router as analyticUnitsRouter } from './routes/analytic_units_router'; import { router as segmentsRouter } from './routes/segments_router'; -import { router as panelRouter } from './routes/panel_router'; import { router as thresholdRouter } from './routes/threshold_router'; import * as AnalyticsController from './controllers/analytics_controller'; @@ -9,6 +8,8 @@ import * as ProcessService from './services/process_service'; import { HASTIC_PORT, PACKAGE_VERSION, GIT_INFO, ZMQ_CONNECTION_STRING } from './config'; +import { convertPanelUrlToPanelId } from './migrations/0.3.2-beta'; + import * as Koa from 'koa'; import * as Router from 'koa-router'; import * as bodyParser from 'koa-bodyparser'; @@ -16,6 +17,7 @@ import * as bodyParser from 'koa-bodyparser'; AnalyticsController.init(); ProcessService.registerExitHandler(AnalyticsController.terminate); +convertPanelUrlToPanelId(); var app = new Koa(); @@ -50,7 +52,6 @@ app.use(async function(ctx, next) { var rootRouter = new Router(); rootRouter.use('/analyticUnits', analyticUnitsRouter.routes(), analyticUnitsRouter.allowedMethods()); rootRouter.use('/segments', segmentsRouter.routes(), segmentsRouter.allowedMethods()); -rootRouter.use('/panel', panelRouter.routes(), panelRouter.allowedMethods()); rootRouter.use('/threshold', thresholdRouter.routes(), thresholdRouter.allowedMethods()); rootRouter.get('/', async (ctx) => { diff --git a/server/src/migrations/0.3.2-beta.ts b/server/src/migrations/0.3.2-beta.ts new file mode 100644 index 0000000..b27d862 --- /dev/null +++ b/server/src/migrations/0.3.2-beta.ts @@ -0,0 +1,35 @@ +import { Collection, makeDBQ } from '../services/data_service'; + +const db = makeDBQ(Collection.ANALYTIC_UNITS); + +export async function convertPanelUrlToPanelId() { + const analyticUnits = await db.findMany({ panelUrl: { $exists: true } }); + console.log(`Found ${analyticUnits.length} analytic units with panelUrl field`); + if(analyticUnits.length === 0) { + console.log('Nothing to migrate'); + return; + } + + const panelUrlRegex = /^(.+)\/d\/(\w+)\/.+panelId=(\d+)/; + const newPanelUrlRegex = /^(.+)\/dashboard\/(\w+).+panelId=(\d+)/; + const updatedAnalyticUnits = analyticUnits.map(analyticUnit => { + const parsedPanelUrl = analyticUnit.panelUrl.match(panelUrlRegex) || analyticUnit.panelUrl.match(newPanelUrlRegex); + const grafanaUrl = parsedPanelUrl[1]; + const dashboardId = parsedPanelUrl[2]; + const oldPanelId = parsedPanelUrl[3]; + const panelId = `${dashboardId}/${oldPanelId}`; + + return { + _id: analyticUnit._id, + grafanaUrl, + panelId + }; + }); + + console.log(updatedAnalyticUnits); + await updatedAnalyticUnits.forEach(analyticUnit => db.updateOne(analyticUnit._id, { + panelUrl: undefined, + ...analyticUnit + })); +} + diff --git a/server/src/models/analytic_unit_model.ts b/server/src/models/analytic_unit_model.ts index d8577df..19fa45b 100644 --- a/server/src/models/analytic_unit_model.ts +++ b/server/src/models/analytic_unit_model.ts @@ -54,35 +54,45 @@ export enum AnalyticUnitStatus { export type FindManyQuery = { name?: string, - panelUrl?: string, + grafanaUrl?: string, + panelId?: string, type?: string, metric?: Metric, alert?: boolean, id?: AnalyticUnitId, lastDetectionTime?: number, status?: AnalyticUnitStatus, - error?: string + error?: string, + labeledColor?: string, + deletedColor?: string, + detectorType?: DetectorType, + visible?: boolean }; export class AnalyticUnit { constructor( public name: string, - public panelUrl: string, + public grafanaUrl: string, + public panelId: string, public type: string, public metric?: Metric, public alert?: boolean, public id?: AnalyticUnitId, public lastDetectionTime?: number, public status?: AnalyticUnitStatus, - public error?: string + public error?: string, + public labeledColor?: string, + public deletedColor?: string, + public detectorType?: DetectorType, + public visible?: boolean ) { if(name === undefined) { throw new Error(`Missing field "name"`); } - if(panelUrl === undefined) { - throw new Error(`Missing field "panelUrl"`); + if(grafanaUrl === undefined) { + throw new Error(`Missing field "grafanaUrl"`); } if(type === undefined) { throw new Error(`Missing field "type"`); @@ -98,7 +108,8 @@ export class AnalyticUnit { return { _id: this.id, name: this.name, - panelUrl: this.panelUrl, + grafanaUrl: this.grafanaUrl, + panelId: this.panelId, type: this.type, metric, alert: this.alert, @@ -108,6 +119,19 @@ export class AnalyticUnit { }; } + public toPanelObject() { + return { + id: this.id, + name: this.name, + type: this.type, + alert: this.alert, + labeledColor: this.labeledColor, + deletedColor: this.deletedColor, + detectorType: this.detectorType, + visible: this.visible + }; + } + static fromObject(obj: any): AnalyticUnit { if(obj === undefined) { throw new Error('obj is undefined'); @@ -118,7 +142,8 @@ export class AnalyticUnit { } return new AnalyticUnit( obj.name, - obj.panelUrl, + obj.grafanaUrl, + obj.panelId, obj.type, metric, obj.alert, @@ -126,6 +151,10 @@ export class AnalyticUnit { obj.lastDetectionTime, obj.status as AnalyticUnitStatus, obj.error, + obj.labeledColor, + obj.deletedColor, + obj.detectorType, + obj.visible ); } @@ -167,7 +196,14 @@ export async function remove(id: AnalyticUnitId): Promise { } export async function update(id: AnalyticUnitId, unit: AnalyticUnit) { - return db.updateOne(id, unit); + const updateObj = { + name: unit.name, + labeledColor: unit.labeledColor, + deletedColor: unit.deletedColor, + visible: unit.visible + }; + + return db.updateOne(id, updateObj); } export async function setStatus(id: AnalyticUnitId, status: string, error?: string) { diff --git a/server/src/models/panel_model.ts b/server/src/models/panel_model.ts deleted file mode 100644 index 7f9a57e..0000000 --- a/server/src/models/panel_model.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { AnalyticUnitId } from './analytic_unit_model'; - -import { Collection, makeDBQ } from '../services/data_service'; - -let db = makeDBQ(Collection.PANELS); - - -export type PanelId = string; - -export class Panel { - constructor( - public panelUrl: string, - public analyticUnits: AnalyticUnitId[], - public id?: PanelId - ) { - if(this.panelUrl === undefined) { - throw new Error('panelUrl is undefined'); - } - } - - public toObject() { - return { - _id: this.id, - panelUrl: this.panelUrl, - analyticUnits: this.analyticUnits - }; - } - - static fromObject(obj: any): Panel { - if(obj === undefined) { - throw new Error('obj is undefined'); - } - return new Panel( - obj.panelUrl, - obj.analyticUnits, - obj._id - ); - } -} - -export type FindOneQuery = { - panelUrl: string -} - -export async function findOne(query: FindOneQuery): Promise { - let panel = await db.findOne(query); - if(panel === null) { - return null; - } - return Panel.fromObject(panel); -} - -export async function insertAnalyticUnit(panelUrl: string, analyticUnitId: AnalyticUnitId) { - const panel = await db.findOne({ panelUrl }); - - return db.updateOne({ panelUrl }, { - analyticUnits: panel.analyticUnits.concat(analyticUnitId) - }); -} - -export async function removeAnalyticUnit(panelUrl: string, analyticUnitId: AnalyticUnitId) { - const panel = await db.findOne({ panelUrl }); - - return db.updateOne({ panelUrl }, { - analyticUnits: panel.analyticUnits.filter(analyticUnit => analyticUnit !== analyticUnitId) - }); -} diff --git a/server/src/routes/analytic_units_router.ts b/server/src/routes/analytic_units_router.ts index f7a4303..f2bb4f3 100644 --- a/server/src/routes/analytic_units_router.ts +++ b/server/src/routes/analytic_units_router.ts @@ -26,36 +26,22 @@ async function getStatus(ctx: Router.IRouterContext) { } } -async function getUnit(ctx: Router.IRouterContext) { - let analyticUnitId = ctx.request.query.id; - if(analyticUnitId === undefined) { - throw new Error('No id param in query'); - } - - let analyticUnit = await AnalyticUnit.findById(analyticUnitId); - if(analyticUnit === null) { - throw new Error(`Cannot find analytic unit with id ${analyticUnitId}`); - } - - ctx.response.body = { - name: analyticUnit.name, - metric: analyticUnit.metric, - status: analyticUnit.status - }; -} - async function getUnits(ctx: Router.IRouterContext) { - const panelUrl = ctx.request.query.panelUrl; - if(panelUrl === undefined) { - throw new Error('Cannot get alerts of undefined panelUrl'); + const panelId = ctx.request.query.panelId; + if(panelId === undefined) { + throw new Error('Cannot get units of undefined panelId'); } - let analyticUnits = await AnalyticUnit.findMany({ panelUrl }); + let analyticUnits = await AnalyticUnit.findMany({ panelId }); if(analyticUnits === null) { analyticUnits = []; } - ctx.response.body = { analyticUnits }; + const analyticUnitObjects = analyticUnits.map(analyticUnit => analyticUnit.toPanelObject()); + + ctx.response.body = { + analyticUnits: analyticUnitObjects + }; } function getTypes(ctx: Router.IRouterContext) { @@ -63,18 +49,18 @@ function getTypes(ctx: Router.IRouterContext) { } async function createUnit(ctx: Router.IRouterContext) { - let id = await createAnalyticUnitFromObject(ctx.request.body); + const id = await createAnalyticUnitFromObject(ctx.request.body); + ctx.response.body = { id }; } async function updateUnit(ctx: Router.IRouterContext) { - const unit = ctx.request.body as AnalyticUnit.AnalyticUnit; - if(unit.id === undefined) { + const analyticUnit = ctx.request.body as AnalyticUnit.AnalyticUnit; + if(analyticUnit.id === undefined) { throw new Error('Cannot update undefined id'); } - // TODO: we can't allow to update everything - AnalyticUnit.update(unit.id, unit); + AnalyticUnit.update(analyticUnit.id, analyticUnit); ctx.response.body = { code: 200, message: 'Success' @@ -147,7 +133,6 @@ async function runDetect(ctx: Router.IRouterContext) { export var router = new Router(); -router.get('/', getUnit); router.get('/units', getUnits); router.get('/status', getStatus); router.get('/types', getTypes); diff --git a/server/src/routes/panel_router.ts b/server/src/routes/panel_router.ts deleted file mode 100644 index 5ee98fb..0000000 --- a/server/src/routes/panel_router.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { AnalyticUnitId } from '../models/analytic_unit_model'; -import * as Panel from '../models/panel_model'; - -import * as Router from 'koa-router'; - - -async function getAnalyticUnits(ctx: Router.IRouterContext) { - let panelUrl: string = ctx.request.query.panelUrl; - if(panelUrl === undefined || panelUrl === '') { - throw new Error('panelUrl is missing'); - } - const analyticUnits = await Panel.findOne({ panelUrl }); - ctx.response.body = { analyticUnits }; -} - -async function addAnalyticUnit(ctx: Router.IRouterContext) { - let { panelUrl, analyticUnitId } = ctx.request.body as { - panelUrl: string, analyticUnitId: AnalyticUnitId - }; - await Panel.insertAnalyticUnit(panelUrl, analyticUnitId); - ctx.response.body = { - code: 200, - message: 'Success' - }; -} - -async function deleteAnalyticUnit(ctx: Router.IRouterContext) { - let { panelUrl, analyticUnitId } = ctx.request.body as { - panelUrl: string, analyticUnitId: AnalyticUnitId - }; - // TODO: stop task when analytic unit is removed - await Panel.removeAnalyticUnit(panelUrl, analyticUnitId); - ctx.response.body = { - code: 200, - message: 'Success' - }; -} - -export const router = new Router(); - -router.get('/', getAnalyticUnits); -router.post('/', addAnalyticUnit); -router.delete('/', deleteAnalyticUnit); diff --git a/server/src/services/alert_service.ts b/server/src/services/alert_service.ts index 5873a78..5b811f7 100644 --- a/server/src/services/alert_service.ts +++ b/server/src/services/alert_service.ts @@ -22,7 +22,7 @@ export class Alert { analyticUnitType: this.analyticUnit.type, analyticUnitName: this.analyticUnit.name, analyticUnitId: this.analyticUnit.id, - panelUrl: this.analyticUnit.panelUrl, + grafanaUrl: this.analyticUnit.grafanaUrl, from: segment.from, to: segment.to }; diff --git a/server/src/services/analytics_service.ts b/server/src/services/analytics_service.ts index c0b3da0..01813ab 100644 --- a/server/src/services/analytics_service.ts +++ b/server/src/services/analytics_service.ts @@ -178,6 +178,7 @@ export class AnalyticsService { private _onAnalyticsUp() { const msg = 'Analytics is up'; for(let i in _.range(this._queue.length)) { + // TODO: check if task is done before removing it from the queue this.sendTask(this._queue.shift(), true); } console.log(msg); diff --git a/server/src/services/data_puller.ts b/server/src/services/data_puller.ts index 2f6724c..13ef936 100644 --- a/server/src/services/data_puller.ts +++ b/server/src/services/data_puller.ts @@ -65,14 +65,14 @@ export class DataPuller { throw Error(`data puller: can't pull undefined unit`); } - let panelUrl; + let grafanaUrl; if(GRAFANA_URL !== null) { - panelUrl = GRAFANA_URL; + grafanaUrl = GRAFANA_URL; } else { - panelUrl = unit.panelUrl; + grafanaUrl = unit.grafanaUrl; } - let data = queryByMetric(unit.metric, panelUrl, from, to, HASTIC_API_KEY); + let data = queryByMetric(unit.metric, grafanaUrl, from, to, HASTIC_API_KEY); return data; } diff --git a/server/src/services/data_service.ts b/server/src/services/data_service.ts index 22aca70..71eddda 100644 --- a/server/src/services/data_service.ts +++ b/server/src/services/data_service.ts @@ -4,7 +4,7 @@ import * as nedb from 'nedb'; import * as fs from 'fs'; -export enum Collection { ANALYTIC_UNITS, SEGMENTS, ANALYTIC_UNIT_CACHES, PANELS, THRESHOLD }; +export enum Collection { ANALYTIC_UNITS, ANALYTIC_UNIT_CACHES, SEGMENTS, THRESHOLD }; /** @@ -209,7 +209,6 @@ checkDataFolders(); // TODO: it's better if models request db which we create if it`s needed db.set(Collection.ANALYTIC_UNITS, new nedb({ filename: config.ANALYTIC_UNITS_DATABASE_PATH, autoload: true })); -db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true })); db.set(Collection.ANALYTIC_UNIT_CACHES, new nedb({ filename: config.ANALYTIC_UNIT_CACHES_DATABASE_PATH, autoload: true })); -db.set(Collection.PANELS, new nedb({ filename: config.PANELS_DATABASE_PATH, autoload: true })); +db.set(Collection.SEGMENTS, new nedb({ filename: config.SEGMENTS_DATABASE_PATH, autoload: true })); db.set(Collection.THRESHOLD, new nedb({ filename: config.THRESHOLD_DATABASE_PATH, autoload: true })); diff --git a/server/src/services/notification_service.ts b/server/src/services/notification_service.ts index 9f0133f..6f94ec2 100644 --- a/server/src/services/notification_service.ts +++ b/server/src/services/notification_service.ts @@ -22,7 +22,7 @@ export declare type AnalyticAlert = { analyticUnitType: string, analyticUnitName: string, analyticUnitId: AnalyticUnit.AnalyticUnitId, - panelUrl: string, + grafanaUrl: string, from: number, to: number params?: any,