diff --git a/server/src/index.ts b/server/src/index.ts index 6b51639..3f8f07f 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -3,7 +3,7 @@ import * as Router from 'koa-router'; import * as bodyParser from 'koa-bodyparser'; -import { router as anomaliesRouter } from './routes/anomalies'; +import { router as anomaliesRouter } from './routes/analytic_units'; import { router as segmentsRouter } from './routes/segments'; import { router as alertsRouter } from './routes/alerts'; diff --git a/server/src/models/analytic_unit.ts b/server/src/models/analytic_unit.ts new file mode 100644 index 0000000..69f963f --- /dev/null +++ b/server/src/models/analytic_unit.ts @@ -0,0 +1,107 @@ +import * as path from 'path' +import { getJsonDataSync, writeJsonDataSync } from './json' +import { ANOMALIES_PATH } from '../config' +import * as fs from 'fs' +import * as crypto from 'crypto'; + +export type Datasource = { + method: string, + data: Object, + params: Object, + type: string, + url: string +} + +export type Metric = { + datasource: string, + targets: string[] +} + +export type AnalyticUnit = { + name: string, + + panelUrl: string, + + pattern: string, + metric: Metric, + datasource: Datasource + status: string, + error?: string, + + lastPredictionTime: number, + nextId: number +} + +export type AnomalyUnitKey = string; + +let anomaliesNameToIdMap = {}; + + +function insertAnomaly(item: AnalyticUnit): AnomalyUnitKey { + const hashString = item.name + (new Date()).toString(); + const predictorId: AnomalyUnitKey = crypto.createHash('md5').update(hashString).digest('hex'); + anomaliesNameToIdMap[item.name] = predictorId; + let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); + if(fs.existsSync(filename)) { + return null; + } + saveAnomaly(predictorId, item); + return predictorId; +} + +function removeItem(predictorId: AnomalyUnitKey) { + let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); + fs.unlinkSync(filename); +} + +function saveAnomaly(predictorId: AnomalyUnitKey, anomaly: AnalyticUnit) { + let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); + return writeJsonDataSync(filename, anomaly); +} + +function loadPredictorById(predictorId: AnomalyUnitKey): AnalyticUnit { + let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); + if(!fs.existsSync(filename)) { + return null; + } + return getJsonDataSync(filename); +} + +function saveAnomalyTypeInfo(info) { + console.log('Saving'); + let filename = path.join(ANOMALIES_PATH, `${info.name}.json`); + if(info.next_id === undefined) { + info.next_id = 0; + } + if(info.last_prediction_time === undefined) { + info.last_prediction_time = 0; + } + + return writeJsonDataSync(filename, info); +} + +function getAnomalyTypeInfo(name) { + return getJsonDataSync(path.join(ANOMALIES_PATH, `${name}.json`)); +} + +function setAnomalyStatus(predictorId: AnomalyUnitKey, status: string, error?: string) { + let info = loadPredictorById(predictorId); + info.status = status; + if(error !== undefined) { + info.error = error; + } else { + info.error = ''; + } + saveAnomaly(predictorId, info); +} + +function setAnomalyPredictionTime(predictorId: AnomalyUnitKey, lastPredictionTime: number) { + let info = loadPredictorById(predictorId); + info.lastPredictionTime = lastPredictionTime; + saveAnomaly(predictorId, info); +} + +export { + saveAnomaly, loadPredictorById, insertAnomaly, removeItem, saveAnomalyTypeInfo, + getAnomalyTypeInfo, setAnomalyStatus, setAnomalyPredictionTime +} diff --git a/server/src/routes/alerts.ts b/server/src/routes/alerts.ts index bd14b3f..6fa3f61 100644 --- a/server/src/routes/alerts.ts +++ b/server/src/routes/alerts.ts @@ -1,4 +1,4 @@ -import { PredictorId, getPredictorIdByName, loadAnomalyById } from '../services/anomalyType'; +import { AnomalyUnitKey, loadPredictorById } from '../models/analytic_unit'; import { getAlertsAnomalies, saveAlertsAnomalies } from '../services/alerts'; import * as Router from 'koa-router'; @@ -6,11 +6,7 @@ import * as Router from 'koa-router'; function getAlert(ctx: Router.IRouterContext) { - let predictorId: PredictorId = ctx.request.query.predictor_id.toLowerCase(); - let anomaly = loadAnomalyById(predictorId) - if(anomaly == null) { - predictorId = getPredictorIdByName(predictorId); - } + let predictorId: AnomalyUnitKey = ctx.request.query.predictor_id.toLowerCase(); let alertsAnomalies = getAlertsAnomalies(); let pos = alertsAnomalies.indexOf(predictorId); @@ -22,12 +18,12 @@ function getAlert(ctx: Router.IRouterContext) { function changeAlert(ctx: Router.IRouterContext) { - let predictorId: PredictorId = ctx.request.body.predictor_id.toLowerCase(); + let predictorId: AnomalyUnitKey = ctx.request.body.predictor_id.toLowerCase(); let enable: boolean = ctx.request.body.enable; - let anomaly = loadAnomalyById(predictorId) - if(anomaly == null) { - predictorId = getPredictorIdByName(predictorId); + let predictor = loadPredictorById(predictorId) + if(predictor == null) { + throw new Error('Predctor is null'); } let alertsAnomalies = getAlertsAnomalies(); diff --git a/server/src/routes/anomalies.ts b/server/src/routes/analytic_units.ts similarity index 74% rename from server/src/routes/anomalies.ts rename to server/src/routes/analytic_units.ts index e1996f9..afdb9bf 100644 --- a/server/src/routes/anomalies.ts +++ b/server/src/routes/analytic_units.ts @@ -3,10 +3,10 @@ import * as Router from 'koa-router'; import { Datasource, Metric, - Anomaly, - saveAnomaly, - insertAnomaly, removeAnomaly, loadAnomalyByName, loadAnomalyById, getPredictorIdByName -} from '../services/anomalyType'; + AnalyticUnit, + + insertAnomaly, removeItem, loadPredictorById +} from '../models/analytic_unit'; import { runLearning } from '../services/analytics' import { saveTargets } from '../services/metrics'; @@ -14,12 +14,12 @@ async function sendAnomalyTypeStatus(ctx: Router.IRouterContext) { let id = ctx.request.query.id; let name = ctx.request.query.name.toLowerCase(); try { - let anomaly: Anomaly; - if(id !== undefined) { - anomaly = loadAnomalyById(id); - } else { - anomaly = loadAnomalyByName(name); + let anomaly: AnalyticUnit; + if(id === undefined) { + throw new Error('Id is undefined'); } + anomaly = loadPredictorById(id); + if(anomaly === null) { ctx.response.status = 404; return; @@ -37,27 +37,32 @@ async function sendAnomalyTypeStatus(ctx: Router.IRouterContext) { } -async function getAnomaly(ctx: Router.IRouterContext) { +async function getAnalyticUnit(ctx: Router.IRouterContext) { try { let id = ctx.request.query.id; let name = ctx.request.query.name.toLowerCase(); - let anomaly:Anomaly; - if(id !== undefined) { - anomaly = loadAnomalyById(id); - } else { - anomaly = loadAnomalyByName(name); + if(id === undefined) { + throw new Error('No id param in query'); } - if(anomaly === null) { + + if(name === undefined) { + throw new Error('No name param in query'); + } + + let unit: AnalyticUnit = loadPredictorById(id); + + if(unit === null) { ctx.response.status = 404; return; } ctx.response.body = { - name: anomaly.name, - metric: anomaly.metric, - status: anomaly.status + name: unit.name, + metric: unit.metric, + status: unit.status }; + } catch(e) { console.error(e); // TODO: better send 404 when we know than isn`t found @@ -66,7 +71,7 @@ async function getAnomaly(ctx: Router.IRouterContext) { } } -async function createAnomaly(ctx: Router.IRouterContext) { +async function createAnalyticUnit(ctx: Router.IRouterContext) { try { let body = ctx.request.body; const metric:Metric = { @@ -74,15 +79,15 @@ async function createAnomaly(ctx: Router.IRouterContext) { targets: saveTargets(body.metric.targets) }; - const anomaly:Anomaly = { + const anomaly:AnalyticUnit = { name: body.name.toLowerCase(), panelUrl: body.panelUrl, pattern: body.pattern.toLowerCase(), metric: metric, datasource: body.datasource, status: 'learning', - last_prediction_time: 0, - next_id: 0 + lastPredictionTime: 0, + nextId: 0 }; let predictorId = insertAnomaly(anomaly); if(predictorId === null) { @@ -111,11 +116,11 @@ function deleteAnomaly(ctx: Router.IRouterContext) { let name = ctx.request.query.name.toLowerCase(); if(id !== undefined) { - removeAnomaly(id); + removeItem(id); } else { - removeAnomaly(name); + removeItem(name); } - + ctx.response.body = { code: 200, message: 'Success' @@ -133,6 +138,6 @@ function deleteAnomaly(ctx: Router.IRouterContext) { export var router = new Router(); router.get('/status', sendAnomalyTypeStatus); -router.get('/', getAnomaly); -router.post('/', createAnomaly); +router.get('/', getAnalyticUnit); +router.post('/', createAnalyticUnit); router.delete('/', deleteAnomaly); diff --git a/server/src/routes/segments.ts b/server/src/routes/segments.ts index 9975a86..b751cca 100644 --- a/server/src/routes/segments.ts +++ b/server/src/routes/segments.ts @@ -7,16 +7,16 @@ import { } from '../services/segments'; import { - Anomaly, PredictorId, getPredictorIdByName, loadAnomalyById -} from '../services/anomalyType'; + AnalyticUnit, AnomalyUnitKey, getPredictorIdByName, loadPredictorById +} from '../models/analytic_unit'; import { runLearning } from '../services/analytics'; async function sendSegments(ctx: Router.IRouterContext) { - let predictorId: PredictorId = ctx.request.query.predictor_id.toLowerCase(); - let anomaly:Anomaly = loadAnomalyById(predictorId); + let predictorId: AnomalyUnitKey = ctx.request.query.predictor_id.toLowerCase(); + let anomaly:AnalyticUnit = loadPredictorById(predictorId); if(anomaly === null) { predictorId = getPredictorIdByName(predictorId); } diff --git a/server/src/services/alerts.ts b/server/src/services/alerts.ts index 5250e48..6cbe463 100644 --- a/server/src/services/alerts.ts +++ b/server/src/services/alerts.ts @@ -1,5 +1,5 @@ import { getJsonDataSync, writeJsonDataSync } from './json'; -import { PredictorId } from './anomalyType'; +import { AnomalyUnitKey } from '../models/analytic_unit'; import { runPredict } from './analytics'; import { sendNotification } from './notification'; import { getLabeledSegments } from './segments'; @@ -13,14 +13,14 @@ import * as fs from 'fs'; const ALERTS_DB_PATH = path.join(ANOMALIES_PATH, `alerts_anomalies.json`); -function getAlertsAnomalies(): PredictorId[] { +function getAlertsAnomalies(): AnomalyUnitKey[] { if(!fs.existsSync(ALERTS_DB_PATH)) { saveAlertsAnomalies([]); } return getJsonDataSync(ALERTS_DB_PATH); } -function saveAlertsAnomalies(anomalies: PredictorId[]) { +function saveAlertsAnomalies(anomalies: AnomalyUnitKey[]) { return writeJsonDataSync(ALERTS_DB_PATH, anomalies); } diff --git a/server/src/services/analytics.ts b/server/src/services/analytics.ts index ef9917e..4918216 100644 --- a/server/src/services/analytics.ts +++ b/server/src/services/analytics.ts @@ -1,13 +1,13 @@ import { - Anomaly, - PredictorId, getAnomalyTypeInfo, - loadAnomalyById, + AnalyticUnit, + AnomalyUnitKey, getAnomalyTypeInfo, + loadPredictorById, setAnomalyPredictionTime, setAnomalyStatus -} from './anomalyType' +} from '../models/analytic_unit' import { getTarget } from './metrics'; import { getLabeledSegments, insertSegments, removeSegments } from './segments' -import { AnalyticsConnection } from './analyticsConnection' +import { AnalyticsConnection } from './analytics_сonnection' const taskMap = {}; @@ -28,7 +28,7 @@ function onResponse(response: any) { } async function runTask(task): Promise { - let anomaly: Anomaly = loadAnomalyById(task.predictor_id); + let anomaly: AnalyticUnit = loadPredictorById(task.predictor_id); task.metric = { datasource: anomaly.metric.datasource, targets: anomaly.metric.targets.map(t => getTarget(t)) @@ -42,10 +42,10 @@ async function runTask(task): Promise { }) } -export async function runLearning(predictorId:PredictorId) { +export async function runLearning(predictorId:AnomalyUnitKey) { let segments = getLabeledSegments(predictorId); setAnomalyStatus(predictorId, 'learning'); - let anomaly:Anomaly = loadAnomalyById(predictorId); + let anomaly:AnalyticUnit = loadPredictorById(predictorId); let pattern = anomaly.pattern; let task = { type: 'learn', @@ -65,14 +65,14 @@ export async function runLearning(predictorId:PredictorId) { } } -export async function runPredict(predictorId:PredictorId) { - let anomaly:Anomaly = loadAnomalyById(predictorId); +export async function runPredict(predictorId:AnomalyUnitKey) { + let anomaly:AnalyticUnit = loadPredictorById(predictorId); let pattern = anomaly.pattern; let task = { type: 'predict', predictor_id: predictorId, pattern, - last_prediction_time: anomaly.last_prediction_time + last_prediction_time: anomaly.lastPredictionTime }; let result = await runTask(task); diff --git a/server/src/services/analyticsConnection.ts b/server/src/services/analytics_connection.ts similarity index 100% rename from server/src/services/analyticsConnection.ts rename to server/src/services/analytics_connection.ts diff --git a/server/src/services/anomalyType.ts b/server/src/services/anomalyType.ts deleted file mode 100644 index 5325685..0000000 --- a/server/src/services/anomalyType.ts +++ /dev/null @@ -1,134 +0,0 @@ -import * as path from 'path' -import { getJsonDataSync, writeJsonDataSync } from './json' -import { ANOMALIES_PATH } from '../config' -import * as fs from 'fs' -import * as crypto from 'crypto'; - -export type Datasource = { - method: string, - data: Object, - params: Object, - type: string, - url: string -} - -export type Metric = { - datasource: string, - targets: string[] -} - -export type Anomaly = { - name: string, - - panelUrl: string, - - pattern: string, - metric: Metric, - datasource: Datasource - status: string, - error?: string, - - last_prediction_time: number, - next_id: number -} - -export type PredictorId = string; - -let anomaliesNameToIdMap = {}; - -function loadAnomaliesMap() { - let filename = path.join(ANOMALIES_PATH, `all_anomalies.json`); - if(!fs.existsSync(filename)) { - saveAnomaliesMap(); - } - anomaliesNameToIdMap = getJsonDataSync(filename); -} - -function saveAnomaliesMap() { - let filename = path.join(ANOMALIES_PATH, `all_anomalies.json`); - writeJsonDataSync(filename, anomaliesNameToIdMap); -} - -function getPredictorIdByName(anomalyName:string): PredictorId { - loadAnomaliesMap(); - anomalyName = anomalyName.toLowerCase(); - if(anomalyName in anomaliesNameToIdMap) { - return anomaliesNameToIdMap[anomalyName]; - } - return anomalyName; -} - -function insertAnomaly(anomaly: Anomaly): PredictorId { - const hashString = anomaly.name + (new Date()).toString(); - const predictorId:PredictorId = crypto.createHash('md5').update(hashString).digest('hex'); - anomaliesNameToIdMap[anomaly.name] = predictorId; - saveAnomaliesMap(); - let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); - if(fs.existsSync(filename)) { - return null; - } - saveAnomaly(predictorId, anomaly); - return predictorId; -} - -function removeAnomaly(predictorId: PredictorId) { - let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); - fs.unlinkSync(filename); -} - -function saveAnomaly(predictorId: PredictorId, anomaly: Anomaly) { - let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); - return writeJsonDataSync(filename, anomaly); -} - -function loadAnomalyById(predictorId: PredictorId): Anomaly { - let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); - if(!fs.existsSync(filename)) { - return null; - } - return getJsonDataSync(filename); -} - -function loadAnomalyByName(anomalyName: string): Anomaly { - let predictorId = getPredictorIdByName(anomalyName); - return loadAnomalyById(predictorId); -} - -function saveAnomalyTypeInfo(info) { - console.log('Saving'); - let filename = path.join(ANOMALIES_PATH, `${info.name}.json`); - if(info.next_id === undefined) { - info.next_id = 0; - } - if(info.last_prediction_time === undefined) { - info.last_prediction_time = 0; - } - - return writeJsonDataSync(filename, info); -} - -function getAnomalyTypeInfo(name) { - return getJsonDataSync(path.join(ANOMALIES_PATH, `${name}.json`)); -} - -function setAnomalyStatus(predictorId: PredictorId, status: string, error?: string) { - let info = loadAnomalyById(predictorId); - info.status = status; - if(error !== undefined) { - info.error = error; - } else { - info.error = ''; - } - saveAnomaly(predictorId, info); -} - -function setAnomalyPredictionTime(predictorId: PredictorId, lastPredictionTime: number) { - let info = loadAnomalyById(predictorId); - info.last_prediction_time = lastPredictionTime; - saveAnomaly(predictorId, info); -} - -export { - saveAnomaly, loadAnomalyById, loadAnomalyByName, insertAnomaly, removeAnomaly, saveAnomalyTypeInfo, - getAnomalyTypeInfo, getPredictorIdByName, setAnomalyStatus, setAnomalyPredictionTime -} diff --git a/server/src/services/notification.ts b/server/src/services/notification.ts index 8430352..b39e02b 100644 --- a/server/src/services/notification.ts +++ b/server/src/services/notification.ts @@ -1,8 +1,8 @@ import axios from 'axios'; -import { loadAnomalyById } from './anomalyType'; +import { loadPredictorById } from '../models/analytic_unit'; export async function sendNotification(predictorId, active) { - let anomalyName = loadAnomalyById(predictorId).name; + let anomalyName = loadPredictorById(predictorId).name; console.log('Notification ' + anomalyName); let notification = { diff --git a/server/src/services/segments.ts b/server/src/services/segments.ts index ab1a3bd..25f36d5 100644 --- a/server/src/services/segments.ts +++ b/server/src/services/segments.ts @@ -2,11 +2,11 @@ import * as path from 'path'; import * as fs from 'fs'; import { getJsonDataSync, writeJsonDataSync } from './json'; import { SEGMENTS_PATH } from '../config'; -import { PredictorId, loadAnomalyById, saveAnomaly } from './anomalyType'; +import { AnomalyUnitKey, loadPredictorById, saveAnomaly } from '../models/analytic_unit'; import * as _ from 'lodash'; -function getLabeledSegments(predictorId: PredictorId) { +function getLabeledSegments(predictorId: AnomalyUnitKey) { let filename = path.join(SEGMENTS_PATH, `${predictorId}_labeled.json`); if(!fs.existsSync(filename)) { @@ -22,7 +22,7 @@ function getLabeledSegments(predictorId: PredictorId) { } } -function getPredictedSegments(predictorId: PredictorId) { +function getPredictedSegments(predictorId: AnomalyUnitKey) { let filename = path.join(SEGMENTS_PATH, `${predictorId}_segments.json`); let jsonData; @@ -35,7 +35,7 @@ function getPredictedSegments(predictorId: PredictorId) { return jsonData; } -function saveSegments(predictorId: PredictorId, segments) { +function saveSegments(predictorId: AnomalyUnitKey, segments) { let filename = path.join(SEGMENTS_PATH, `${predictorId}_labeled.json`); try { @@ -46,9 +46,9 @@ function saveSegments(predictorId: PredictorId, segments) { } } -function insertSegments(predictorId: PredictorId, addedSegments, labeled:boolean) { +function insertSegments(predictorId: AnomalyUnitKey, addedSegments, labeled:boolean) { // Set status - let info = loadAnomalyById(predictorId); + let info = loadPredictorById(predictorId); let segments = getLabeledSegments(predictorId); let nextId = info.next_id; @@ -66,7 +66,7 @@ function insertSegments(predictorId: PredictorId, addedSegments, labeled:boolean return addedIds; } -function removeSegments(predictorId: PredictorId, removedSegments) { +function removeSegments(predictorId: AnomalyUnitKey, removedSegments) { let segments = getLabeledSegments(predictorId); for (let segmentId of removedSegments) { segments = segments.filter(el => el.id !== segmentId);