diff --git a/server/src/models/analytic_unit.ts b/server/src/models/analytic_unit.ts index 69f963f..4bf00b0 100644 --- a/server/src/models/analytic_unit.ts +++ b/server/src/models/analytic_unit.ts @@ -1,9 +1,12 @@ -import * as path from 'path' -import { getJsonDataSync, writeJsonDataSync } from './json' +import { getJsonDataSync, writeJsonDataSync } from '../services/json' import { ANOMALIES_PATH } from '../config' -import * as fs from 'fs' + import * as crypto from 'crypto'; +import * as path from 'path' +import * as fs from 'fs' + + export type Datasource = { method: string, data: Object, @@ -32,34 +35,31 @@ export type AnalyticUnit = { nextId: number } -export type AnomalyUnitKey = string; - -let anomaliesNameToIdMap = {}; +export type AnalyticUnitId = string; -function insertAnomaly(item: AnalyticUnit): AnomalyUnitKey { +function createItem(item: AnalyticUnit): AnalyticUnitId { const hashString = item.name + (new Date()).toString(); - const predictorId: AnomalyUnitKey = crypto.createHash('md5').update(hashString).digest('hex'); - anomaliesNameToIdMap[item.name] = predictorId; - let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); + const newId: AnalyticUnitId = crypto.createHash('md5').update(hashString).digest('hex'); + let filename = path.join(ANOMALIES_PATH, `${newId}.json`); if(fs.existsSync(filename)) { return null; } - saveAnomaly(predictorId, item); - return predictorId; + save(newId, item); + return newId; } -function removeItem(predictorId: AnomalyUnitKey) { - let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); +function removeItem(key: AnalyticUnitId) { + let filename = path.join(ANOMALIES_PATH, `${key}.json`); fs.unlinkSync(filename); } -function saveAnomaly(predictorId: AnomalyUnitKey, anomaly: AnalyticUnit) { +function save(predictorId: AnalyticUnitId, anomaly: AnalyticUnit) { let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); return writeJsonDataSync(filename, anomaly); } -function loadPredictorById(predictorId: AnomalyUnitKey): AnalyticUnit { +function loadById(predictorId: AnalyticUnitId): AnalyticUnit { let filename = path.join(ANOMALIES_PATH, `${predictorId}.json`); if(!fs.existsSync(filename)) { return null; @@ -84,24 +84,24 @@ function getAnomalyTypeInfo(name) { return getJsonDataSync(path.join(ANOMALIES_PATH, `${name}.json`)); } -function setAnomalyStatus(predictorId: AnomalyUnitKey, status: string, error?: string) { - let info = loadPredictorById(predictorId); +function setAnomalyStatus(predictorId: AnalyticUnitId, status: string, error?: string) { + let info = loadById(predictorId); info.status = status; if(error !== undefined) { info.error = error; } else { info.error = ''; } - saveAnomaly(predictorId, info); + save(predictorId, info); } -function setAnomalyPredictionTime(predictorId: AnomalyUnitKey, lastPredictionTime: number) { - let info = loadPredictorById(predictorId); +function setAnomalyPredictionTime(predictorId: AnalyticUnitId, lastPredictionTime: number) { + let info = loadById(predictorId); info.lastPredictionTime = lastPredictionTime; - saveAnomaly(predictorId, info); + save(predictorId, info); } export { - saveAnomaly, loadPredictorById, insertAnomaly, removeItem, saveAnomalyTypeInfo, + save, loadById, createItem, removeItem, saveAnomalyTypeInfo, getAnomalyTypeInfo, setAnomalyStatus, setAnomalyPredictionTime } diff --git a/server/src/routes/alerts.ts b/server/src/routes/alerts.ts index 6fa3f61..0c54cb8 100644 --- a/server/src/routes/alerts.ts +++ b/server/src/routes/alerts.ts @@ -1,4 +1,4 @@ -import { AnomalyUnitKey, loadPredictorById } from '../models/analytic_unit'; +import { AnalyticUnitId, loadById } from '../models/analytic_unit'; import { getAlertsAnomalies, saveAlertsAnomalies } from '../services/alerts'; import * as Router from 'koa-router'; @@ -6,7 +6,7 @@ import * as Router from 'koa-router'; function getAlert(ctx: Router.IRouterContext) { - let predictorId: AnomalyUnitKey = ctx.request.query.predictor_id.toLowerCase(); + let predictorId: AnalyticUnitId = ctx.request.query.predictor_id.toLowerCase(); let alertsAnomalies = getAlertsAnomalies(); let pos = alertsAnomalies.indexOf(predictorId); @@ -18,10 +18,10 @@ function getAlert(ctx: Router.IRouterContext) { function changeAlert(ctx: Router.IRouterContext) { - let predictorId: AnomalyUnitKey = ctx.request.body.predictor_id.toLowerCase(); + let predictorId: AnalyticUnitId = ctx.request.body.predictor_id.toLowerCase(); let enable: boolean = ctx.request.body.enable; - let predictor = loadPredictorById(predictorId) + let predictor = loadById(predictorId) if(predictor == null) { throw new Error('Predctor is null'); } diff --git a/server/src/routes/analytic_units.ts b/server/src/routes/analytic_units.ts index afdb9bf..706862a 100644 --- a/server/src/routes/analytic_units.ts +++ b/server/src/routes/analytic_units.ts @@ -1,33 +1,28 @@ import * as Router from 'koa-router'; -import { - Datasource, - Metric, - AnalyticUnit, +import * as AnalyticUnit from '../models/analytic_unit'; - insertAnomaly, removeItem, loadPredictorById -} from '../models/analytic_unit'; import { runLearning } from '../services/analytics' import { saveTargets } from '../services/metrics'; -async function sendAnomalyTypeStatus(ctx: Router.IRouterContext) { +async function sendStatus(ctx: Router.IRouterContext) { let id = ctx.request.query.id; - let name = ctx.request.query.name.toLowerCase(); + let name = ctx.request.query.name; try { - let anomaly: AnalyticUnit; + let unit: AnalyticUnit.AnalyticUnit; if(id === undefined) { throw new Error('Id is undefined'); } - anomaly = loadPredictorById(id); + unit = AnalyticUnit.loadById(id); - if(anomaly === null) { + if(unit === null) { ctx.response.status = 404; return; } - if(anomaly.status === undefined) { + if(unit.status === undefined) { throw new Error('No status for ' + name); } - ctx.response.body = { status: anomaly.status, errorMessage: anomaly.error }; + ctx.response.body = { status: unit.status, errorMessage: unit.error }; } catch(e) { console.error(e); // TODO: better send 404 when we know than isn`t found @@ -37,20 +32,15 @@ async function sendAnomalyTypeStatus(ctx: Router.IRouterContext) { } -async function getAnalyticUnit(ctx: Router.IRouterContext) { +async function findItem(ctx: Router.IRouterContext) { try { let id = ctx.request.query.id; - let name = ctx.request.query.name.toLowerCase(); if(id === undefined) { throw new Error('No id param in query'); } - if(name === undefined) { - throw new Error('No name param in query'); - } - - let unit: AnalyticUnit = loadPredictorById(id); + let unit: AnalyticUnit.AnalyticUnit = AnalyticUnit.loadById(id); if(unit === null) { ctx.response.status = 404; @@ -67,19 +57,19 @@ async function getAnalyticUnit(ctx: Router.IRouterContext) { console.error(e); // TODO: better send 404 when we know than isn`t found ctx.response.status = 500; - ctx.response.body = 'Can`t get anything'; + ctx.response.body = 'Can`t find anything'; } } -async function createAnalyticUnit(ctx: Router.IRouterContext) { +async function createItem(ctx: Router.IRouterContext) { try { let body = ctx.request.body; - const metric:Metric = { + const metric: AnalyticUnit.Metric = { datasource: body.metric.datasource, targets: saveTargets(body.metric.targets) }; - const anomaly:AnalyticUnit = { + const unit: AnalyticUnit.AnalyticUnit = { name: body.name.toLowerCase(), panelUrl: body.panelUrl, pattern: body.pattern.toLowerCase(), @@ -89,36 +79,34 @@ async function createAnalyticUnit(ctx: Router.IRouterContext) { lastPredictionTime: 0, nextId: 0 }; - let predictorId = insertAnomaly(anomaly); - if(predictorId === null) { + + let newId = AnalyticUnit.createItem(unit); + if(newId === null) { ctx.response.status = 403; ctx.response.body = { code: 403, - message: 'Already exists' + message: 'Item exists' }; } - ctx.response.body = { predictor_id: predictorId }; + ctx.response.body = { id: newId }; - runLearning(predictorId); + runLearning(newId); } catch(e) { ctx.response.status = 500; ctx.response.body = { code: 500, - message: `Anomaly creation error: ${e.message}` + message: `Creation error: ${e.message}` }; } } -function deleteAnomaly(ctx: Router.IRouterContext) { +function deleteItem(ctx: Router.IRouterContext) { try { let id = ctx.request.query.id; - let name = ctx.request.query.name.toLowerCase(); if(id !== undefined) { - removeItem(id); - } else { - removeItem(name); + AnalyticUnit.removeItem(id); } ctx.response.body = { @@ -129,7 +117,7 @@ function deleteAnomaly(ctx: Router.IRouterContext) { ctx.response.status = 500; ctx.response.body = { code: 500, - message: `Anomaly deletion error: ${e.message}` + message: `Deletion error: ${e.message}` }; } } @@ -137,7 +125,7 @@ function deleteAnomaly(ctx: Router.IRouterContext) { export var router = new Router(); -router.get('/status', sendAnomalyTypeStatus); -router.get('/', getAnalyticUnit); -router.post('/', createAnalyticUnit); -router.delete('/', deleteAnomaly); +router.get('/status', sendStatus); +router.get('/', findItem); +router.post('/', createItem); +router.delete('/', deleteItem); diff --git a/server/src/routes/segments.ts b/server/src/routes/segments.ts index b751cca..ae15fbf 100644 --- a/server/src/routes/segments.ts +++ b/server/src/routes/segments.ts @@ -7,25 +7,25 @@ import { } from '../services/segments'; import { - AnalyticUnit, AnomalyUnitKey, getPredictorIdByName, loadPredictorById + AnalyticUnit, AnalyticUnitId, loadById } from '../models/analytic_unit'; import { runLearning } from '../services/analytics'; async function sendSegments(ctx: Router.IRouterContext) { + let id: AnalyticUnitId = ctx.request.query.id; + let unit: AnalyticUnit = loadById(id); - let predictorId: AnomalyUnitKey = ctx.request.query.predictor_id.toLowerCase(); - let anomaly:AnalyticUnit = loadPredictorById(predictorId); - if(anomaly === null) { - predictorId = getPredictorIdByName(predictorId); + if(unit === null) { + throw new Error(`Can't find Analitic unit with id ${id}`); } - let lastSegmentId = ctx.request.query.last_segment; + let lastSegmentId = ctx.request.query.lastSegmentId; let timeFrom = ctx.request.query.from; let timeTo = ctx.request.query.to; - let segments = getLabeledSegments(predictorId); + let segments = getLabeledSegments(id); // Id filtering if(lastSegmentId !== undefined) { @@ -49,19 +49,14 @@ async function updateSegments(ctx: Router.IRouterContext) { try { let segmentsUpdate = ctx.request.body; - let predictorId = segmentsUpdate.predictor_id; - let anomalyName = segmentsUpdate.name.toLowerCase(); + let key = segmentsUpdate.analyticUnitKey; - if(predictorId === undefined) { - predictorId = getPredictorIdByName(anomalyName); - } + let addedIds = insertSegments(key, segmentsUpdate.addedSegments, true); + removeSegments(key, segmentsUpdate.removedSegments); - let addedIds = insertSegments(predictorId, segmentsUpdate.added_segments, true); - removeSegments(predictorId, segmentsUpdate.removed_segments); + ctx.response.body = { addedIds }; - ctx.response.body = { added_ids: addedIds }; - - runLearning(predictorId); + runLearning(key); } catch(e) { ctx.response.status = 500; ctx.response.body = { diff --git a/server/src/services/alerts.ts b/server/src/services/alerts.ts index 6cbe463..bad4a33 100644 --- a/server/src/services/alerts.ts +++ b/server/src/services/alerts.ts @@ -1,5 +1,5 @@ import { getJsonDataSync, writeJsonDataSync } from './json'; -import { AnomalyUnitKey } from '../models/analytic_unit'; +import { AnalyticUnitId } from '../models/analytic_unit'; import { runPredict } from './analytics'; import { sendNotification } from './notification'; import { getLabeledSegments } from './segments'; @@ -13,14 +13,14 @@ import * as fs from 'fs'; const ALERTS_DB_PATH = path.join(ANOMALIES_PATH, `alerts_anomalies.json`); -function getAlertsAnomalies(): AnomalyUnitKey[] { +function getAlertsAnomalies(): AnalyticUnitId[] { if(!fs.existsSync(ALERTS_DB_PATH)) { saveAlertsAnomalies([]); } return getJsonDataSync(ALERTS_DB_PATH); } -function saveAlertsAnomalies(anomalies: AnomalyUnitKey[]) { +function saveAlertsAnomalies(anomalies: AnalyticUnitId[]) { return writeJsonDataSync(ALERTS_DB_PATH, anomalies); } diff --git a/server/src/services/analytics.ts b/server/src/services/analytics.ts index 3fe07a1..5d8600e 100644 --- a/server/src/services/analytics.ts +++ b/server/src/services/analytics.ts @@ -1,7 +1,7 @@ import { AnalyticUnit, - AnomalyUnitKey, getAnomalyTypeInfo, - loadPredictorById, + AnalyticUnitId, getAnomalyTypeInfo, + loadById, setAnomalyPredictionTime, setAnomalyStatus } from '../models/analytic_unit' @@ -28,7 +28,7 @@ function onResponse(response: any) { } async function runTask(task): Promise { - let anomaly: AnalyticUnit = loadPredictorById(task.predictor_id); + let anomaly: AnalyticUnit = loadById(task.predictor_id); task.metric = { datasource: anomaly.metric.datasource, targets: anomaly.metric.targets.map(t => getTarget(t)) @@ -42,10 +42,10 @@ async function runTask(task): Promise { }) } -export async function runLearning(predictorId:AnomalyUnitKey) { +export async function runLearning(predictorId:AnalyticUnitId) { let segments = getLabeledSegments(predictorId); setAnomalyStatus(predictorId, 'learning'); - let anomaly:AnalyticUnit = loadPredictorById(predictorId); + let anomaly:AnalyticUnit = loadById(predictorId); let pattern = anomaly.pattern; let task = { type: 'learn', @@ -65,8 +65,8 @@ export async function runLearning(predictorId:AnomalyUnitKey) { } } -export async function runPredict(predictorId:AnomalyUnitKey) { - let anomaly:AnalyticUnit = loadPredictorById(predictorId); +export async function runPredict(predictorId:AnalyticUnitId) { + let anomaly:AnalyticUnit = loadById(predictorId); let pattern = anomaly.pattern; let task = { type: 'predict', diff --git a/server/src/services/notification.ts b/server/src/services/notification.ts index b39e02b..6f57876 100644 --- a/server/src/services/notification.ts +++ b/server/src/services/notification.ts @@ -1,8 +1,8 @@ import axios from 'axios'; -import { loadPredictorById } from '../models/analytic_unit'; +import { loadById } from '../models/analytic_unit'; export async function sendNotification(predictorId, active) { - let anomalyName = loadPredictorById(predictorId).name; + let anomalyName = loadById(predictorId).name; console.log('Notification ' + anomalyName); let notification = { diff --git a/server/src/services/segments.ts b/server/src/services/segments.ts index 25f36d5..1817957 100644 --- a/server/src/services/segments.ts +++ b/server/src/services/segments.ts @@ -1,12 +1,14 @@ -import * as path from 'path'; -import * as fs from 'fs'; import { getJsonDataSync, writeJsonDataSync } from './json'; +import { AnalyticUnitId, loadById, save } from '../models/analytic_unit'; import { SEGMENTS_PATH } from '../config'; -import { AnomalyUnitKey, loadPredictorById, saveAnomaly } from '../models/analytic_unit'; import * as _ from 'lodash'; -function getLabeledSegments(predictorId: AnomalyUnitKey) { +import * as path from 'path'; +import * as fs from 'fs'; + + +export function getLabeledSegments(predictorId: AnalyticUnitId) { let filename = path.join(SEGMENTS_PATH, `${predictorId}_labeled.json`); if(!fs.existsSync(filename)) { @@ -22,7 +24,7 @@ function getLabeledSegments(predictorId: AnomalyUnitKey) { } } -function getPredictedSegments(predictorId: AnomalyUnitKey) { +export function getPredictedSegments(predictorId: AnalyticUnitId) { let filename = path.join(SEGMENTS_PATH, `${predictorId}_segments.json`); let jsonData; @@ -35,8 +37,8 @@ function getPredictedSegments(predictorId: AnomalyUnitKey) { return jsonData; } -function saveSegments(predictorId: AnomalyUnitKey, segments) { - let filename = path.join(SEGMENTS_PATH, `${predictorId}_labeled.json`); +export function saveSegments(id: AnalyticUnitId, segments) { + let filename = path.join(SEGMENTS_PATH, `${id}_labeled.json`); try { return writeJsonDataSync(filename, _.uniqBy(segments, 'start')); @@ -46,12 +48,12 @@ function saveSegments(predictorId: AnomalyUnitKey, segments) { } } -function insertSegments(predictorId: AnomalyUnitKey, addedSegments, labeled:boolean) { +export function insertSegments(id: AnalyticUnitId, addedSegments, labeled:boolean) { // Set status - let info = loadPredictorById(predictorId); - let segments = getLabeledSegments(predictorId); + let info = loadById(id); + let segments = getLabeledSegments(id); - let nextId = info.next_id; + let nextId = info.nextId; let addedIds = [] for (let segment of addedSegments) { segment.id = nextId; @@ -60,18 +62,16 @@ function insertSegments(predictorId: AnomalyUnitKey, addedSegments, labeled:bool nextId++; segments.push(segment); } - info.next_id = nextId; - saveSegments(predictorId, segments); - saveAnomaly(predictorId, info); + info.nextId = nextId; + saveSegments(id, segments); + save(id, info); return addedIds; } -function removeSegments(predictorId: AnomalyUnitKey, removedSegments) { +export function removeSegments(predictorId: AnalyticUnitId, removedSegments) { let segments = getLabeledSegments(predictorId); for (let segmentId of removedSegments) { segments = segments.filter(el => el.id !== segmentId); } saveSegments(predictorId, segments); } - -export { getLabeledSegments, getPredictedSegments, saveSegments, insertSegments, removeSegments }